11package org.modelix.workspace.manager
22
33import io.kubernetes.client.openapi.apis.BatchV1Api
4+ import io.kubernetes.client.openapi.apis.CoreV1Api
45import io.kubernetes.client.openapi.models.V1Job
56import io.kubernetes.client.openapi.models.V1Toleration
67import io.kubernetes.client.util.Yaml
@@ -18,6 +19,7 @@ import kotlin.time.Duration.Companion.minutes
1819abstract class KubernetesJobTask <Out : Any >(scope : CoroutineScope ) : TaskInstance<Out>(scope) {
1920 companion object {
2021 const val JOB_ID_LABEL = " modelix.workspace.job.id"
22+ private val LOG = mu.KotlinLogging .logger {}
2123 }
2224
2325 abstract suspend fun tryGetResult (): Out ?
@@ -52,10 +54,30 @@ abstract class KubernetesJobTask<Out : Any>(scope: CoroutineScope) : TaskInstanc
5254 }
5355 }
5456 checkNotNull(tryGetResult()) {
55- " Job finished without producing the expected result. Status : ${findJob()?.status?.let { Yaml .dump(it) }} "
57+ " Job finished without producing the expected result. \n Status : ${findJob()?.status?.let { Yaml .dump(it) }} \n Pod logs: \n ${getPodLogs() }"
5658 }
5759 }
5860
61+ fun getPodLogs (): String? {
62+ try {
63+ val coreApi = CoreV1Api ()
64+ val pods = coreApi.listNamespacedPod(KUBERNETES_NAMESPACE ).timeoutSeconds(10 ).execute()
65+ for (pod in pods.items) {
66+ if (pod.metadata!! .labels?.get(JOB_ID_LABEL ) != id.toString()) continue
67+ return coreApi
68+ .readNamespacedPodLog(pod.metadata!! .name, KUBERNETES_NAMESPACE )
69+ .container(pod.spec!! .containers[0 ].name)
70+ .pretty(" true" )
71+ .tailLines(10_000 )
72+ .execute()
73+ }
74+ } catch (e: Exception ) {
75+ LOG .error(" " , e)
76+ return null
77+ }
78+ return null
79+ }
80+
5981 private suspend fun createJob () {
6082 suspendCoroutine {
6183 val job = generateJobYaml().apply {
0 commit comments