From e961780d1b707bda42fe477dbcff55c469122380 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Bidoul?= <stephane.bidoul@gmail.com>
Date: Sun, 28 Nov 2021 13:30:39 +0100
Subject: [PATCH] Better way to kill jobs

---
 src/runboat/k8s.py    | 24 +++++++++++++-----------
 src/runboat/models.py |  8 +++++---
 2 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/src/runboat/k8s.py b/src/runboat/k8s.py
index 5c2a063..d78062c 100644
--- a/src/runboat/k8s.py
+++ b/src/runboat/k8s.py
@@ -242,18 +242,20 @@ async def delete_resources(build_name: str) -> None:
     )
 
 
-async def delete_job(build_name: str, job_kind: DeploymentMode) -> None:
+@sync_to_async
+def kill_job(build_name: str, job_kind: DeploymentMode) -> None:
     # TODO delete all resources with runboat/build and runboat/job-kind label
-    await _kubectl(
-        [
-            "-n",
-            settings.build_namespace,
-            "delete",
-            "job",
-            "-l",
-            f"runboat/build={build_name},runboat/job-kind={job_kind}",
-            "--wait=false",
-        ]
+    batchv1 = client.BatchV1Api()
+    batchv1.delete_collection_namespaced_job(
+        namespace=settings.build_namespace,
+        label_selector=f"runboat/build={build_name},runboat/job-kind={job_kind}",
+        grace_period_seconds=0,
+    )
+    corev1 = client.CoreV1Api()
+    corev1.delete_collection_namespaced_pod(
+        namespace=settings.build_namespace,
+        label_selector=f"runboat/build={build_name},runboat/job-kind={job_kind}",
+        grace_period_seconds=0,
     )
 
 
diff --git a/src/runboat/models.py b/src/runboat/models.py
index 6bc8aed..6ebe7cf 100644
--- a/src/runboat/models.py
+++ b/src/runboat/models.py
@@ -214,7 +214,7 @@ class Build(BaseModel):
             return
         elif self.status == BuildStatus.failed:
             _logger.info(f"Marking failed {self} for reinitialization.")
-            await k8s.delete_job(self.name, job_kind=k8s.DeploymentMode.initialize)
+            await k8s.kill_job(self.name, job_kind=k8s.DeploymentMode.initialize)
             if await self._patch(init_status=BuildInitStatus.todo, desired_replicas=0):
                 await github.notify_status(
                     self.commit_info.repo,
@@ -259,8 +259,10 @@ class Build(BaseModel):
 
     async def cleanup(self) -> None:
         """Launch the clenaup job."""
-        # Delete the initialization job to reduce conflict with the cleanup job.
-        await k8s.delete_job(self.name, job_kind=k8s.DeploymentMode.initialize)
+        # Kill the initialization job to reduce conflict with the cleanup job, such as
+        # the database being created by the initialization after the cleanup job has
+        # completed.
+        await k8s.kill_job(self.name, job_kind=k8s.DeploymentMode.initialize)
         # Be sure the deployment is stopped.
         await self._patch(desired_replicas=0, not_found_ok=True)
         # Start cleanup job. on_cleanup_{started,succeeded,failed} callbacks will follow