itsulu/runboat
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Better way to kill jobs

Browse source
This commit is contained in:
Stéphane Bidoul 2021-11-28 13:30:39 +01:00
parent f720cbcac5
commit e961780d1b
No known key found for this signature in database
GPG key ID: BCAB2555446B5B92
2 changed files with 18 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
src/runboat/k8s.py
View file

@ -242,18 +242,20 @@ async def delete_resources(build_name: str) -> None:
) )
async def delete_job(build_name: str, job_kind: DeploymentMode) -> None: @sync_to_async
def kill_job(build_name: str, job_kind: DeploymentMode) -> None:
# TODO delete all resources with runboat/build and runboat/job-kind label # TODO delete all resources with runboat/build and runboat/job-kind label
await _kubectl( batchv1 = client.BatchV1Api()
[ batchv1.delete_collection_namespaced_job(
"-n", namespace=settings.build_namespace,
settings.build_namespace, label_selector=f"runboat/build={build_name},runboat/job-kind={job_kind}",
"delete", grace_period_seconds=0,
"job", )
"-l", corev1 = client.CoreV1Api()
f"runboat/build={build_name},runboat/job-kind={job_kind}", corev1.delete_collection_namespaced_pod(
"--wait=false", namespace=settings.build_namespace,
] label_selector=f"runboat/build={build_name},runboat/job-kind={job_kind}",
grace_period_seconds=0,
) )

8
src/runboat/models.py
View file

@ -214,7 +214,7 @@ class Build(BaseModel):
return return
elif self.status == BuildStatus.failed: elif self.status == BuildStatus.failed:
_logger.info(f"Marking failed {self} for reinitialization.") _logger.info(f"Marking failed {self} for reinitialization.")
await k8s.delete_job(self.name, job_kind=k8s.DeploymentMode.initialize) await k8s.kill_job(self.name, job_kind=k8s.DeploymentMode.initialize)
if await self._patch(init_status=BuildInitStatus.todo, desired_replicas=0): if await self._patch(init_status=BuildInitStatus.todo, desired_replicas=0):
await github.notify_status( await github.notify_status(
self.commit_info.repo, self.commit_info.repo,
@ -259,8 +259,10 @@ class Build(BaseModel):
async def cleanup(self) -> None: async def cleanup(self) -> None:
"""Launch the clenaup job.""" """Launch the clenaup job."""
# Delete the initialization job to reduce conflict with the cleanup job. # Kill the initialization job to reduce conflict with the cleanup job, such as
await k8s.delete_job(self.name, job_kind=k8s.DeploymentMode.initialize) # the database being created by the initialization after the cleanup job has
# completed.
await k8s.kill_job(self.name, job_kind=k8s.DeploymentMode.initialize)
# Be sure the deployment is stopped. # Be sure the deployment is stopped.
await self._patch(desired_replicas=0, not_found_ok=True) await self._patch(desired_replicas=0, not_found_ok=True)
# Start cleanup job. on_cleanup_{started,succeeded,failed} callbacks will follow # Start cleanup job. on_cleanup_{started,succeeded,failed} callbacks will follow