itsulu/runboat
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve k8s watcher error handling

Browse source
This commit is contained in:
Stéphane Bidoul 2021-11-11 17:52:35 +01:00
parent 3dbc251b94
commit f2f9a73ccd
No known key found for this signature in database
GPG key ID: BCAB2555446B5B92
2 changed files with 11 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/runboat/controller.py
View file

@ -143,8 +143,6 @@ class Controller:
await build.cleanup()
elif event_type == "DELETED":
should_wakeup = self.db.remove(build_name)
else:
_logger.error(f"Unexpected k8s event type {event_type}.")
if should_wakeup:
self._wakeup()
@ -194,8 +192,6 @@ class Controller:
await build.on_cleanup_failed()
elif event_type == "DELETED":
pass
else:
_logger.error(f"Unexpected k8s event type {event_type}.")
async def initializer(self) -> None:
while True:

18
src/runboat/k8s.py
View file

@ -79,6 +79,7 @@ def _watch(list_method, *args, **kwargs):
# perform a first query
res = list_method(*args, **kwargs)
resource_version = res.metadata.resource_version
assert resource_version
for item in res.items:
yield None, item
# stream until timeout
@ -91,13 +92,16 @@ def _watch(list_method, *args, **kwargs):
resource_version=resource_version,
_request_timeout=60,
):
if event["type"] == "ERROR":
raise RuntimeError("Kubernetes watch error")
resource_version = event["object"].metadata.resource_version
yield event["type"], event["object"]
except urllib3.exceptions.TimeoutError:
continue
except TimeoutError:
event_type = event["type"]
event_object = event["object"]
if event_type == "ERROR":
raise RuntimeError("Kubernetes watch ERROR")
elif event_type not in ("ADDED", "MODIFIED", "DELETED"):
raise RuntimeError(f"Unexpected event {event_type}")
resource_version = event_object.metadata.resource_version
assert resource_version
yield event_type, event_object
except (urllib3.exceptions.TimeoutError, TimeoutError):
continue
except Exception as e:
delay = 5