itsulu/runboat
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Improve k8s watcher error handling

Browse source
This commit is contained in:
Stéphane Bidoul 2021-11-11 17:52:35 +01:00
parent 3dbc251b94
commit f2f9a73ccd
No known key found for this signature in database
GPG key ID: BCAB2555446B5B92
2 changed files with 11 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
src/runboat/controller.py
View file

@ -143,8 +143,6 @@ class Controller:
await build.cleanup() await build.cleanup()
elif event_type == "DELETED": elif event_type == "DELETED":
should_wakeup = self.db.remove(build_name) should_wakeup = self.db.remove(build_name)
else:
_logger.error(f"Unexpected k8s event type {event_type}.")
if should_wakeup: if should_wakeup:
self._wakeup() self._wakeup()
@ -194,8 +192,6 @@ class Controller:
await build.on_cleanup_failed() await build.on_cleanup_failed()
elif event_type == "DELETED": elif event_type == "DELETED":
pass pass
else:
_logger.error(f"Unexpected k8s event type {event_type}.")
async def initializer(self) -> None: async def initializer(self) -> None:
while True: while True:

18
src/runboat/k8s.py
View file

@ -79,6 +79,7 @@ def _watch(list_method, *args, **kwargs):
# perform a first query # perform a first query
res = list_method(*args, **kwargs) res = list_method(*args, **kwargs)
resource_version = res.metadata.resource_version resource_version = res.metadata.resource_version
assert resource_version
for item in res.items: for item in res.items:
yield None, item yield None, item
# stream until timeout # stream until timeout
@ -91,13 +92,16 @@ def _watch(list_method, *args, **kwargs):
resource_version=resource_version, resource_version=resource_version,
_request_timeout=60, _request_timeout=60,
): ):
if event["type"] == "ERROR": event_type = event["type"]
raise RuntimeError("Kubernetes watch error") event_object = event["object"]
resource_version = event["object"].metadata.resource_version if event_type == "ERROR":
yield event["type"], event["object"] raise RuntimeError("Kubernetes watch ERROR")
except urllib3.exceptions.TimeoutError: elif event_type not in ("ADDED", "MODIFIED", "DELETED"):
continue raise RuntimeError(f"Unexpected event {event_type}")
except TimeoutError: resource_version = event_object.metadata.resource_version
assert resource_version
yield event_type, event_object
except (urllib3.exceptions.TimeoutError, TimeoutError):
continue continue
except Exception as e: except Exception as e:
delay = 5 delay = 5