Merge pull request #18 from sbidoul/flex-config

More flexible configuration

.env.sample

@@ -1,4 +1,4 @@
-RUNBOAT_SUPPORTED_REPOS=["OCA/mis-builder", "shopinvader/odoo-shopinvader", "OCA/server-env"]
+RUNBOAT_REPOS=[{"repo": "^oca/.*", "branch": "^15.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}]}]
 RUNBOAT_API_ADMIN_USER="admin"
 RUNBOAT_API_ADMIN_PASSWD="admin"
 RUNBOAT_BUILD_NAMESPACE=runboat-builds

.env.test

@@ -1,4 +1,4 @@
-RUNBOAT_SUPPORTED_REPOS=["OCA/mis-builder", "shopinvader/odoo-shopinvader", "OCA/server-env"]
+RUNBOAT_REPOS=[{"repo": "^oca/.*", "branch": "^15.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}]}]
 RUNBOAT_API_ADMIN_USER="admin"
 RUNBOAT_API_ADMIN_PASSWD="admin"
 RUNBOAT_BUILD_NAMESPACE=runboat-builds
@@ -8,4 +8,3 @@ RUNBOAT_BUILD_SECRET_ENV={"PGPASSWORD": "thepgpassword"}
 RUNBOAT_BUILD_TEMPLATE_VARS={"storageClassName": "my-storage-class"}
 RUNBOAT_GITHUB_TOKEN=
 RUNBOAT_LOG_CONFIG=log-config.yaml
-RUNBOAT_BUILD_IMAGES={"15.0": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}

Dockerfile

@@ -10,8 +10,7 @@ RUN curl -L \
 COPY requirements.txt /tmp/requirements.txt
 RUN pip install --no-cache-dir -r /tmp/requirements.txt
 
-ENV RUNBOAT_SUPPORTED_REPOS='["OCA/mis-builder", "shopinvader/odoo-shopinvader", "OCA/server-env"]'
+ENV RUNBOAT_REPOS='[{"repo": "^oca/.*", "branch": "^15.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}]}]'
-ENV RUNBOAT_BUILD_IMAGES='{"15.0": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}'
 ENV RUNBOAT_API_ADMIN_USER="admin"
 ENV RUNBOAT_API_ADMIN_PASSWD="admin"
 ENV RUNBOAT_BUILD_NAMESPACE=runboat-builds

src/runboat/api.py

@@ -12,7 +12,6 @@ from starlette.status import HTTP_404_NOT_FOUND
 from . import github, models
 from .controller import Controller, controller
 from .deps import authenticated
-from .settings import settings
 
 router = APIRouter()
 
@@ -73,7 +72,7 @@ async def controller_status() -> Controller:
 
 @router.get("/repos", response_model=list[Repo])
 async def repos() -> list[models.Repo]:
-    return [models.Repo(name=name) for name in settings.supported_repos]
+    return controller.db.repos()
 
 
 @router.get(

src/runboat/build_images.py

@@ -1,39 +0,0 @@
-import re
-
-from .exceptions import BranchNotSupported
-from .settings import settings
-
-TARGET_BRANCH_RE = re.compile(r"^(\d+\.\d+)")
-
-
-def get_main_branch(branch_name: str) -> str:
-    mo = TARGET_BRANCH_RE.match(branch_name)
-    if not mo:
-        raise BranchNotSupported(
-            f"Malformed branch name {branch_name} "
-            f"(it should start with an Odoo branch name)."
-        )
-    key = mo.group(1)
-    if key not in settings.build_images:
-        raise BranchNotSupported(
-            f"No build image configured for {key} (from {branch_name})."
-        )
-    return key
-
-
-def is_branch_supported(branch_name: str) -> bool:
-    try:
-        return bool(get_main_branch(branch_name))
-    except BranchNotSupported:
-        return False
-
-
-def is_main_branch(branch_name: str) -> bool:
-    try:
-        return branch_name == get_main_branch(branch_name)
-    except BranchNotSupported:
-        return False
-
-
-def get_build_image(branch_name: str) -> str:
-    return settings.build_images[get_main_branch(branch_name)]

src/runboat/db.py

@@ -3,7 +3,7 @@ import sqlite3
 from typing import Iterator, Protocol, cast
 from weakref import WeakSet
 
-from .models import Build, BuildEvent, BuildInitStatus, BuildStatus
+from .models import Build, BuildEvent, BuildInitStatus, BuildStatus, Repo
 
 _logger = logging.getLogger(__name__)
 
@@ -184,6 +184,10 @@ class BuildsDb:
         ).fetchall()
         return [self._build_from_row(row) for row in rows]
 
+    def repos(self) -> list[Repo]:
+        rows = self._con.execute("SELECT DISTINCT repo FROM builds ORDER BY repo")
+        return [Repo(name=row[0]) for row in rows]
+
     def search(
         self,
         repo: str | None = None,

src/runboat/exceptions.py

@@ -14,5 +14,5 @@ class NotFoundOnGitHub(ClientError):
     pass
 
 
-class BranchNotSupported(ClientError):
+class RepoOrBranchNotSupported(ClientError):
     pass

src/runboat/models.py

@@ -8,9 +8,8 @@ from kubernetes.client.models.v1_deployment import V1Deployment
 from pydantic import BaseModel
 
 from . import github, k8s
-from .build_images import get_build_image
 from .github import GitHubStatusState
-from .settings import settings
+from .settings import get_build_settings, settings
 from .utils import slugify
 
 _logger = logging.getLogger(__name__)
@@ -175,7 +174,11 @@ class Build(BaseModel):
         name = f"b{uuid.uuid4()}"
         slug = cls.make_slug(repo, target_branch, pr, git_commit)
         _logger.info(f"Deploying {slug} ({name}).")
-        image = get_build_image(target_branch)
+        build_settings = get_build_settings(repo, target_branch)
+        if len(build_settings) > 1:
+            raise NotImplementedError(
+                "Having more than one build per commit is not supported yet."
+            )
         deployment_vars = k8s.make_deployment_vars(
             k8s.DeploymentMode.deployment,
             name,
@@ -184,7 +187,7 @@ class Build(BaseModel):
             target_branch,
             pr,
             git_commit,
-            image,
+            build_settings[0].image,
         )
         await k8s.deploy(deployment_vars)
         await github.notify_status(

src/runboat/settings.py

@@ -1,14 +1,28 @@
+import re
 from typing import Optional
 
-from pydantic import BaseSettings, validator
+from pydantic import BaseSettings
+from pydantic.main import BaseModel
+
+from .exceptions import RepoOrBranchNotSupported
+
+
+class BuildSettings(BaseModel):
+    image: str  # container image:tag
+
+
+class RepoSettings(BaseModel):
+    repo: str  # regex
+    branch: str  # regex
+    builds: list[BuildSettings]
 
 
 class Settings(BaseSettings):
+    # Configuration for supported repositories and branches.
+    repos: list[RepoSettings]
     # A user and password to protect the most sensitive operations of the API.
     api_admin_user: str
     api_admin_passwd: str
-    # A JSON list of supported repositories in the form owner/repo.
-    supported_repos: set[str]
     # The maximum number of concurrent initialization jobs.
     max_initializing: int = 2
     # The maximum number of builds that are started.
@@ -27,15 +41,6 @@ class Settings(BaseSettings):
     # A dictionary of variables to be set in the jinja rendering context for the
     # kubefiles.
     build_template_vars: Optional[dict[str, str]]
-    # A mapping of main branch names to container images used to run the builds.
-    build_images: dict[str, str] = {
-        "15.0": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest",
-        "14.0": "ghcr.io/oca/oca-ci/py3.6-odoo14.0:latest",
-        "13.0": "ghcr.io/oca/oca-ci/py3.6-odoo13.0:latest",
-        "12.0": "ghcr.io/oca/oca-ci/py3.6-odoo12.0:latest",
-        "11.0": "ghcr.io/oca/oca-ci/py3.5-odoo11.0:latest",
-        "10.0": "ghcr.io/oca/oca-ci/py2.7-odoo10.0:latest",
-    }
     # The token to use for the GitHub api calls (to query branches and pull requests,
     # and report build statuses).
     github_token: Optional[str]
@@ -48,10 +53,15 @@ class Settings(BaseSettings):
     class Config:
         env_prefix = "RUNBOAT_"
 
-    @validator("supported_repos")
-    @classmethod
-    def validate_supported_repos(cls, v: set[str]) -> set[str]:
-        return {item.lower() for item in v}
-
 
 settings = Settings()
+
+
+def get_build_settings(repo: str, target_branch: str) -> list[BuildSettings]:
+    for repo_settings in settings.repos:
+        if not re.match(repo_settings.repo, repo, re.IGNORECASE):
+            continue
+        if not re.match(repo_settings.branch, target_branch):
+            continue
+        return repo_settings.builds
+    raise RepoOrBranchNotSupported(f"Branch {target_branch} of {repo} not supported.")

src/runboat/webhooks.py

@@ -2,10 +2,7 @@ import logging
 
 from fastapi import APIRouter, BackgroundTasks, Header, Request
 
-from runboat.build_images import is_branch_supported, is_main_branch
-
 from .controller import controller
-from .settings import settings
 
 _logger = logging.getLogger(__name__)
 
@@ -20,48 +17,24 @@ async def receive_payload(
 ) -> None:
     # TODO check x-hub-signature
     payload = await request.json()
-    repo = payload["repository"]["full_name"]
+    repo = payload.get("repository").get("full_name")
     if not repo:
         return
-    repo = repo.lower()
-    if repo not in settings.supported_repos:
-        _logger.debug(f"Ignoring webhook delivery for unsupported repo {repo}.")
-        return
     action = payload.get("action")
     if x_github_event == "pull_request":
         if action in ("opened", "synchronize"):
-            target_branch = payload["pull_request"]["base"]["ref"]
-            if not is_branch_supported(target_branch):
-                _logger.debug(
-                    f"Ignoring webhook delivery for pull request "
-                    f"to unsupported branch {target_branch}"
-                )
-                return
             background_tasks.add_task(
                 controller.deploy_or_start,
                 repo=repo,
-                target_branch=target_branch,
+                target_branch=payload["pull_request"]["base"]["ref"],
                 pr=payload["pull_request"]["number"],
                 git_commit=payload["pull_request"]["head"]["sha"],
             )
     elif x_github_event == "push":
-        target_branch = payload["ref"].split("/")[-1]
-        if not is_branch_supported(target_branch):
-            _logger.debug(
-                f"Ignoring webhook delivery for push "
-                f"to unsupported branch {target_branch}"
-            )
-            return
-        if not is_main_branch(target_branch):
-            _logger.debug(
-                f"Ignoring webhook delivery for push "
-                f"to non-main branch {target_branch}"
-            )
-            return
         background_tasks.add_task(
             controller.deploy_or_start,
             repo=repo,
-            target_branch=target_branch,
+            target_branch=payload["ref"].split("/")[-1],
             pr=None,
             git_commit=payload["after"],
         )

tests/test_build_images.py

@@ -1,24 +0,0 @@
-import pytest
-
-from runboat.build_images import get_build_image, get_main_branch, is_branch_supported
-from runboat.exceptions import BranchNotSupported
-
-
-def test_get_main_branch() -> None:
-    assert get_main_branch("15.0") == "15.0"
-    assert get_main_branch("15.0-ocabot-merge") == "15.0"
-    with pytest.raises(BranchNotSupported):
-        get_main_branch("8.0")
-
-
-def test_is_branch_supported() -> None:
-    assert is_branch_supported("15.0")
-    assert is_branch_supported("15.0-ocabot-merge")
-    assert not is_branch_supported("8.0")
-
-
-def test_get_build_image() -> None:
-    assert get_build_image("15.0") == "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"
-    assert get_build_image("15.0-zzz") == "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"
-    with pytest.raises(BranchNotSupported):
-        get_build_image("8.0")

tests/test_db.py

@@ -2,7 +2,7 @@ import datetime
 from unittest.mock import MagicMock
 
 from runboat.db import BuildsDb
-from runboat.models import Build, BuildInitStatus, BuildStatus
+from runboat.models import Build, BuildInitStatus, BuildStatus, Repo
 
 
 def _make_build(
@@ -106,3 +106,10 @@ def test_count_all() -> None:
     assert db.count_all() == 1
     db.add(_make_build(name="b2"))
     assert db.count_all() == 2
+
+
+def test_repos() -> None:
+    db = BuildsDb()
+    db.add(_make_build(name="b1", repo="oca/repo1"))
+    db.add(_make_build(name="b2", repo="oca/repo2"))
+    assert db.repos() == [Repo(name="oca/repo1"), Repo(name="oca/repo2")]

tests/test_settings.py

@@ -0,0 +1,14 @@
+import pytest
+
+from runboat.exceptions import RepoOrBranchNotSupported
+from runboat.settings import BuildSettings, get_build_settings
+
+
+def test_get_build_settings() -> None:
+    assert get_build_settings("OCA/mis-builder", "15.0") == [
+        BuildSettings(image="ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest")
+    ]
+    with pytest.raises(RepoOrBranchNotSupported):
+        get_build_settings("acsone/mis-builder", "15.0")
+    with pytest.raises(RepoOrBranchNotSupported):
+        assert not get_build_settings("OCA/mis-builder", "15.0-stuff")