diff --git a/.env.sample b/.env.sample index d5c57cc..311ebc2 100644 --- a/.env.sample +++ b/.env.sample @@ -1,4 +1,4 @@ -RUNBOAT_SUPPORTED_REPOS=["OCA/mis-builder", "shopinvader/odoo-shopinvader", "OCA/server-env"] +RUNBOAT_REPOS=[{"repo": "^oca/.*", "branch": "^15.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}]}] RUNBOAT_API_ADMIN_USER="admin" RUNBOAT_API_ADMIN_PASSWD="admin" RUNBOAT_BUILD_NAMESPACE=runboat-builds diff --git a/.env.test b/.env.test index 3f6543d..e28b733 100644 --- a/.env.test +++ b/.env.test @@ -1,4 +1,4 @@ -RUNBOAT_SUPPORTED_REPOS=["OCA/mis-builder", "shopinvader/odoo-shopinvader", "OCA/server-env"] +RUNBOAT_REPOS=[{"repo": "^oca/.*", "branch": "^15.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}]}] RUNBOAT_API_ADMIN_USER="admin" RUNBOAT_API_ADMIN_PASSWD="admin" RUNBOAT_BUILD_NAMESPACE=runboat-builds @@ -8,4 +8,3 @@ RUNBOAT_BUILD_SECRET_ENV={"PGPASSWORD": "thepgpassword"} RUNBOAT_BUILD_TEMPLATE_VARS={"storageClassName": "my-storage-class"} RUNBOAT_GITHUB_TOKEN= RUNBOAT_LOG_CONFIG=log-config.yaml -RUNBOAT_BUILD_IMAGES={"15.0": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"} diff --git a/Dockerfile b/Dockerfile index a90f776..a5ab924 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,7 @@ RUN curl -L \ COPY requirements.txt /tmp/requirements.txt RUN pip install --no-cache-dir -r /tmp/requirements.txt -ENV RUNBOAT_SUPPORTED_REPOS='["OCA/mis-builder", "shopinvader/odoo-shopinvader", "OCA/server-env"]' -ENV RUNBOAT_BUILD_IMAGES='{"15.0": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}' +ENV RUNBOAT_REPOS='[{"repo": "^oca/.*", "branch": "^15.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}]}]' ENV RUNBOAT_API_ADMIN_USER="admin" ENV RUNBOAT_API_ADMIN_PASSWD="admin" ENV RUNBOAT_BUILD_NAMESPACE=runboat-builds diff --git a/src/runboat/api.py b/src/runboat/api.py index 925cbfa..d8479e7 100644 --- a/src/runboat/api.py +++ b/src/runboat/api.py @@ -12,7 +12,6 @@ from starlette.status import HTTP_404_NOT_FOUND from . import github, models from .controller import Controller, controller from .deps import authenticated -from .settings import settings router = APIRouter() @@ -73,7 +72,7 @@ async def controller_status() -> Controller: @router.get("/repos", response_model=list[Repo]) async def repos() -> list[models.Repo]: - return [models.Repo(name=name) for name in settings.supported_repos] + return controller.db.repos() @router.get( diff --git a/src/runboat/build_images.py b/src/runboat/build_images.py deleted file mode 100644 index 54effca..0000000 --- a/src/runboat/build_images.py +++ /dev/null @@ -1,39 +0,0 @@ -import re - -from .exceptions import BranchNotSupported -from .settings import settings - -TARGET_BRANCH_RE = re.compile(r"^(\d+\.\d+)") - - -def get_main_branch(branch_name: str) -> str: - mo = TARGET_BRANCH_RE.match(branch_name) - if not mo: - raise BranchNotSupported( - f"Malformed branch name {branch_name} " - f"(it should start with an Odoo branch name)." - ) - key = mo.group(1) - if key not in settings.build_images: - raise BranchNotSupported( - f"No build image configured for {key} (from {branch_name})." - ) - return key - - -def is_branch_supported(branch_name: str) -> bool: - try: - return bool(get_main_branch(branch_name)) - except BranchNotSupported: - return False - - -def is_main_branch(branch_name: str) -> bool: - try: - return branch_name == get_main_branch(branch_name) - except BranchNotSupported: - return False - - -def get_build_image(branch_name: str) -> str: - return settings.build_images[get_main_branch(branch_name)] diff --git a/src/runboat/db.py b/src/runboat/db.py index f3db7c6..c7b3937 100644 --- a/src/runboat/db.py +++ b/src/runboat/db.py @@ -3,7 +3,7 @@ import sqlite3 from typing import Iterator, Protocol, cast from weakref import WeakSet -from .models import Build, BuildEvent, BuildInitStatus, BuildStatus +from .models import Build, BuildEvent, BuildInitStatus, BuildStatus, Repo _logger = logging.getLogger(__name__) @@ -184,6 +184,10 @@ class BuildsDb: ).fetchall() return [self._build_from_row(row) for row in rows] + def repos(self) -> list[Repo]: + rows = self._con.execute("SELECT DISTINCT repo FROM builds ORDER BY repo") + return [Repo(name=row[0]) for row in rows] + def search( self, repo: str | None = None, diff --git a/src/runboat/exceptions.py b/src/runboat/exceptions.py index 6a678d1..6956f35 100644 --- a/src/runboat/exceptions.py +++ b/src/runboat/exceptions.py @@ -14,5 +14,5 @@ class NotFoundOnGitHub(ClientError): pass -class BranchNotSupported(ClientError): +class RepoOrBranchNotSupported(ClientError): pass diff --git a/src/runboat/models.py b/src/runboat/models.py index 5b0acb0..5a73fbb 100644 --- a/src/runboat/models.py +++ b/src/runboat/models.py @@ -8,9 +8,8 @@ from kubernetes.client.models.v1_deployment import V1Deployment from pydantic import BaseModel from . import github, k8s -from .build_images import get_build_image from .github import GitHubStatusState -from .settings import settings +from .settings import get_build_settings, settings from .utils import slugify _logger = logging.getLogger(__name__) @@ -175,7 +174,11 @@ class Build(BaseModel): name = f"b{uuid.uuid4()}" slug = cls.make_slug(repo, target_branch, pr, git_commit) _logger.info(f"Deploying {slug} ({name}).") - image = get_build_image(target_branch) + build_settings = get_build_settings(repo, target_branch) + if len(build_settings) > 1: + raise NotImplementedError( + "Having more than one build per commit is not supported yet." + ) deployment_vars = k8s.make_deployment_vars( k8s.DeploymentMode.deployment, name, @@ -184,7 +187,7 @@ class Build(BaseModel): target_branch, pr, git_commit, - image, + build_settings[0].image, ) await k8s.deploy(deployment_vars) await github.notify_status( diff --git a/src/runboat/settings.py b/src/runboat/settings.py index 17d5443..832dfbd 100644 --- a/src/runboat/settings.py +++ b/src/runboat/settings.py @@ -1,14 +1,28 @@ +import re from typing import Optional -from pydantic import BaseSettings, validator +from pydantic import BaseSettings +from pydantic.main import BaseModel + +from .exceptions import RepoOrBranchNotSupported + + +class BuildSettings(BaseModel): + image: str # container image:tag + + +class RepoSettings(BaseModel): + repo: str # regex + branch: str # regex + builds: list[BuildSettings] class Settings(BaseSettings): + # Configuration for supported repositories and branches. + repos: list[RepoSettings] # A user and password to protect the most sensitive operations of the API. api_admin_user: str api_admin_passwd: str - # A JSON list of supported repositories in the form owner/repo. - supported_repos: set[str] # The maximum number of concurrent initialization jobs. max_initializing: int = 2 # The maximum number of builds that are started. @@ -27,15 +41,6 @@ class Settings(BaseSettings): # A dictionary of variables to be set in the jinja rendering context for the # kubefiles. build_template_vars: Optional[dict[str, str]] - # A mapping of main branch names to container images used to run the builds. - build_images: dict[str, str] = { - "15.0": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest", - "14.0": "ghcr.io/oca/oca-ci/py3.6-odoo14.0:latest", - "13.0": "ghcr.io/oca/oca-ci/py3.6-odoo13.0:latest", - "12.0": "ghcr.io/oca/oca-ci/py3.6-odoo12.0:latest", - "11.0": "ghcr.io/oca/oca-ci/py3.5-odoo11.0:latest", - "10.0": "ghcr.io/oca/oca-ci/py2.7-odoo10.0:latest", - } # The token to use for the GitHub api calls (to query branches and pull requests, # and report build statuses). github_token: Optional[str] @@ -48,10 +53,15 @@ class Settings(BaseSettings): class Config: env_prefix = "RUNBOAT_" - @validator("supported_repos") - @classmethod - def validate_supported_repos(cls, v: set[str]) -> set[str]: - return {item.lower() for item in v} - settings = Settings() + + +def get_build_settings(repo: str, target_branch: str) -> list[BuildSettings]: + for repo_settings in settings.repos: + if not re.match(repo_settings.repo, repo, re.IGNORECASE): + continue + if not re.match(repo_settings.branch, target_branch): + continue + return repo_settings.builds + raise RepoOrBranchNotSupported(f"Branch {target_branch} of {repo} not supported.") diff --git a/src/runboat/webhooks.py b/src/runboat/webhooks.py index 5614526..3643414 100644 --- a/src/runboat/webhooks.py +++ b/src/runboat/webhooks.py @@ -2,10 +2,7 @@ import logging from fastapi import APIRouter, BackgroundTasks, Header, Request -from runboat.build_images import is_branch_supported, is_main_branch - from .controller import controller -from .settings import settings _logger = logging.getLogger(__name__) @@ -20,48 +17,24 @@ async def receive_payload( ) -> None: # TODO check x-hub-signature payload = await request.json() - repo = payload["repository"]["full_name"] + repo = payload.get("repository").get("full_name") if not repo: return - repo = repo.lower() - if repo not in settings.supported_repos: - _logger.debug(f"Ignoring webhook delivery for unsupported repo {repo}.") - return action = payload.get("action") if x_github_event == "pull_request": if action in ("opened", "synchronize"): - target_branch = payload["pull_request"]["base"]["ref"] - if not is_branch_supported(target_branch): - _logger.debug( - f"Ignoring webhook delivery for pull request " - f"to unsupported branch {target_branch}" - ) - return background_tasks.add_task( controller.deploy_or_start, repo=repo, - target_branch=target_branch, + target_branch=payload["pull_request"]["base"]["ref"], pr=payload["pull_request"]["number"], git_commit=payload["pull_request"]["head"]["sha"], ) elif x_github_event == "push": - target_branch = payload["ref"].split("/")[-1] - if not is_branch_supported(target_branch): - _logger.debug( - f"Ignoring webhook delivery for push " - f"to unsupported branch {target_branch}" - ) - return - if not is_main_branch(target_branch): - _logger.debug( - f"Ignoring webhook delivery for push " - f"to non-main branch {target_branch}" - ) - return background_tasks.add_task( controller.deploy_or_start, repo=repo, - target_branch=target_branch, + target_branch=payload["ref"].split("/")[-1], pr=None, git_commit=payload["after"], ) diff --git a/tests/test_build_images.py b/tests/test_build_images.py deleted file mode 100644 index d45086b..0000000 --- a/tests/test_build_images.py +++ /dev/null @@ -1,24 +0,0 @@ -import pytest - -from runboat.build_images import get_build_image, get_main_branch, is_branch_supported -from runboat.exceptions import BranchNotSupported - - -def test_get_main_branch() -> None: - assert get_main_branch("15.0") == "15.0" - assert get_main_branch("15.0-ocabot-merge") == "15.0" - with pytest.raises(BranchNotSupported): - get_main_branch("8.0") - - -def test_is_branch_supported() -> None: - assert is_branch_supported("15.0") - assert is_branch_supported("15.0-ocabot-merge") - assert not is_branch_supported("8.0") - - -def test_get_build_image() -> None: - assert get_build_image("15.0") == "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest" - assert get_build_image("15.0-zzz") == "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest" - with pytest.raises(BranchNotSupported): - get_build_image("8.0") diff --git a/tests/test_db.py b/tests/test_db.py index 56251dc..e59e5d4 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -2,7 +2,7 @@ import datetime from unittest.mock import MagicMock from runboat.db import BuildsDb -from runboat.models import Build, BuildInitStatus, BuildStatus +from runboat.models import Build, BuildInitStatus, BuildStatus, Repo def _make_build( @@ -106,3 +106,10 @@ def test_count_all() -> None: assert db.count_all() == 1 db.add(_make_build(name="b2")) assert db.count_all() == 2 + + +def test_repos() -> None: + db = BuildsDb() + db.add(_make_build(name="b1", repo="oca/repo1")) + db.add(_make_build(name="b2", repo="oca/repo2")) + assert db.repos() == [Repo(name="oca/repo1"), Repo(name="oca/repo2")] diff --git a/tests/test_settings.py b/tests/test_settings.py new file mode 100644 index 0000000..6fe83e1 --- /dev/null +++ b/tests/test_settings.py @@ -0,0 +1,14 @@ +import pytest + +from runboat.exceptions import RepoOrBranchNotSupported +from runboat.settings import BuildSettings, get_build_settings + + +def test_get_build_settings() -> None: + assert get_build_settings("OCA/mis-builder", "15.0") == [ + BuildSettings(image="ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest") + ] + with pytest.raises(RepoOrBranchNotSupported): + get_build_settings("acsone/mis-builder", "15.0") + with pytest.raises(RepoOrBranchNotSupported): + assert not get_build_settings("OCA/mis-builder", "15.0-stuff")