itsulu/runboat
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #18 from sbidoul/flex-config

Browse source 
More flexible configuration
This commit is contained in:
Stéphane Bidoul 2021-11-20 12:22:41 +01:00 committed by GitHub
commit f1ee13d2b7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 69 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.env.sample
View file

@ -1,4 +1,4 @@
RUNBOAT_SUPPORTED_REPOS=["OCA/mis-builder", "shopinvader/odoo-shopinvader", "OCA/server-env"]
RUNBOAT_REPOS=[{"repo": "^oca/.*", "branch": "^15.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}]}]
RUNBOAT_API_ADMIN_USER="admin"
RUNBOAT_API_ADMIN_PASSWD="admin"
RUNBOAT_BUILD_NAMESPACE=runboat-builds

3
.env.test
View file

@ -1,4 +1,4 @@
RUNBOAT_SUPPORTED_REPOS=["OCA/mis-builder", "shopinvader/odoo-shopinvader", "OCA/server-env"]
RUNBOAT_REPOS=[{"repo": "^oca/.*", "branch": "^15.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}]}]
RUNBOAT_API_ADMIN_USER="admin"
RUNBOAT_API_ADMIN_PASSWD="admin"
RUNBOAT_BUILD_NAMESPACE=runboat-builds
@ -8,4 +8,3 @@ RUNBOAT_BUILD_SECRET_ENV={"PGPASSWORD": "thepgpassword"}
RUNBOAT_BUILD_TEMPLATE_VARS={"storageClassName": "my-storage-class"}
RUNBOAT_GITHUB_TOKEN=
RUNBOAT_LOG_CONFIG=log-config.yaml
RUNBOAT_BUILD_IMAGES={"15.0": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}

3
Dockerfile
View file

@ -10,8 +10,7 @@ RUN curl -L \
COPY requirements.txt /tmp/requirements.txt
RUN pip install --no-cache-dir -r /tmp/requirements.txt
ENV RUNBOAT_SUPPORTED_REPOS='["OCA/mis-builder", "shopinvader/odoo-shopinvader", "OCA/server-env"]'
ENV RUNBOAT_BUILD_IMAGES='{"15.0": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}'
ENV RUNBOAT_REPOS='[{"repo": "^oca/.*", "branch": "^15.0$", "builds": [{"image": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"}]}]'
ENV RUNBOAT_API_ADMIN_USER="admin"
ENV RUNBOAT_API_ADMIN_PASSWD="admin"
ENV RUNBOAT_BUILD_NAMESPACE=runboat-builds

3
src/runboat/api.py
View file

@ -12,7 +12,6 @@ from starlette.status import HTTP_404_NOT_FOUND
from . import github, models
from .controller import Controller, controller
from .deps import authenticated
from .settings import settings
router = APIRouter()
@ -73,7 +72,7 @@ async def controller_status() -> Controller:
@router.get("/repos", response_model=list[Repo])
async def repos() -> list[models.Repo]:
return [models.Repo(name=name) for name in settings.supported_repos]
return controller.db.repos()
@router.get(

39
src/runboat/build_images.py
View file

@ -1,39 +0,0 @@
import re
from .exceptions import BranchNotSupported
from .settings import settings
TARGET_BRANCH_RE = re.compile(r"^(\d+\.\d+)")
def get_main_branch(branch_name: str) -> str:
mo = TARGET_BRANCH_RE.match(branch_name)
if not mo:
raise BranchNotSupported(
f"Malformed branch name {branch_name} "
f"(it should start with an Odoo branch name)."
)
key = mo.group(1)
if key not in settings.build_images:
raise BranchNotSupported(
f"No build image configured for {key} (from {branch_name})."
)
return key
def is_branch_supported(branch_name: str) -> bool:
try:
return bool(get_main_branch(branch_name))
except BranchNotSupported:
return False
def is_main_branch(branch_name: str) -> bool:
try:
return branch_name == get_main_branch(branch_name)
except BranchNotSupported:
return False
def get_build_image(branch_name: str) -> str:
return settings.build_images[get_main_branch(branch_name)]

6
src/runboat/db.py
View file

@ -3,7 +3,7 @@ import sqlite3
from typing import Iterator, Protocol, cast
from weakref import WeakSet
from .models import Build, BuildEvent, BuildInitStatus, BuildStatus
from .models import Build, BuildEvent, BuildInitStatus, BuildStatus, Repo
_logger = logging.getLogger(__name__)
@ -184,6 +184,10 @@ class BuildsDb:
).fetchall()
return [self._build_from_row(row) for row in rows]
def repos(self) -> list[Repo]:
rows = self._con.execute("SELECT DISTINCT repo FROM builds ORDER BY repo")
return [Repo(name=row[0]) for row in rows]
def search(
self,
repo: str | None = None,

2
src/runboat/exceptions.py
View file

@ -14,5 +14,5 @@ class NotFoundOnGitHub(ClientError):
pass
class BranchNotSupported(ClientError):
class RepoOrBranchNotSupported(ClientError):
pass

11
src/runboat/models.py
View file

@ -8,9 +8,8 @@ from kubernetes.client.models.v1_deployment import V1Deployment
from pydantic import BaseModel
from . import github, k8s
from .build_images import get_build_image
from .github import GitHubStatusState
from .settings import settings
from .settings import get_build_settings, settings
from .utils import slugify
_logger = logging.getLogger(__name__)
@ -175,7 +174,11 @@ class Build(BaseModel):
name = f"b{uuid.uuid4()}"
slug = cls.make_slug(repo, target_branch, pr, git_commit)
_logger.info(f"Deploying {slug} ({name}).")
image = get_build_image(target_branch)
build_settings = get_build_settings(repo, target_branch)
if len(build_settings) > 1:
raise NotImplementedError(
"Having more than one build per commit is not supported yet."
)
deployment_vars = k8s.make_deployment_vars(
k8s.DeploymentMode.deployment,
name,
@ -184,7 +187,7 @@ class Build(BaseModel):
target_branch,
pr,
git_commit,
image,
build_settings[0].image,
)
await k8s.deploy(deployment_vars)
await github.notify_status(

44
src/runboat/settings.py
View file

@ -1,14 +1,28 @@
import re
from typing import Optional
from pydantic import BaseSettings, validator
from pydantic import BaseSettings
from pydantic.main import BaseModel
from .exceptions import RepoOrBranchNotSupported
class BuildSettings(BaseModel):
image: str # container image:tag
class RepoSettings(BaseModel):
repo: str # regex
branch: str # regex
builds: list[BuildSettings]
class Settings(BaseSettings):
# Configuration for supported repositories and branches.
repos: list[RepoSettings]
# A user and password to protect the most sensitive operations of the API.
api_admin_user: str
api_admin_passwd: str
# A JSON list of supported repositories in the form owner/repo.
supported_repos: set[str]
# The maximum number of concurrent initialization jobs.
max_initializing: int = 2
# The maximum number of builds that are started.
@ -27,15 +41,6 @@ class Settings(BaseSettings):
# A dictionary of variables to be set in the jinja rendering context for the
# kubefiles.
build_template_vars: Optional[dict[str, str]]
# A mapping of main branch names to container images used to run the builds.
build_images: dict[str, str] = {
"15.0": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest",
"14.0": "ghcr.io/oca/oca-ci/py3.6-odoo14.0:latest",
"13.0": "ghcr.io/oca/oca-ci/py3.6-odoo13.0:latest",
"12.0": "ghcr.io/oca/oca-ci/py3.6-odoo12.0:latest",
"11.0": "ghcr.io/oca/oca-ci/py3.5-odoo11.0:latest",
"10.0": "ghcr.io/oca/oca-ci/py2.7-odoo10.0:latest",
}
# The token to use for the GitHub api calls (to query branches and pull requests,
# and report build statuses).
github_token: Optional[str]
@ -48,10 +53,15 @@ class Settings(BaseSettings):
class Config:
env_prefix = "RUNBOAT_"
@validator("supported_repos")
@classmethod
def validate_supported_repos(cls, v: set[str]) -> set[str]:
return {item.lower() for item in v}
settings = Settings()
def get_build_settings(repo: str, target_branch: str) -> list[BuildSettings]:
for repo_settings in settings.repos:
if not re.match(repo_settings.repo, repo, re.IGNORECASE):
continue
if not re.match(repo_settings.branch, target_branch):
continue
return repo_settings.builds
raise RepoOrBranchNotSupported(f"Branch {target_branch} of {repo} not supported.")

33
src/runboat/webhooks.py
View file

@ -2,10 +2,7 @@ import logging
from fastapi import APIRouter, BackgroundTasks, Header, Request
from runboat.build_images import is_branch_supported, is_main_branch
from .controller import controller
from .settings import settings
_logger = logging.getLogger(__name__)
@ -20,48 +17,24 @@ async def receive_payload(
) -> None:
# TODO check x-hub-signature
payload = await request.json()
repo = payload["repository"]["full_name"]
repo = payload.get("repository").get("full_name")
if not repo:
return
repo = repo.lower()
if repo not in settings.supported_repos:
_logger.debug(f"Ignoring webhook delivery for unsupported repo {repo}.")
return
action = payload.get("action")
if x_github_event == "pull_request":
if action in ("opened", "synchronize"):
target_branch = payload["pull_request"]["base"]["ref"]
if not is_branch_supported(target_branch):
_logger.debug(
f"Ignoring webhook delivery for pull request "
f"to unsupported branch {target_branch}"
)
return
background_tasks.add_task(
controller.deploy_or_start,
repo=repo,
target_branch=target_branch,
target_branch=payload["pull_request"]["base"]["ref"],
pr=payload["pull_request"]["number"],
git_commit=payload["pull_request"]["head"]["sha"],
)
elif x_github_event == "push":
target_branch = payload["ref"].split("/")[-1]
if not is_branch_supported(target_branch):
_logger.debug(
f"Ignoring webhook delivery for push "
f"to unsupported branch {target_branch}"
)
return
if not is_main_branch(target_branch):
_logger.debug(
f"Ignoring webhook delivery for push "
f"to non-main branch {target_branch}"
)
return
background_tasks.add_task(
controller.deploy_or_start,
repo=repo,
target_branch=target_branch,
target_branch=payload["ref"].split("/")[-1],
pr=None,
git_commit=payload["after"],
)

24
tests/test_build_images.py
View file

@ -1,24 +0,0 @@
import pytest
from runboat.build_images import get_build_image, get_main_branch, is_branch_supported
from runboat.exceptions import BranchNotSupported
def test_get_main_branch() -> None:
assert get_main_branch("15.0") == "15.0"
assert get_main_branch("15.0-ocabot-merge") == "15.0"
with pytest.raises(BranchNotSupported):
get_main_branch("8.0")
def test_is_branch_supported() -> None:
assert is_branch_supported("15.0")
assert is_branch_supported("15.0-ocabot-merge")
assert not is_branch_supported("8.0")
def test_get_build_image() -> None:
assert get_build_image("15.0") == "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"
assert get_build_image("15.0-zzz") == "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest"
with pytest.raises(BranchNotSupported):
get_build_image("8.0")

9
tests/test_db.py
View file

@ -2,7 +2,7 @@ import datetime
from unittest.mock import MagicMock
from runboat.db import BuildsDb
from runboat.models import Build, BuildInitStatus, BuildStatus
from runboat.models import Build, BuildInitStatus, BuildStatus, Repo
def _make_build(
@ -106,3 +106,10 @@ def test_count_all() -> None:
assert db.count_all() == 1
db.add(_make_build(name="b2"))
assert db.count_all() == 2
def test_repos() -> None:
db = BuildsDb()
db.add(_make_build(name="b1", repo="oca/repo1"))
db.add(_make_build(name="b2", repo="oca/repo2"))
assert db.repos() == [Repo(name="oca/repo1"), Repo(name="oca/repo2")]

14
tests/test_settings.py Normal file
View file

@ -0,0 +1,14 @@
import pytest
from runboat.exceptions import RepoOrBranchNotSupported
from runboat.settings import BuildSettings, get_build_settings
def test_get_build_settings() -> None:
assert get_build_settings("OCA/mis-builder", "15.0") == [
BuildSettings(image="ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest")
]
with pytest.raises(RepoOrBranchNotSupported):
get_build_settings("acsone/mis-builder", "15.0")
with pytest.raises(RepoOrBranchNotSupported):
assert not get_build_settings("OCA/mis-builder", "15.0-stuff")