Initial commit

This commit is contained in:
Stéphane Bidoul 2021-10-16 11:09:52 +02:00
commit ac0bd848d4
No known key found for this signature in database
GPG key ID: BCAB2555446B5B92
25 changed files with 943 additions and 0 deletions

5
.flake8 Normal file
View file

@ -0,0 +1,5 @@
[flake8]
max-line-length = 88
; E203 for black
; B008 does not like fastapi Depends
extend-ignore = E203,B008

131
.gitignore vendored Normal file
View file

@ -0,0 +1,131 @@
tmp/
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
pip-wheel-metadata/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/

42
.pre-commit-config.yaml Normal file
View file

@ -0,0 +1,42 @@
default_language_version:
python: python3
repos:
- repo: https://github.com/psf/black
rev: 21.9b0
hooks:
- id: black
- repo: https://github.com/myint/autoflake
rev: v1.4
hooks:
- id: autoflake
args:
- --in-place
- --ignore-init-module-imports
- --remove-all-unused-imports
- --remove-duplicate-keys
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v4.0.1
hooks:
- id: check-toml
- id: check-yaml
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/PyCQA/flake8
rev: "4.0.1"
hooks:
- id: flake8
additional_dependencies: ["flake8-bugbear==21.4.3"]
- repo: https://github.com/pre-commit/mirrors-isort
rev: v5.9.3
hooks:
- id: isort
- repo: https://github.com/myint/docformatter
rev: v1.4
hooks:
- id: docformatter
args: ["--in-place", "--wrap-summaries=88"]
- repo: https://github.com/asottile/pyupgrade
rev: v2.29.0
hooks:
- id: pyupgrade
args: ["--py38-plus"]

21
LICENSE Normal file
View file

@ -0,0 +1,21 @@
MIT License
Copyright (c) 2021 Stéphane Bidoul
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

3
README.md Normal file
View file

@ -0,0 +1,3 @@
# runboat ☸️
A simple runbot lookalike on kubernetes. Main goal is replacing the OCA runbot.

28
pyproject.toml Normal file
View file

@ -0,0 +1,28 @@
[build-system]
requires = ["flit_core >=3.4,<4"]
build-backend = "flit_core.buildapi"
[project]
name = "runboat"
authors = [{name = "St\u00e9phane Bidoul", email = "stephane.bidoul@gmail.com"}]
readme = "README.md"
classifiers = [
"License :: OSI Approved :: MIT License",
]
dependencies = [
"fastapi",
"jinja2",
"kubernetes",
"psycopg2",
"sqlalchemy",
"uvicorn",
]
dynamic = ["version", "description"]
[project.urls]
Home = "https://github.com/sbidoul/runboat"
[tool.isort]
profile = 'black'
# flake8 config is in .flake8

33
requirements.txt Normal file
View file

@ -0,0 +1,33 @@
# frozen requirements generated by pip-deepfreeze
anyio==3.3.3
asgiref==3.4.1
cachetools==4.2.4
certifi==2021.10.8
charset-normalizer==2.0.7
click==8.0.3
fastapi==0.70.0
google-auth==2.3.0
greenlet==1.1.2
h11==0.12.0
idna==3.3
Jinja2==3.0.2
kubernetes==18.20.0
MarkupSafe==2.0.1
oauthlib==3.1.1
psycopg2==2.9.1
pyasn1==0.4.8
pyasn1-modules==0.2.8
pydantic==1.8.2
python-dateutil==2.8.2
PyYAML==6.0
requests==2.26.0
requests-oauthlib==1.3.0
rsa==4.7.2
six==1.16.0
sniffio==1.2.0
SQLAlchemy==1.4.25
starlette==0.16.0
typing-extensions==3.10.0.2
urllib3==1.26.7
uvicorn==0.15.0
websocket-client==1.2.1

2
src/runboat/__init__.py Normal file
View file

@ -0,0 +1,2 @@
"""A simple runbot lookalike using kubernetes."""
__version__ = "0.1"

131
src/runboat/api.py Normal file
View file

@ -0,0 +1,131 @@
import datetime
from enum import Enum
from typing import List
from fastapi import Depends
from fastapi.responses import StreamingResponse
from pydantic import BaseModel, Field
from sqlalchemy.orm import Session
from . import github, models
from .app import app
from .db import get_db
class Repo(BaseModel):
id: str
created: datetime.datetime
display_name: str
display_url: str = Field(title="Link to view the repo")
class Config:
orm_mode = True
@app.get(
"/repos",
response_model=List[Repo],
)
def repos(db: Session = Depends(get_db)):
return db.query(models.Repo).all()
class Branch(BaseModel):
id: str
created: datetime.datetime
display_name: str
display_url: str = Field(title="Link to view the branch or PR")
class Config:
orm_mode = True
@app.get(
"/repos/{repo_id}/branches",
response_model=List[Branch],
)
def branches(repo_id: str, db: Session = Depends(get_db)):
return db.query(models.Branch).filter(models.Branch.repo_id == repo_id).all()
class BuildStatus(str, Enum):
stopped = "stopped"
running = "running"
deploying = "deploying"
not_deployed = "not_deployed"
class Build(BaseModel):
id: str
created: datetime.datetime
display_name: str
display_url: str = Field(title="Link to open the build")
status: BuildStatus
class Config:
orm_mode = True
@app.get(
"/repos/{repo_id}/branches/{branch_id}/builds",
response_model=List[Build],
)
def builds(repo_id: str, branch_id: str):
...
@app.get(
"/repos/{repo_id}/branches/{branch_id}/builds/{build_id}/init-log",
response_class=StreamingResponse,
responses={200: {"content": {"text/plain": {}}}},
)
def init_log(repo_id: str, branch_id: str, build_id: str):
...
@app.get(
"/repos/{repo_id}/branches/{branch_id}/builds/{build_id}/log",
response_class=StreamingResponse,
responses={200: {"content": {"text/plain": {}}}},
)
def log(repo_id: str, branch_id: str, build_id: str):
...
@app.post(
"/repos/{repo_id}/branches/{branch_id}/builds/{build_id}/start",
)
def start(repo_id: str, branch_id: str, build_id: str):
"""Start the deployment.
If already running, drop the db and restart.
"""
...
@app.post(
"/repos/{repo_id}/branches/{branch_id}/builds/{build_id}/stop",
)
def stop(repo_id: str, branch_id: str, build_id: str):
"""Stop the deployment, drop the database."""
...
@app.post(
"/trigger-branch",
response_model=Build,
)
def trigger_branch(org: str, repo: str, branch: str, db: Session = Depends(get_db)):
branch_info = github.get_branch_info(org, repo, branch)
branch = models.Branch.for_github_branch(db, branch_info)
return models.Build.for_branch(db, branch, branch_info.head_sha)
@app.post(
"/trigger-pr",
response_model=Build,
)
def trigger_pr(org: str, repo: str, pr: int, db: Session = Depends(get_db)):
pr_info = github.get_pr_info(org, repo, pr)
branch = models.Branch.for_github_pr(db, pr_info)
return models.Build.for_branch(db, branch, pr_info.head_sha)

10
src/runboat/app.py Normal file
View file

@ -0,0 +1,10 @@
from fastapi import FastAPI
from .db import create_tables
app = FastAPI(title="Runboat")
@app.on_event("startup")
async def startup():
create_tables()

View file

@ -0,0 +1,14 @@
from typing import Optional
images = {
"15.0": "ghcr.io/oca/oca-ci/py3.8-odoo15.0:latest",
"14.0": "ghcr.io/oca/oca-ci/py3.6-odoo14.0:latest",
"13.0": "ghcr.io/oca/oca-ci/py3.6-odoo13.0:latest",
"12.0": "ghcr.io/oca/oca-ci/py3.6-odoo12.0:latest",
"11.0": "ghcr.io/oca/oca-ci/py3.5-odoo11.0:latest",
"10.0": "ghcr.io/oca/oca-ci/py2.7-odoo10.0:latest",
}
def get_build_image(target_branch: str) -> Optional[str]:
return images.get(target_branch)

21
src/runboat/db.py Normal file
View file

@ -0,0 +1,21 @@
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session, sessionmaker
from .settings import settings
engine = create_engine(settings.database_url)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base()
def create_tables():
Base.metadata.create_all(engine)
def get_db() -> Session:
db = SessionLocal()
try:
yield db
finally:
db.close()

14
src/runboat/exceptions.py Normal file
View file

@ -0,0 +1,14 @@
class ClientError(Exception):
pass
class RepoNotFound(ClientError):
pass
class BranchNotFound(ClientError):
pass
class NotFoundOnGithub(ClientError):
pass

56
src/runboat/github.py Normal file
View file

@ -0,0 +1,56 @@
from dataclasses import dataclass
from typing import Any
import requests
from .exceptions import NotFoundOnGithub
def _github_get(url: str) -> Any:
full_url = f"https://api.github.com{url}"
headers = {
"Accept": "application/vnd.github.v3+json",
}
response = requests.get(full_url, headers)
if response.status_code == 404:
raise NotFoundOnGithub(full_url)
response.raise_for_status()
return response.json()
@dataclass
class BranchInfo:
org: str
repo: str
name: str
head_sha: str
def get_branch_info(org: str, repo: str, branch: str) -> BranchInfo:
branch_data = _github_get(f"/repos/{org}/{repo}/git/ref/heads/{branch}")
return BranchInfo(
org=org,
repo=repo,
name=branch,
head_sha=branch_data["object"]["sha"],
)
@dataclass
class PullRequestInfo:
org: str
repo: str
number: int
head_sha: str
target_branch: str
def get_pr_info(org: str, repo: str, pr: int) -> PullRequestInfo:
pr_data = _github_get(f"/repos/{org}/{repo}/pulls/{pr}")
return PullRequestInfo(
org=org,
repo=repo,
number=pr,
head_sha=pr_data["head"]["sha"],
target_branch=pr_data["base"]["ref"],
)

View file

@ -0,0 +1,49 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: odoo
spec:
replicas: 1
strategy:
type: Recreate
selector:
matchLabels:
app: odoo
template:
metadata:
labels:
app: odoo
spec:
initContainers:
- name: odoo-init
image: odoo
volumeMounts:
- name: runboat-scripts
mountPath: /runboat
envFrom:
- secretRef:
name: odoosecretenv
- configMapRef:
name: odooenv
args: ["bash", "/runboat/runboat-init.sh"]
containers:
- name: odoo
image: odoo
volumeMounts:
- name: runboat-scripts
mountPath: /runboat
envFrom:
- secretRef:
name: odoosecretenv
- configMapRef:
name: odooenv
ports:
- name: web
containerPort: 8069
- name: longpolling
containerPort: 8072
args: ["bash", "/runboat/runboat-start.sh"]
volumes:
- name: runboat-scripts
configMap:
name: runboat-scripts

View file

@ -0,0 +1,23 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: odoo
spec:
rules:
- host: $(HOSTNAME)
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: odoo
port:
number: 8069
- path: /longpolling
pathType: Prefix
backend:
service:
name: odoo
port:
number: 8072

View file

@ -0,0 +1,46 @@
resources:
- deployment.yaml
- service.yaml
- ingress.yaml
namePrefix: "{{ name_prefix }}-"
commonLabels:
runboat-build: "{{ name_prefix}}"
images:
- name: odoo
newName: "{{ image_name }}"
newTag: latest
secretGenerator:
- name: odoosecretenv
literals:
- DB_PASSWORD={{ db_passwd }}
- ADMIN_PASSWD={{ admin_passwd }}
configMapGenerator:
- name: odooenv
literals:
- DB_HOST={{ db_host }}
- DB_PORT={{ db_port }}
- DB_USER={{ db_user }}
- DB_NAME={{ db_name }}
- LIST_DB=False
- name: runboat-scripts
files:
- runboat-clone-and-install.sh
- runboat-init.sh
- runboat-start.sh
- name: vars
literals:
- HOSTNAME={{ hostname }}
vars:
- name: HOSTNAME
objref:
name: vars
kind: ConfigMap
apiVersion: v1
fieldref:
fieldpath: data.HOSTNAME

View file

@ -0,0 +1,15 @@
#!/bin/bash
set -ex
#
# Clone an addons repository at git reference in $ADDONS_DIR.
# Run oca_install_addons and oca_init_db on it.
#
git clone --filter=blob:none $REPO $ADDONS_DIR
cd $ADDONS_DIR
git fetch origin $REF:build
git checkout build
oca_install_addons

View file

@ -0,0 +1,21 @@
#!/bin/bash
#
# Install all addons to test.
#
set -ex
bash /runboat/runboat-clone-and-install.sh
oca_wait_for_postgres
# TODO: rather, do nothing if db exists, so we can start instantly ?
dropdb --if-exists $PGDATABASE
ADDONS=$(addons --addons-dir ${ADDONS_DIR} --include "${INCLUDE}" --exclude "${EXCLUDE}" list)
unbuffer $(which odoo || which openerp-server) \
-d ${PGDATABASE} \
-i ${ADDONS:-base} \
--stop-after-init

View file

@ -0,0 +1,14 @@
#!/bin/bash
#
# Start Odoo
#
set -ex
bash /runboat/runboat-clone-and-install.sh
oca_wait_for_postgres
unbuffer $(which odoo || which openerp-server) \
-d ${PGDATABASE}

View file

@ -0,0 +1,17 @@
apiVersion: v1
kind: Service
metadata:
name: odoo
spec:
type: ClusterIP
ports:
- port: 8069
targetPort: 8069
protocol: TCP
name: web
- port: 8072
targetPort: 8072
protocol: TCP
name: longpolling
selector:
app: odoo

205
src/runboat/models.py Normal file
View file

@ -0,0 +1,205 @@
from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, func
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.orm import relationship
from sqlalchemy.sql import expression
from .build_images import get_build_image
from .db import Base, Session
from .exceptions import RepoNotFound
from .github import BranchInfo, PullRequestInfo
from .settings import settings
from .utils import slugify
class utcnow(expression.FunctionElement):
type = DateTime()
@compiles(utcnow, "postgresql")
def pg_utcnow(element, compiler, **kw):
return "TIMEZONE('utc', CURRENT_TIMESTAMP)"
class Repo(Base):
__tablename__ = "repo"
id = Column(Integer, primary_key=True, index=True)
created = Column(DateTime, nullable=False, server_default=utcnow())
org = Column(String, nullable=False)
name = Column(String, nullable=False)
branches = relationship("Branch", back_populates="repo")
@property
def display_name(self) -> str:
return f"{self.org}/{self.name}"
@property
def display_url(self) -> str:
return f"https://github.com/{self.org}/{self.name}"
@property
def slug(self) -> str:
return f"{self.org}-{self.name}"
@classmethod
def get_repo(cls, db: Session, org: str, name: str) -> "Repo":
repo = (
db.query(Repo)
.filter(
func.lower(Repo.org) == func.lower(org),
func.lower(Repo.name) == func.lower(name),
)
.one_or_none()
)
if repo is None:
raise RepoNotFound()
return repo
class Branch(Base):
__tablename__ = "branch"
id = Column(Integer, primary_key=True, index=True)
created = Column(DateTime, nullable=False, server_default=utcnow())
target_branch = Column(String, nullable=False)
pr = Column(Integer, nullable=True)
repo_id = Column(Integer, ForeignKey("repo.id"), nullable=False, index=True)
repo = relationship("Repo", back_populates="branches")
builds = relationship("Build", back_populates="branch")
@property
def display_name(self) -> str:
name = f"{self.target_branch}"
if self.pr:
name += f" #{self.pr}"
return name
@property
def display_url(self) -> str:
url = f"https://github.com/{self.repo.org}/{self.repo.name}"
if self.pr:
return f"{url}/pull/{self.pr}"
return f"{url}/tree/{self.target_branch}"
@property
def slug(self) -> str:
slug = f"{self.repo.slug}-{slugify(self.target_branch)}"
if self.pr:
return f"{slug}-pr{self.pr}"
return slug
@classmethod
def for_github_branch(cls, db: Session, branch_info: BranchInfo) -> "Branch":
repo = Repo.get_repo(db, branch_info.org, branch_info.repo)
branch = (
db.query(Branch)
.filter(
Branch.repo == repo,
Branch.target_branch == branch_info.name,
Branch.pr is None,
)
.one_or_none()
)
if branch is None:
branch = Branch(
repo=repo,
target_branch=branch_info.name,
pr=None,
)
db.add(branch)
db.flush()
return branch
@classmethod
def for_github_pr(cls, db: Session, pr_info: PullRequestInfo) -> "Branch":
repo = Repo.get_repo(db, pr_info.org, pr_info.repo)
branch = (
db.query(Branch)
.filter(
Branch.repo == repo,
Branch.target_branch == pr_info.target_branch,
Branch.pr == pr_info.number,
)
.one_or_none()
)
if branch is None:
branch = Branch(
repo=repo,
target_branch=pr_info.target_branch,
pr=pr_info.number,
)
db.add(branch)
db.flush()
return branch
class Build(Base):
__tablename__ = "build"
id = Column(Integer, primary_key=True, index=True)
created = Column(DateTime, nullable=False, server_default=utcnow())
branch_id = Column(Integer, ForeignKey("branch.id"), nullable=False, index=True)
branch = relationship("Branch", back_populates="builds")
build_image = Column(String, nullable=False)
git_sha = Column(String, nullable=False)
status = Column(String, nullable=False)
# ressource_label = Column(String, nullable=False, unique=True, index=True)
# TODO: add unique constraint on branch_id + git_sha
@property
def display_name(self) -> str:
return f"{self.created} {self.git_sha[:6]}"
@property
def display_url(self) -> str:
return f"http://{self.slug}.{settings.build_domain}"
@property
def slug(self) -> str:
return f"{self.branch.slug}-{self.id}"
@classmethod
def for_branch(cls, db: Session, branch: Branch, git_sha: str) -> "Build":
print("*******", branch)
build = (
db.query(Build)
.filter(
Build.branch == branch,
Build.git_sha == git_sha,
)
.one_or_none()
)
if build is None:
build_image = get_build_image(branch.target_branch)
build = Build(
branch=branch,
git_sha=git_sha,
build_image=build_image,
status="not_deployed", # TODO use same enum as in API
)
db.add(build)
db.flush()
return build
def delete(self) -> None:
# self.stop
# delete from db
...
def start(self) -> None:
...
def stop(self) -> None:
...
def init_log(self) -> str:
...
def log(self, tail: int = 1000) -> str:
...

15
src/runboat/settings.py Normal file
View file

@ -0,0 +1,15 @@
from typing import Optional
from pydantic import BaseSettings
class Settings(BaseSettings):
database_url: str
build_pghost: Optional[str]
build_pgport: Optional[str]
build_pguser: Optional[str]
build_pgpassword: Optional[str]
build_domain: str
settings = Settings()

5
src/runboat/utils.py Normal file
View file

@ -0,0 +1,5 @@
import re
def slugify(s: str) -> str:
return re.sub(r"[^a-z0-9]", "-", s.lower())

22
src/runboat/webhooks.py Normal file
View file

@ -0,0 +1,22 @@
def on_pr_open_or_update():
# find Repo
# find image from target branch (exit if not found)
# find or create Branch
# create Build
# start build (enqueue)
...
def on_pr_close_or_merge():
# find Repo, Branch
# delete branch (enqueue)
...
def on_push():
# find Repo, branch
# find image from target branch (exit if not found)
# find or create Branch
# create Build
# start build (enqueue)
...