Commit 72ba58a7 authored by Guilhem Saurel's avatar Guilhem Saurel
Browse files

fix tests

parent ca7c1b56
......@@ -22,5 +22,5 @@ build:
script:
- docker build -t ${DOCKER_TAG} .
- docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e GITHUB_PIPELINE_TOKEN -e GITLAB_PIPELINE_TOKEN -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py migrate
- docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e GITHUB_PIPELINE_TOKEN -e GITLAB_PIPELINE_TOKEN -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py test
- docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e GITHUB_PIPELINE_TOKEN -e GITLAB_PIPELINE_TOKEN -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py test --parallel 1
- docker push ${DOCKER_TAG}
import asyncio
from ipaddress import ip_address, ip_network
from django.conf import settings
from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
from django.shortcuts import reverse
from django.utils.decorators import sync_and_async_middleware
from rest_framework import permissions
......@@ -15,19 +17,30 @@ def ip_laas(request: HttpRequest) -> bool:
return any(forwarded_for in ip_network(net) for net in settings.LAAS_NETWORKS)
class LAASPermsMiddleware:
def __init__(self, get_response):
self.get_response = get_response
def allowed(request: HttpRequest) -> bool:
"""Allow access to pages protected at a higher application level,
or if the user is authenticated,
or if the request comes from a trusted IP.
"""
return (any(request.path.startswith(f'/{url}/') for url in ALLOWED_URLS)
or request.user and request.user.is_authenticated
or request.method in permissions.SAFE_METHODS and ip_laas(request))
def __call__(self, request: HttpRequest) -> HttpResponse:
"""Allow access to pages protected at a higher application level,
or if the user is authenticated,
or if the request comes from a trusted IP.
"""
allowed = (any(request.path.startswith(f'/{url}/') for url in ALLOWED_URLS)
or request.user and request.user.is_authenticated
or request.method in permissions.SAFE_METHODS and ip_laas(request))
if allowed:
return self.get_response(request)
return HttpResponseRedirect(reverse('login'))
@sync_and_async_middleware
def laas_perms_middleware(get_response):
# One-time configuration and initialization goes here.
if asyncio.iscoroutinefunction(get_response):
async def middleware(request) -> HttpResponse:
if allowed(request):
return await get_response(request)
return HttpResponseRedirect(reverse('login'))
else:
def middleware(request) -> HttpResponse:
if allowed(request):
return get_response(request)
return HttpResponseRedirect(reverse('login'))
return middleware
......@@ -58,7 +58,7 @@ MIDDLEWARE = [
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
f'{PROJECT}.middleware.LAASPermsMiddleware',
f'{PROJECT}.middleware.laas_perms_middleware',
]
ROOT_URLCONF = f'{PROJECT}.urls'
......
This diff is collapsed.
......@@ -7,6 +7,7 @@ from hashlib import sha1
from ipaddress import ip_address, ip_network
from json import loads
from asgiref.sync import sync_to_async, async_to_sync
from django.conf import settings
from django.core.mail import mail_admins
from django.http import HttpRequest
......@@ -27,8 +28,11 @@ from . import models
logger = logging.getLogger(__name__)
PR_MASTER_MSG = """Hi ! This project doesn't usually accept pull requests on master. If this wasn't intentionnal, you
can change the base branch of this pull request to devel (No need to close it for that). Best, a bot."""
def check_suite(request: HttpRequest, rep: str) -> HttpResponse:
async def check_suite(request: HttpRequest, rep: str) -> HttpResponse:
"""Manage Github's check suites."""
data = loads(request.body.decode())
slug = slugify(data['repository']['name'])
......@@ -36,39 +40,43 @@ def check_suite(request: HttpRequest, rep: str) -> HttpResponse:
if 'ros-release' in slug: # Don't run check suites on ros-release repositories
return HttpResponse(rep)
models.GithubCheckSuite.objects.get_or_create(id=data['check_suite']['id'])
await sync_to_async(models.GithubCheckSuite.objects.get_or_create)(id=data['check_suite']['id'])
return HttpResponse(rep)
def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
async def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
"""Manage Github's Pull Requests."""
logger.info('process gh pr')
data = loads(request.body.decode())
event = data['action']
branch = f'pr/{data["number"]}'
login = slugify(data["pull_request"]["head"]["repo"]["owner"]["login"])
namespace = get_object_or_404(Namespace, slug_github=slugify(data['repository']['owner']['login']))
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
git_repo = project.git()
namespace = await sync_to_async(get_object_or_404)(Namespace,
slug_github=slugify(data['repository']['owner']['login']))
project = await sync_to_async(get_object_or_404)(Project,
main_namespace=namespace,
slug=slugify(data['repository']['name']))
git_repo = await sync_to_async(project.git)()
logger.debug(f'{namespace.slug}/{project.slug}: Pull request on {branch}: {event}')
# Prevent pull requests on master when necessary
if event in ['opened', 'reopened']:
gh = project.github()
pr = gh.get_pull(data["number"])
gh = await sync_to_async(project.github)()
pr = await sync_to_async(gh.get_pull)(data["number"])
pr_branch = pr.base.ref
if not project.accept_pr_to_master and pr_branch == 'master' \
and 'devel' in [b.name for b in gh.get_branches()] and login != namespace.slug_github:
if not project.accept_pr_to_master and pr_branch == 'master' and 'devel' in [
b.name for b in await sync_to_async(gh.get_branches)()
] and login != namespace.slug_github:
logger.info(f"{namespace.slug}/{project.slug}: New pr {data['number']} to master")
pr.create_issue_comment("Hi ! This project doesn't usually accept pull requests on master. If this wasn't "
"intentionnal, you can change the base branch of this pull request to devel "
"(No need to close it for that). Best, a bot.")
await sync_to_async(pr.create_issue_comment)(PR_MASTER_MSG)
gh_remote_name = f'github/{login}'
if gh_remote_name not in git_repo.remotes:
remote = git_repo.create_remote(gh_remote_name, data["pull_request"]["head"]["repo"]["clone_url"])
remote = await sync_to_async(git_repo.create_remote)(gh_remote_name,
data["pull_request"]["head"]["repo"]["clone_url"])
else:
remote = git_repo.remote(gh_remote_name)
remote = await sync_to_async(git_repo.remote)(gh_remote_name)
# Sync the pull request with the pr/XX branch on Gitlab
if event in ['opened', 'reopened', 'synchronize']:
......@@ -79,12 +87,13 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
if branch in git_repo.branches:
git_repo.heads[branch].commit = commit
else:
git_repo.create_head(branch, commit=commit)
await sync_to_async(git_repo.create_head)(branch, commit=commit)
# Create a gitlab remote if it doesn't exist
gl_remote_name = f'gitlab/{namespace.slug}'
if gl_remote_name not in git_repo.remotes:
git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
url = await sync_to_async(project.remote_url_gitlab)()
await sync_to_async(git_repo.create_remote)(gl_remote_name, url=url)
# Push the changes to gitlab
logger.info(f'{namespace.slug}/{project.slug}: Pushing {commit} on {branch} on gitlab')
......@@ -99,14 +108,16 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
if branch in git_repo.branches:
git_repo.delete_head(branch, force=True)
git_repo.delete_remote(gh_remote_name)
project.gitlab().branches.delete(branch)
gitlab = await sync_to_async(project.gitlab)()
await sync_to_async(gitlab.branches.delete)(branch)
logger.info(f'{namespace.slug}/{project.slug}: Deleted branch {branch}')
return HttpResponse(rep)
def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
async def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
"""Someone pushed on github or gitlab. Synchronise local & remote repos."""
logger.debug('start gh.views.push')
data = loads(request.body.decode())
slug = slugify(data['repository']['name'])
......@@ -114,18 +125,20 @@ def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
return HttpResponse(rep)
if source == SOURCES.gitlab:
namespace = get_object_or_404(Namespace,
slug_gitlab=slugify(data['project']['path_with_namespace'].split('/')[0]))
namespace = await sync_to_async(get_object_or_404)(Namespace,
slug_gitlab=slugify(
data['project']['path_with_namespace'].split('/')[0]))
else:
namespace = get_object_or_404(Namespace, slug_github=slugify(data['repository']['owner']['login']))
namespace = await sync_to_async(get_object_or_404)(Namespace,
slug_github=slugify(data['repository']['owner']['login']))
project = get_object_or_404(Project, main_namespace=namespace, slug=slug)
project = await sync_to_async(get_object_or_404)(Project, main_namespace=namespace, slug=slug)
branch = data['ref'][11:] # strip 'refs/heads/'
commit = data['after']
gl_remote_name = f'gitlab/{namespace.slug}'
gh_remote_name = f'github/{namespace.slug}'
git_repo = project.git()
git_repo = await sync_to_async(project.git)()
logger.debug(f'{namespace.slug}/{slug}: Push detected on {source.name} {branch} (commit {commit})')
if branch.startswith('pr/'): # Don't sync pr/XX branches here, they are already handled by pull_request()
......@@ -136,16 +149,18 @@ def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
# Fetch the latest commit from gitlab
if gl_remote_name in git_repo.remotes:
gl_remote = git_repo.remote(gl_remote_name)
gl_remote = await sync_to_async(git_repo.remote)(gl_remote_name)
else:
gl_remote = git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
url = await sync_to_async(project.remote_url_gitlab)()
gl_remote = await sync_to_async(git_repo.create_remote)(gl_remote_name, url=url)
gl_remote.fetch()
# Fetch the latest commit from github
if gh_remote_name in git_repo.remotes:
gh_remote = git_repo.remote(gh_remote_name)
gh_remote = await sync_to_async(git_repo.remote)(gh_remote_name)
else:
gh_remote = git_repo.create_remote(gh_remote_name, url=project.remote_url_github())
url = await sync_to_async(project.remote_url_github)()
gh_remote = await sync_to_async(git_repo.create_remote)(gh_remote_name, url=url)
gh_remote.fetch()
# The branch was deleted on one remote, delete the branch on the other remote as well
......@@ -153,9 +168,11 @@ def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
if branch in git_repo.branches:
git_repo.delete_head(branch, force=True)
if source == SOURCES.gitlab:
project.github().get_git_ref(f'heads/{branch}').delete()
github = await sync_to_async(project.github)()
github.get_git_ref(f'heads/{branch}').delete()
else:
project.gitlab().branches.delete(branch)
gitlab = await sync_to_async(project.gitlab)()
gitlab.branches.delete(branch)
logger.info(f'{namespace.slug}/{slug}: Deleted branch {branch}')
return HttpResponse(rep)
......@@ -170,16 +187,16 @@ def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
if branch in git_repo.branches:
git_repo.heads[branch].commit = commit
else:
git_repo.create_head(branch, commit=commit)
await sync_to_async(git_repo.create_head)(branch, commit=commit)
# Push the changes to other remote
try:
if source == SOURCES.gitlab and (branch not in gh_remote.refs or str(gh_remote.refs[branch].commit) != commit):
logger.info(f'{namespace.slug}/{slug}: Pushing {commit} on {branch} on github')
git_repo.git.push(gh_remote_name, branch)
await sync_to_async(git_repo.git.push)(gh_remote_name, branch)
elif branch not in gl_remote.refs or str(gl_remote.refs[branch].commit) != commit:
logger.info(f'{namespace.slug}/{slug}: Pushing {commit} on {branch} on gitlab')
git_repo.git.push(gl_remote_name, branch)
await sync_to_async(git_repo.git.push)(gl_remote_name, branch)
else:
return HttpResponse('already synced')
except git.exc.GitCommandError:
......@@ -187,19 +204,24 @@ def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
logger.exception(f'{namespace.slug}/{slug}: Forge sync failed')
message = traceback.format_exc()
message = re.sub(r'://.*@', '://[REDACTED]@', message) # Hide access tokens in the mail
mail_admins(f'Forge sync failed for {namespace.slug}/{slug}', message)
await sync_to_async(mail_admins)(f'Forge sync failed for {namespace.slug}/{slug}', message)
return HttpResponse(rep)
def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
async def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
"""Something happened on a Gitlab pipeline. Tell Github if necessary."""
logger.debug('start gh.views.pipeline')
data = loads(request.body.decode())
branch, commit, gl_status, pipeline_id = (data['object_attributes'][key] for key in ['ref', 'sha', 'status', 'id'])
namespace = get_object_or_404(Namespace, slug_gitlab=slugify(data['project']['path_with_namespace'].split('/')[0]))
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['project']['name']))
gh_repo = project.github()
ci_web_url = project.url_gitlab() + '/pipelines/' + str(pipeline_id)
namespace = await sync_to_async(get_object_or_404)(Namespace,
slug_gitlab=slugify(
data['project']['path_with_namespace'].split('/')[0]))
project = await sync_to_async(get_object_or_404)(Project,
main_namespace=namespace,
slug=slugify(data['project']['name']))
gh_repo = await sync_to_async(project.github)()
ci_web_url = f'{project.url_gitlab()}/pipelines/{pipeline_id}'
logger.debug(f'{namespace.slug}/{project.slug}: Pipeline #{pipeline_id} on commit {commit} for branch {branch}, '
f'status: {gl_status}')
......@@ -207,11 +229,14 @@ def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
if gl_status in ['pending', 'success', 'failed']:
gh_status = gl_status if gl_status != 'failed' else 'failure'
if branch.startswith('pr/'):
gh_repo.get_commit(sha=commit).create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
sha = await sync_to_async(gh_repo.get_commit)(sha=commit)
await sync_to_async(sha.create_status)(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
else:
try:
gh_commit = gh_repo.get_branch(branch).commit
gh_commit.create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
sha = await sync_to_async(gh_repo.get_branch)(branch)
await sync_to_async(sha.commit.create_status)(state=gh_status,
target_url=ci_web_url,
context='gitlab-ci')
except github.GithubException as e:
if e.status == 404:
# Happens when a new branch is created on gitlab and the pipeline event comes before the push event
......@@ -222,13 +247,16 @@ def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
return HttpResponse(rep)
@sync_to_async
@csrf_exempt
def webhook(request: HttpRequest) -> HttpResponse:
@async_to_sync
async def webhook(request: HttpRequest) -> HttpResponse:
"""
Process request incoming from a github webhook.
thx https://simpleisbetterthancomplex.com/tutorial/2016/10/31/how-to-handle-github-webhooks-using-django.html
"""
logger.info('GH webhook')
# validate ip source
forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR').split(', ')[0]
# networks = httpx.get('https://api.github.com/meta').json()['hooks'] # Fails if API rate limit exceeded
......@@ -257,17 +285,19 @@ def webhook(request: HttpRequest) -> HttpResponse:
if event == 'ping':
return HttpResponse('pong')
if event == 'push':
return push(request, SOURCES.github, 'push event detected')
return await push(request, SOURCES.github, 'push event detected')
if event == 'check_suite':
return check_suite(request, 'check_suite event detected')
return await check_suite(request, 'check_suite event detected')
if event == 'pull_request':
return pull_request(request, 'pull_request event detected')
return await pull_request(request, 'pull_request event detected')
return HttpResponseForbidden('event not found')
@sync_to_async
@csrf_exempt
def gl_webhook(request: HttpRequest) -> HttpResponse:
@async_to_sync
async def gl_webhook(request: HttpRequest) -> HttpResponse:
"""Process request incoming from a gitlab webhook."""
# validate ip source
......@@ -288,8 +318,8 @@ def gl_webhook(request: HttpRequest) -> HttpResponse:
if event == 'ping':
return HttpResponse('pong')
elif event == 'Pipeline Hook':
return pipeline(request, 'pipeline event detected')
return await pipeline(request, 'pipeline event detected')
elif event == 'Push Hook':
return push(request, SOURCES.gitlab, 'push event detected')
return await push(request, SOURCES.gitlab, 'push event detected')
return HttpResponseForbidden('event not found')
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment