Commit 8a48591a authored by Tom Pillot's avatar Tom Pillot
Browse files

Sync working both ways between gitlab and github

parent 0a3e0ac8
......@@ -6,6 +6,7 @@ from json import loads
from pprint import pprint
import git
import github
import requests
from autoslug.utils import slugify
from django.conf import settings
......@@ -18,6 +19,7 @@ from django.views.decorators.csrf import csrf_exempt
from dashboard.middleware import ip_laas
from rainboard.models import Namespace, Project
from rainboard.utils import SOURCES
from . import models
......@@ -28,26 +30,6 @@ def check_suite(request: HttpRequest, rep: str) -> HttpResponse:
return HttpResponse(rep)
def push_to_gitlab(project: Project, namespace: Namespace, commit: str, branch: str, force: bool):
"""Push project repo to Gitlab on given branch."""
git_repo = project.git()
# Update branch to the latest commit
if branch in git_repo.branches:
git_repo.heads[branch].commit = commit
else:
git_repo.create_head(branch, commit=commit)
# Create a gitlab remote if it doesn't exist
gl_remote_name = f'gitlab/{namespace.slug}'
if gl_remote_name not in git_repo.remotes:
git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
# Push the changes to gitlab
print(f'Pushing {commit} on {branch} on gitlab')
git_repo.git.push(gl_remote_name, branch, force=force)
def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
"""Manage Github's Pull Requests."""
data = loads(request.body.decode())
......@@ -80,11 +62,24 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
remote.fetch()
commit = data['pull_request']['head']['sha']
# Update branch to the latest commit
if branch in git_repo.branches:
git_repo.heads[branch].commit = commit
else:
git_repo.create_head(branch, commit=commit)
# Create a gitlab remote if it doesn't exist
gl_remote_name = f'gitlab/{namespace.slug}'
if gl_remote_name not in git_repo.remotes:
git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
# Push the changes to gitlab
print(f'Pushing {commit} on {branch} on gitlab')
try:
push_to_gitlab(project, namespace, commit, branch, force=False)
git_repo.git.push(gl_remote_name, branch)
except git.exc.GitCommandError:
print(f'Failed to push on {branch}, trying to force push.')
push_to_gitlab(project, namespace, commit, branch, force=True)
git_repo.git.push(gl_remote_name, branch, force=True)
# The pull request was closed, delete the branch pr/XX on Gitlab
elif event == 'closed':
......@@ -97,45 +92,72 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
return HttpResponse(rep)
def push(request: HttpRequest, rep: str) -> HttpResponse:
"""Someone pushed on github. Synchronise local repo & gitlab."""
def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
"""Someone pushed on github or gitlab. Synchronise local & remote repos."""
data = loads(request.body.decode())
# namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['name']))
namespace = get_object_or_404(Namespace, slug='tpillot')
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
branch = data['ref'][11:] # strip 'refs/heads/'
commit = data['after']
gl_remote_name = f'gitlab/{namespace.slug}'
gh_remote_name = f'github/{namespace.slug}'
git_repo = project.git()
print(f'Push detected on github: {branch}')
print(f'Push detected on {source.name}: {branch} (commit {commit})')
if branch.startswith('pr/'): # Don't sync pr/XX branches here, they are already handled by pull_request()
return HttpResponse(rep)
# Fetch the latest commit from gitlab
if gl_remote_name in git_repo.remotes:
gl_remote = git_repo.remote(gl_remote_name)
else:
gl_remote = git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
gl_remote.fetch()
# Fetch the changes from github
# Fetch the latest commit from github
if gh_remote_name in git_repo.remotes:
gh_remote = git_repo.remote(gh_remote_name)
else:
gh_remote = git_repo.create_remote(gh_remote_name, url=project.remote_url_github())
gh_remote.fetch()
# The branch was deleted, delete the branch on Gitlab
# The branch was deleted on one remote, delete the branch on the other remote as well
if commit == "0000000000000000000000000000000000000000":
print("Branch deleted")
if branch in git_repo.branches:
git_repo.delete_head(branch, force=True)
project.gitlab().branches.delete(branch)
if source == SOURCES.gitlab:
project.github().get_git_ref(f'heads/{branch}').delete()
else:
project.gitlab().branches.delete(branch)
print(f"Deleted branch {branch}")
return HttpResponse(rep)
# Make sure we fetched the latest commit
gh_ref = gh_remote.refs[branch]
if str(gh_ref.commit) != commit:
fail = f'Push: wrong commit: {gh_ref.commit} vs {commit}'
ref = gl_remote.refs[branch] if source == SOURCES.gitlab else gh_remote.refs[branch]
if str(ref.commit) != commit:
fail = f'Push: wrong commit: {ref.commit} vs {commit}'
print(fail)
return HttpResponseBadRequest(fail)
if data['forced']:
print('Force push detected, sync canceled.')
# TODO: Send mail to admins
# Update the branch to the latest commit
if branch in git_repo.branches:
git_repo.heads[branch].commit = commit
else:
push_to_gitlab(project, namespace, commit, branch, force=False)
git_repo.create_head(branch, commit=commit)
# Push the changes to other remote
try:
if source == SOURCES.gitlab and (branch not in gh_remote.refs or str(gh_remote.refs[branch].commit) != commit):
print(f'Pushing {commit} on {branch} on github.')
git_repo.git.push(gh_remote_name, branch)
elif branch not in gl_remote.refs or str(gl_remote.refs[branch].commit) != commit:
print(f'Pushing {commit} on {branch} on gitlab.')
git_repo.git.push(gl_remote_name, branch)
except git.exc.GitCommandError:
# Probably failed because of a force push
# TODO: send an email to the admins ?
raise
return HttpResponse(rep)
......@@ -156,12 +178,18 @@ def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
if gl_status in ['pending', 'success', 'failed']:
gh_status = gl_status if gl_status != 'failed' else 'failure'
if branch.startswith('pr/'):
pr_number = int(branch[3:])
gh_repo.get_commit(sha=commit).create_status(state=gh_status, target_url=ci_web_url,
context='gitlab-ci')
else:
gh_commit = gh_repo.get_branch(branch).commit
gh_commit.create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
try:
gh_commit = gh_repo.get_branch(branch).commit
gh_commit.create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
except github.GithubException as e:
if e.status == 404:
# Happens when a new branch is created on gitlab and the pipeline event comes before the push event
print(f"Branch {branch} does not exist on github, unable to report the pipeline status.")
else:
raise
return HttpResponse(rep)
......@@ -201,7 +229,7 @@ def webhook(request: HttpRequest) -> HttpResponse:
pprint(loads(request.body.decode()))
return HttpResponse('pong')
if event == 'push':
return push(request, 'push event detected')
return push(request, SOURCES.github, 'push event detected')
if event == 'check_suite':
return check_suite(request, 'check_suite event detected')
if event == 'pull_request':
......@@ -232,7 +260,9 @@ def gl_webhook(request: HttpRequest) -> HttpResponse:
if event == 'ping':
pprint(loads(request.body.decode()))
return HttpResponse('pong')
if event == 'Pipeline Hook':
elif event == 'Pipeline Hook':
return pipeline(request, 'pipeline event detected')
elif event == 'Push Hook':
return push(request, SOURCES.gitlab, 'push event detected')
return HttpResponseForbidden('event not found')
......@@ -25,7 +25,7 @@ class RainboardTests(TestCase):
self.assertEqual(models.License.objects.count(), license_count+1)
self.assertEqual(models.Project.objects.count(), project_count+1)
project = models.Project.objects.first()
project = models.Project.objects.get(name='Rainboard Tests')
self.assertEqual(project.slug, 'rainboard-tests')
self.assertEqual(project.registry(), 'memmos.laas.fr:5000')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment