Commit 8a48591a authored by Tom Pillot's avatar Tom Pillot
Browse files

Sync working both ways between gitlab and github

parent 0a3e0ac8
...@@ -6,6 +6,7 @@ from json import loads ...@@ -6,6 +6,7 @@ from json import loads
from pprint import pprint from pprint import pprint
import git import git
import github
import requests import requests
from autoslug.utils import slugify from autoslug.utils import slugify
from django.conf import settings from django.conf import settings
...@@ -18,6 +19,7 @@ from django.views.decorators.csrf import csrf_exempt ...@@ -18,6 +19,7 @@ from django.views.decorators.csrf import csrf_exempt
from dashboard.middleware import ip_laas from dashboard.middleware import ip_laas
from rainboard.models import Namespace, Project from rainboard.models import Namespace, Project
from rainboard.utils import SOURCES
from . import models from . import models
...@@ -28,26 +30,6 @@ def check_suite(request: HttpRequest, rep: str) -> HttpResponse: ...@@ -28,26 +30,6 @@ def check_suite(request: HttpRequest, rep: str) -> HttpResponse:
return HttpResponse(rep) return HttpResponse(rep)
def push_to_gitlab(project: Project, namespace: Namespace, commit: str, branch: str, force: bool):
"""Push project repo to Gitlab on given branch."""
git_repo = project.git()
# Update branch to the latest commit
if branch in git_repo.branches:
git_repo.heads[branch].commit = commit
else:
git_repo.create_head(branch, commit=commit)
# Create a gitlab remote if it doesn't exist
gl_remote_name = f'gitlab/{namespace.slug}'
if gl_remote_name not in git_repo.remotes:
git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
# Push the changes to gitlab
print(f'Pushing {commit} on {branch} on gitlab')
git_repo.git.push(gl_remote_name, branch, force=force)
def pull_request(request: HttpRequest, rep: str) -> HttpResponse: def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
"""Manage Github's Pull Requests.""" """Manage Github's Pull Requests."""
data = loads(request.body.decode()) data = loads(request.body.decode())
...@@ -80,11 +62,24 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse: ...@@ -80,11 +62,24 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
remote.fetch() remote.fetch()
commit = data['pull_request']['head']['sha'] commit = data['pull_request']['head']['sha']
# Update branch to the latest commit
if branch in git_repo.branches:
git_repo.heads[branch].commit = commit
else:
git_repo.create_head(branch, commit=commit)
# Create a gitlab remote if it doesn't exist
gl_remote_name = f'gitlab/{namespace.slug}'
if gl_remote_name not in git_repo.remotes:
git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
# Push the changes to gitlab
print(f'Pushing {commit} on {branch} on gitlab')
try: try:
push_to_gitlab(project, namespace, commit, branch, force=False) git_repo.git.push(gl_remote_name, branch)
except git.exc.GitCommandError: except git.exc.GitCommandError:
print(f'Failed to push on {branch}, trying to force push.') print(f'Failed to push on {branch}, trying to force push.')
push_to_gitlab(project, namespace, commit, branch, force=True) git_repo.git.push(gl_remote_name, branch, force=True)
# The pull request was closed, delete the branch pr/XX on Gitlab # The pull request was closed, delete the branch pr/XX on Gitlab
elif event == 'closed': elif event == 'closed':
...@@ -97,45 +92,72 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse: ...@@ -97,45 +92,72 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
return HttpResponse(rep) return HttpResponse(rep)
def push(request: HttpRequest, rep: str) -> HttpResponse: def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
"""Someone pushed on github. Synchronise local repo & gitlab.""" """Someone pushed on github or gitlab. Synchronise local & remote repos."""
data = loads(request.body.decode()) data = loads(request.body.decode())
# namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['name'])) # namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['name']))
namespace = get_object_or_404(Namespace, slug='tpillot') namespace = get_object_or_404(Namespace, slug='tpillot')
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name'])) project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
branch = data['ref'][11:] # strip 'refs/heads/' branch = data['ref'][11:] # strip 'refs/heads/'
commit = data['after'] commit = data['after']
gl_remote_name = f'gitlab/{namespace.slug}'
gh_remote_name = f'github/{namespace.slug}' gh_remote_name = f'github/{namespace.slug}'
git_repo = project.git() git_repo = project.git()
print(f'Push detected on github: {branch}') print(f'Push detected on {source.name}: {branch} (commit {commit})')
if branch.startswith('pr/'): # Don't sync pr/XX branches here, they are already handled by pull_request()
return HttpResponse(rep)
# Fetch the latest commit from gitlab
if gl_remote_name in git_repo.remotes:
gl_remote = git_repo.remote(gl_remote_name)
else:
gl_remote = git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
gl_remote.fetch()
# Fetch the changes from github # Fetch the latest commit from github
if gh_remote_name in git_repo.remotes: if gh_remote_name in git_repo.remotes:
gh_remote = git_repo.remote(gh_remote_name) gh_remote = git_repo.remote(gh_remote_name)
else: else:
gh_remote = git_repo.create_remote(gh_remote_name, url=project.remote_url_github()) gh_remote = git_repo.create_remote(gh_remote_name, url=project.remote_url_github())
gh_remote.fetch() gh_remote.fetch()
# The branch was deleted, delete the branch on Gitlab # The branch was deleted on one remote, delete the branch on the other remote as well
if commit == "0000000000000000000000000000000000000000": if commit == "0000000000000000000000000000000000000000":
print("Branch deleted")
if branch in git_repo.branches: if branch in git_repo.branches:
git_repo.delete_head(branch, force=True) git_repo.delete_head(branch, force=True)
project.gitlab().branches.delete(branch) if source == SOURCES.gitlab:
project.github().get_git_ref(f'heads/{branch}').delete()
else:
project.gitlab().branches.delete(branch)
print(f"Deleted branch {branch}")
return HttpResponse(rep) return HttpResponse(rep)
# Make sure we fetched the latest commit # Make sure we fetched the latest commit
gh_ref = gh_remote.refs[branch] ref = gl_remote.refs[branch] if source == SOURCES.gitlab else gh_remote.refs[branch]
if str(gh_ref.commit) != commit: if str(ref.commit) != commit:
fail = f'Push: wrong commit: {gh_ref.commit} vs {commit}' fail = f'Push: wrong commit: {ref.commit} vs {commit}'
print(fail) print(fail)
return HttpResponseBadRequest(fail) return HttpResponseBadRequest(fail)
if data['forced']: # Update the branch to the latest commit
print('Force push detected, sync canceled.') if branch in git_repo.branches:
# TODO: Send mail to admins git_repo.heads[branch].commit = commit
else: else:
push_to_gitlab(project, namespace, commit, branch, force=False) git_repo.create_head(branch, commit=commit)
# Push the changes to other remote
try:
if source == SOURCES.gitlab and (branch not in gh_remote.refs or str(gh_remote.refs[branch].commit) != commit):
print(f'Pushing {commit} on {branch} on github.')
git_repo.git.push(gh_remote_name, branch)
elif branch not in gl_remote.refs or str(gl_remote.refs[branch].commit) != commit:
print(f'Pushing {commit} on {branch} on gitlab.')
git_repo.git.push(gl_remote_name, branch)
except git.exc.GitCommandError:
# Probably failed because of a force push
# TODO: send an email to the admins ?
raise
return HttpResponse(rep) return HttpResponse(rep)
...@@ -156,12 +178,18 @@ def pipeline(request: HttpRequest, rep: str) -> HttpResponse: ...@@ -156,12 +178,18 @@ def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
if gl_status in ['pending', 'success', 'failed']: if gl_status in ['pending', 'success', 'failed']:
gh_status = gl_status if gl_status != 'failed' else 'failure' gh_status = gl_status if gl_status != 'failed' else 'failure'
if branch.startswith('pr/'): if branch.startswith('pr/'):
pr_number = int(branch[3:])
gh_repo.get_commit(sha=commit).create_status(state=gh_status, target_url=ci_web_url, gh_repo.get_commit(sha=commit).create_status(state=gh_status, target_url=ci_web_url,
context='gitlab-ci') context='gitlab-ci')
else: else:
gh_commit = gh_repo.get_branch(branch).commit try:
gh_commit.create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci') gh_commit = gh_repo.get_branch(branch).commit
gh_commit.create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
except github.GithubException as e:
if e.status == 404:
# Happens when a new branch is created on gitlab and the pipeline event comes before the push event
print(f"Branch {branch} does not exist on github, unable to report the pipeline status.")
else:
raise
return HttpResponse(rep) return HttpResponse(rep)
...@@ -201,7 +229,7 @@ def webhook(request: HttpRequest) -> HttpResponse: ...@@ -201,7 +229,7 @@ def webhook(request: HttpRequest) -> HttpResponse:
pprint(loads(request.body.decode())) pprint(loads(request.body.decode()))
return HttpResponse('pong') return HttpResponse('pong')
if event == 'push': if event == 'push':
return push(request, 'push event detected') return push(request, SOURCES.github, 'push event detected')
if event == 'check_suite': if event == 'check_suite':
return check_suite(request, 'check_suite event detected') return check_suite(request, 'check_suite event detected')
if event == 'pull_request': if event == 'pull_request':
...@@ -232,7 +260,9 @@ def gl_webhook(request: HttpRequest) -> HttpResponse: ...@@ -232,7 +260,9 @@ def gl_webhook(request: HttpRequest) -> HttpResponse:
if event == 'ping': if event == 'ping':
pprint(loads(request.body.decode())) pprint(loads(request.body.decode()))
return HttpResponse('pong') return HttpResponse('pong')
if event == 'Pipeline Hook': elif event == 'Pipeline Hook':
return pipeline(request, 'pipeline event detected') return pipeline(request, 'pipeline event detected')
elif event == 'Push Hook':
return push(request, SOURCES.gitlab, 'push event detected')
return HttpResponseForbidden('event not found') return HttpResponseForbidden('event not found')
...@@ -25,7 +25,7 @@ class RainboardTests(TestCase): ...@@ -25,7 +25,7 @@ class RainboardTests(TestCase):
self.assertEqual(models.License.objects.count(), license_count+1) self.assertEqual(models.License.objects.count(), license_count+1)
self.assertEqual(models.Project.objects.count(), project_count+1) self.assertEqual(models.Project.objects.count(), project_count+1)
project = models.Project.objects.first() project = models.Project.objects.get(name='Rainboard Tests')
self.assertEqual(project.slug, 'rainboard-tests') self.assertEqual(project.slug, 'rainboard-tests')
self.assertEqual(project.registry(), 'memmos.laas.fr:5000') self.assertEqual(project.registry(), 'memmos.laas.fr:5000')
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment