Commit 615708a2 authored by Tom Pillot's avatar Tom Pillot
Browse files

Test sync with push on github

parent 510930e9
Pipeline #10351 failed with stage
in 2 minutes and 25 seconds
.env
db.sqlite3
__pycache__/
.idea
\ No newline at end of file
.idea
tmp/
\ No newline at end of file
......@@ -21,6 +21,6 @@ build:
image: docker
script:
- docker build -t ${DOCKER_TAG} .
- docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py migrate
- docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py test
- docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e GITHUB_PIPELINE_TOKEN -e GITLAB_PIPELINE_TOKEN -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py migrate
- docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e GITHUB_PIPELINE_TOKEN -e GITLAB_PIPELINE_TOKEN -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py test
- docker push ${DOCKER_TAG}
......@@ -34,9 +34,7 @@ RUN pipenv install --system --deploy
ADD . .
CMD mkdir -p /srv/dashboard/robotpkg \
&& git clone https://github.com/Ozon2/example-adder.git /srv/dashboard/robotpkg/example-adder || true \
&& while ! nc -z postgres 5432; do sleep 1; done \
CMD while ! nc -z postgres 5432; do sleep 1; done \
&& ./manage.py migrate \
&& ./manage.py collectstatic --no-input \
&& gunicorn \
......
......@@ -28,7 +28,7 @@ SERVER_EMAIL = f'{EMAIL_USER}+{PROJECT}@{EMAIL_FQDN}'
DEFAULT_FROM_EMAIL = f'{PROJECT_VERBOSE} <{EMAIL_USER}@{EMAIL_FQDN}>'
EMAIL_BACKEND = 'django.core.mail.backends.%s' % ('filebased.EmailBackend' if DEBUG else 'smtp.EmailBackend')
EMAIL_SUBJECT_PREFIX = f'[{PROJECT_VERBOSE}] '
EMAIL_FILE_PATH = 'tmp/email-messages/'
EMAIL_FILE_PATH = 'email/'
ADMINS = ((os.environ.get('ADMIN_NAME',
f'{PROJECT_VERBOSE} webmaster'), os.environ.get('ADMIN_MAIL', f'webmaster@{DOMAIN_NAME}')), )
......
import hmac
import time
from hashlib import sha1
from os import getenv
from pathlib import Path
from unittest import mock
import git
from autoslug.utils import slugify
from django.conf import settings
from django.test import TestCase
from django.urls import reverse
......@@ -12,7 +16,56 @@ from rainboard.models import Project, Namespace, Forge
class GhTests(TestCase):
def send_github_event(self, event, data=None):
@classmethod
def setUpClass(cls):
"""Run only once, avoid getting the objects in each test."""
super(GhTests, cls).setUpClass()
if getenv('GITLAB_PIPELINE_TOKEN'): # Inside gitlab pipeline
cls.namespace, _ = Namespace.objects.get_or_create(name='gsaurel',
slug_gitlab='gsaurel',
slug_github='hrp2-14')
cls.project, _ = Project.objects.get_or_create(name='rainboard-tests',
main_namespace=Namespace.objects.get(name='gsaurel'),
main_forge=Forge.objects.get(name='Gitlab'))
else:
cls.namespace, _ = Namespace.objects.get_or_create(name='tpillot',
slug_gitlab=slugify('tpillot'),
slug_github=slugify('Ozon2'))
cls.project, _ = Project.objects.get_or_create(name='example-adder',
main_namespace=Namespace.objects.get(name='tpillot'),
main_forge=Forge.objects.get(name='Gitlab'))
cls.github = cls.project.github()
cls.gitlab = cls.project.gitlab()
@classmethod
@mock.patch.object(Project, 'git_path', return_value=Path('tmp'))
def tearDownClass(cls, mock_git_path):
"""Force both repos to be synced after the tests."""
print('Syncing both repos after the tests ...')
for branch in ('master', 'devel'):
last_commit_github = cls.github.get_branch(branch).commit.sha
last_commit_gitlab = cls.gitlab.commits.list(ref_name=branch)[0].id
if last_commit_github != last_commit_gitlab:
print(f'{branch} is not synced, force pushing to github')
git_repo = cls.project.git()
try:
git_repo.git.push(f'github/{cls.namespace.slug}', branch, force=True)
except git.GitCommandError:
print(f'Failed to push on github {branch}')
def send_github_event(self, event, last_commit=''):
data = {
'repository': {
'name': 'example-adder',
'owner': {
'name': 'Ozon2'
}
},
'ref': 'refs/heads/master',
'after': last_commit
}
encoded_data = self.client._encode_json({} if data is None else data, content_type='application/json')
request_body = self.client._encode_data(encoded_data, content_type='application/json')
msg = force_bytes(request_body)
......@@ -70,35 +123,25 @@ class GhTests(TestCase):
HTTP_X_GITLAB_EVENT='ping')
self.assertEqual(response.status_code, 200)
@mock.patch.object(Project, 'git_path')
def test_push(self, git_path):
git_path.return_value = Path('tmp')
namespace, _ = Namespace.objects.get_or_create(name='tpillot')
project, _ = Project.objects.get_or_create(name='example-adder',
main_namespace=Namespace.objects.get(name='tpillot'),
main_forge=Forge.objects.get(name='Gitlab'))
git_repo = project.git()
github = project.github()
last_commit = github.get_branch('master').commit.sha
@mock.patch.object(Project, 'git_path', return_value=Path('tmp'))
def test_push(self, mock_git_path):
last_commit = self.github.get_branch('master').commit.sha
# Push when both repos are already synced
response = self.send_github_event(
'push', {
'repository': {
'name': 'example-adder',
'owner': {
'name': 'tpillot'
}
},
'ref': 'refs/heads/master',
'after': last_commit
})
response = self.send_github_event('push', last_commit)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content.decode(), 'already synced')
# Push new commit on github
file = github.get_contents('README.md')
github.update_file('README.md', message='Test push on github', content=last_commit[:8], sha=file.sha)
# TODO: Make sure both repos are still synced
file = self.github.get_contents('README.md')
self.github.update_file('README.md', message='Test push on github', content=last_commit[:8], sha=file.sha)
last_commit_github = self.github.get_branch('master').commit.sha
self.assertNotEqual(last_commit, last_commit_github)
response = self.send_github_event('push', last_commit_github)
self.assertEqual(response.status_code, 200)
self.assertEqual(response.content.decode(), 'push event detected')
last_commit_gitlab = self.gitlab.commits.list()[0].id
self.assertEqual(last_commit_github, last_commit_gitlab)
......@@ -12,6 +12,7 @@ import github
from autoslug.utils import slugify
from django.conf import settings
from django.core.mail import mail_admins
from django.db.models import Q
from django.http import HttpRequest
from django.http.response import (HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseRedirect,
HttpResponseServerError)
......@@ -40,8 +41,7 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
event = data['action']
branch = f'pr/{data["number"]}'
# namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['login']))
namespace = get_object_or_404(Namespace, slug='tpillot')
namespace = get_object_or_404(Namespace, slug_github=slugify(data['repository']['owner']['name']))
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
git_repo = project.git()
logger.debug(f'{namespace.slug}/{project.slug}: Pull request on {branch}: {event}')
......@@ -99,9 +99,14 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
"""Someone pushed on github or gitlab. Synchronise local & remote repos."""
data = loads(request.body.decode())
# namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['name']))
namespace = get_object_or_404(Namespace, slug='tpillot')
if source == SOURCES.gitlab:
namespace = get_object_or_404(Namespace, slug_gitlab=slugify(data['project']['namespace']))
else:
namespace = get_object_or_404(Namespace, slug_github=slugify(data['repository']['owner']['name']))
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
branch = data['ref'][11:] # strip 'refs/heads/'
commit = data['after']
gl_remote_name = f'gitlab/{namespace.slug}'
......@@ -175,8 +180,7 @@ def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
"""Something happened on a Gitlab pipeline. Tell Github if necessary."""
data = loads(request.body.decode())
branch, commit, gl_status, pipeline_id = (data['object_attributes'][key] for key in ['ref', 'sha', 'status', 'id'])
# namespace = get_object_or_404(Namespace, slug=slugify(data['project']['namespace']))
namespace = get_object_or_404(Namespace, slug='tpillot')
namespace = get_object_or_404(Namespace, slug_gitlab=slugify(data['project']['namespace']))
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['project']['name']))
gh_repo = project.github()
ci_web_url = project.url_gitlab() + '/pipelines/' + str(pipeline_id)
......
......@@ -10,10 +10,16 @@ from rainboard.utils import SOURCES
def forges(apps, schema_editor):
Forge = apps.get_model('rainboard', 'Forge')
gitlab_token = os.getenv('GITLAB_TOKEN')
if gitlab_token is None:
gitlab_token = os.getenv('GITLAB_PIPELINE_TOKEN')
Forge.objects.create(name='Gitlab', source=SOURCES.gitlab, url='https://gepgitlab.laas.fr',
token=os.getenv('GITLAB_TOKEN'))
token=gitlab_token)
github_token = os.getenv('GITHUB_TOKEN')
if github_token is None:
github_token = os.getenv('GITHUB_PIPELINE_TOKEN')
Forge.objects.create(name='Github', source=SOURCES.github, url='https://github.com',
token=os.getenv('GITHUB_TOKEN'))
token=github_token)
Forge.objects.create(name='Redmine', source=SOURCES.redmine, url='https://redmine.laas.fr',
token=os.getenv('REDMINE_TOKEN'))
Forge.objects.create(name='Openrobots', source=SOURCES.redmine, url='https://git.openrobots.org',
......
# Generated by Django 3.0.8 on 2020-07-29 08:29
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('rainboard', '0045_project_accept_pr_to_master'),
]
operations = [
migrations.AddField(
model_name='namespace',
name='slug_github',
field=models.CharField(default='', max_length=200),
),
migrations.AddField(
model_name='namespace',
name='slug_gitlab',
field=models.CharField(default='', max_length=200),
),
]
......@@ -5,7 +5,7 @@ from django.db import migrations
def add_namespace(apps, schema_editor):
Namespace = apps.get_model('rainboard', 'Namespace')
Namespace.objects.create(name='tpillot', group=False)
Namespace.objects.create(name='tpillot', slug_gitlab='tpillot', slug_github='Ozon2')
def add_project(apps, schema_editor):
......@@ -37,10 +37,10 @@ def add_project(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('rainboard', '0045_project_accept_pr_to_master'),
('rainboard', '0046_namespace_slug_gitlab_github'),
]
operations = [
migrations.RunPython(add_namespace),
migrations.RunPython(add_project),
# migrations.RunPython(add_namespace),
# migrations.RunPython(add_project),
]
......@@ -22,7 +22,7 @@ def add_robotpkg(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('rainboard', '0046_example-adder'),
('rainboard', '0047_example-adder'),
]
operations = [
......
......@@ -51,6 +51,15 @@ GITLAB_STATUS = {'failed': False, 'success': True, 'pending': None, 'skipped': N
class Namespace(NamedModel):
group = models.BooleanField(default=False)
slug_gitlab = models.CharField(max_length=200, default='')
slug_github = models.CharField(max_length=200, default='')
def save(self, *args, **kwargs):
if self.slug_gitlab is None:
self.slug_gitlab = self.slug
if self.slug_github is None:
self.slug_github = self.slug
super(Namespace, self).save(*args, **kwargs)
class License(models.Model):
......@@ -214,13 +223,12 @@ class Project(Links, NamedModel, TimeStampedModel):
def github(self):
github_forge = Forge.objects.get(slug='github')
gh = Github(github_forge.token)
# return gh.get_repo(f'{self.main_namespace.slug}/{self.slug}')
return gh.get_repo(f'Ozon2/{self.slug}')
return gh.get_repo(f'{self.main_namespace.slug_github}/{self.slug}')
def gitlab(self):
gitlab_forge = Forge.objects.get(slug='gitlab')
gl = Gitlab(gitlab_forge.url, private_token=gitlab_forge.token)
gl_repo = gl.projects.get(f'{self.main_namespace.slug}/{self.slug}')
gl_repo = gl.projects.get(f'{self.main_namespace.slug_gitlab}/{self.slug}')
return gl_repo
def main_repo(self):
......@@ -403,15 +411,14 @@ class Project(Links, NamedModel, TimeStampedModel):
return f'https://travis-ci.org/{self.main_namespace.slug}/{self.slug}'
def url_gitlab(self):
return f'https://gitlab.laas.fr/{self.main_namespace.slug}/{self.slug}'
return f'https://gitlab.laas.fr/{self.main_namespace.slug_gitlab}/{self.slug}'
def remote_url_gitlab(self):
gitlab_forge = Forge.objects.get(source=SOURCES.gitlab)
return self.url_gitlab().replace('://', f'://gitlab-ci-token:{gitlab_forge.token}@')
def url_github(self):
# return f'https://github.com/{self.main_namespace.slug}/{self.slug}'
return f'https://github.com/Ozon2/{self.slug}'
return f'https://github.com/{self.main_namespace.slug_github}/{self.slug}'
def remote_url_github(self):
github_forge = Forge.objects.get(source=SOURCES.github)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment