Commit 3492c7d3 authored by Tom Pillot's avatar Tom Pillot
Browse files

gitlab pipeline

parent 4004cb11
......@@ -20,9 +20,7 @@ RUN apt-get update -qq && apt-get install -qqy \
netcat-openbsd \
make \
msmtp \
&& curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \
&& echo "deb [arch=armhf] https://download.docker.com/linux/debian buster stable" >> /etc/apt/sources.list \
&& apt-get update -qq && apt-get install -qqy docker-ce
&& curl -sSL https://get.docker.com/ | sh
RUN pip3 install --no-cache-dir \
gunicorn \
......@@ -36,10 +34,8 @@ RUN pipenv install --system --deploy
ADD . .
CMD rm -f /opt/openrobots/etc/robotpkg.conf \
&& git clone git://git.openrobots.org/robots/robotpkg /srv/dashboard/robotpkg || true \
&& git clone git://git.openrobots.org/robots/robotpkg/robotpkg-wip /srv/dashboard/robotpkg/wip || true \
&& /srv/dashboard/robotpkg/bootstrap/bootstrap \
CMD mkdir -p /srv/dashboard/robotpkg \
&& git clone https://github.com/Ozon2/example-adder.git /srv/dashboard/robotpkg/example-adder || true \
&& while ! nc -z postgres 5432; do sleep 1; done \
&& ./manage.py migrate \
&& ./manage.py collectstatic --no-input \
......
"""Views for dashboard_apps."""
import hmac
import git
from hashlib import sha1
from ipaddress import ip_address, ip_network
from json import loads
from pprint import pprint
import git
import requests
from autoslug.utils import slugify
from django.conf import settings
from django.http import HttpRequest
from django.http.response import (HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseRedirect,
......@@ -14,12 +16,8 @@ from django.shortcuts import get_object_or_404, reverse
from django.utils.encoding import force_bytes
from django.views.decorators.csrf import csrf_exempt
import requests
from autoslug.utils import slugify
from dashboard.middleware import ip_laas
from rainboard.models import Forge, Namespace, Project
from rainboard.models import Namespace, Project
from . import models
......@@ -37,7 +35,8 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
branch = f'pr/{data["number"]}'
print(f'Pull request: {event}')
namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['login']))
# namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['login']))
namespace = get_object_or_404(Namespace, slug='tpillot')
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
git_repo = project.git()
......@@ -81,8 +80,8 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
if branch in git_repo.branches:
git_repo.delete_head(branch, force=True)
git_repo.delete_remote(gh_remote_name)
project.gitlab().branches.delete(branch)
print(f'Deleted branch {branch}')
project.gitlab().branches.delete(branch)
print(f'Deleted branch {branch}')
return HttpResponse(rep)
......@@ -90,67 +89,81 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
def push(request: HttpRequest, rep: str) -> HttpResponse:
"""Someone pushed on github. Synchronise local repo & gitlab."""
data = loads(request.body.decode())
namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['name']))
# namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['name']))
namespace = get_object_or_404(Namespace, slug='tpillot')
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
branch_name = data['ref'][11:] # strip 'refs/heads/'
print(f'Push detected on github: {branch_name}')
branch = data['ref'][11:] # strip 'refs/heads/'
gh_remote_name = f'github/{namespace.slug}'
gl_remote_name = f'gitlab/{namespace.slug}'
gh_branch_name = f'{gh_remote_name}/{branch_name}'
gl_branch_name = f'{gl_remote_name}/{branch_name}'
git_repo = project.git()
gh_remote = git_repo.remote(gh_remote_name)
print(f'Push detected on github: {branch}')
# TODO: hard reset if necessary ?
if gh_remote_name in git_repo.remotes:
gh_remote = git_repo.remote(gh_remote_name)
else:
gh_remote = git_repo.create_remote(gh_remote_name, url=project.url_github())
gh_remote.fetch()
# The branch was deleted, delete the branch on Gitlab
if data['after'] == "0000000000000000000000000000000000000000":
print("Branch deleted")
for branch in [gh_branch_name, gl_branch_name, branch_name]:
if branch in git_repo.branches:
git_repo.delete_head(branch, force=True)
project.gitlab().branches.delete(branch_name)
if branch in git_repo.branches:
git_repo.delete_head(branch, force=True)
project.gitlab().branches.delete(branch)
return HttpResponse(rep)
# TODO: Is that necessary ?
gh_ref = gh_remote.refs[branch_name]
gh_ref = gh_remote.refs[branch]
if str(gh_ref.commit) != data['after']:
fail = f'push: wrong commit: {gh_ref.commit} vs {data["after"]}'
fail = f'Push: wrong commit: {gh_ref.commit} vs {data["after"]}'
print(fail)
return HttpResponseBadRequest(fail)
# TODO: Is that necessary ?
if gl_remote_name not in git_repo.remotes:
print(f'project {project} not available on {gl_remote_name}')
return HttpResponse(rep)
# Update branch to the last commit
if branch in git_repo.branches:
git_repo.branches[branch].commit = data['after']
else:
git_repo.create_head(branch, commit=data['after'])
# Update branches to the last commit
for branch in [gh_branch_name, gl_branch_name, branch_name]:
if branch in git_repo.branches:
git_repo.branches[branch].commit = data['after']
else:
git_repo.create_head(branch, commit=data['after'])
if gl_remote_name in git_repo.remotes:
gl_remote = git_repo.remote(gl_remote_name)
else:
gl_remote = git_repo.create_remote(gl_remote_name, url=project.url_gitlab())
# Push the changes to gitlab
# TODO: force push
gl_remote = git_repo.remote(gl_remote_name)
# TODO: Force push
gl_remote.fetch()
if branch_name not in gl_remote.refs or str(gl_remote.refs[branch_name].commit) != data['after']:
print(f'pushing {data["after"]} on {branch_name} on gitlab')
gl_remote.push(branch_name)
if branch not in gl_remote.refs or str(gl_remote.refs[branch].commit) != data['after']:
print(f'pushing {data["after"]} on {branch} on gitlab')
gl_remote.push(branch)
return HttpResponse(rep)
def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
"""Something happened on a Gitlab pipeline. Tell Github if necessary."""
print('pipeline')
data = loads(request.body.decode())
namespace = get_object_or_404(Namespace, slug=slugify(data['project']['namespace']))
branch, commit, gl_status, pipeline_id = (data['object_attributes'][key] for key in ['ref', 'sha', 'status', 'id'])
print(f'Pipeline id #{pipeline_id} on commit {commit} for branch {branch}, status: {gl_status}')
# namespace = get_object_or_404(Namespace, slug=slugify(data['project']['namespace']))
namespace = get_object_or_404(Namespace, slug='tpillot')
project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['project']['name']))
branch, commit, status = (data['object_attributes'][key] for key in ['ref', 'sha', 'status'])
# status in ['pending', 'running', 'success', 'failed']
print(namespace, project, branch, commit, status)
gh_repo = project.github()
ci_web_url = project.url_gitlab() + '/pipelines/' + str(pipeline_id)
# Report the status to Github
if gl_status in ['pending', 'success', 'failed']:
gh_status = gl_status if gl_status != 'failed' else 'failure'
if branch.startswith('pr/'):
pr_number = int(branch[3:])
pr_commit_sha = gh_repo.get_pull(pr_number).head.sha
gh_repo.get_commit(sha=pr_commit_sha).create_status(state=gh_status, target_url=ci_web_url,
context='gitlab-ci')
else:
gh_commit = gh_repo.get_branch(branch).commit
gh_commit.create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
return HttpResponse(rep)
......@@ -162,7 +175,7 @@ def webhook(request: HttpRequest) -> HttpResponse:
thx https://simpleisbetterthancomplex.com/tutorial/2016/10/31/how-to-handle-github-webhooks-using-django.html
"""
# validate ip source
forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR').split(', ')[0]
networks = requests.get('https://api.github.com/meta').json()['hooks']
if not any(ip_address(forwarded_for) in ip_network(net) for net in networks):
print('not from github IP')
......@@ -200,6 +213,8 @@ def webhook(request: HttpRequest) -> HttpResponse:
@csrf_exempt
def gl_webhook(request: HttpRequest) -> HttpResponse:
"""Process request incoming from a gitlab webhook."""
# validate ip source
if not ip_laas(request):
print('not from LAAS IP')
......
# Generated by Django 3.0.8 on 2020-07-24 13:45
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('rainboard', '0044_focal'),
]
operations = [
migrations.AddField(
model_name='project',
name='accept_pr_to_master',
field=models.BooleanField(default=False),
),
]
......@@ -3,6 +3,11 @@
from django.db import migrations
def add_namespace(apps, schema_editor):
Namespace = apps.get_model('rainboard', 'Namespace')
Namespace.objects.create(name='tpillot', group=False)
def add_project(apps, schema_editor):
Project = apps.get_model('rainboard', 'Project')
Namespace = apps.get_model('rainboard', 'Namespace')
......@@ -11,28 +16,7 @@ def add_project(apps, schema_editor):
git_project = Project(
name='example-adder',
public=True,
main_namespace=Namespace.objects.get(name='Gepetto'),
main_forge=Forge.objects.get(name='Gitlab'),
license=None,
homepage=None,
description=None,
version=None,
updated=None,
tests=True,
docs=True,
from_gepetto=True,
cmake_name=None,
archived=False,
suffix='',
allow_format_failure=True,
has_python=True
)
git_project.save()
git_project = Project(
name='talos-data',
public=True,
main_namespace=Namespace.objects.get(name='Gepetto'),
main_namespace=Namespace.objects.get(name='tpillot'),
main_forge=Forge.objects.get(name='Gitlab'),
license=None,
homepage=None,
......@@ -53,9 +37,10 @@ def add_project(apps, schema_editor):
class Migration(migrations.Migration):
dependencies = [
('rainboard', '0044_focal'),
('rainboard', '0045_project_accept_pr_to_master'),
]
operations = [
migrations.RunPython(add_namespace),
migrations.RunPython(add_project),
]
......@@ -7,22 +7,22 @@ path = settings.RAINBOARD_RPKG
def add_robotpkg(apps, schema_editor):
for project in [Project.objects.get(slug='example-adder'), Project.objects.get(slug='talos-data')]:
project = Project.objects.get(slug='example-adder')
for slug in [project.slug, project.slug.replace('_', '-')]:
for pkg in itertools.chain(path.glob(f'*/{slug}{project.suffix}'), path.glob(f'*/py-{slug}{project.suffix}')):
obj, created = Robotpkg.objects.get_or_create(name=pkg.name, category=pkg.parent.name, project=project)
if created:
print(project, pkg)
obj.update(pull=False)
for slug in [project.slug, project.slug.replace('_', '-')]:
for pkg in itertools.chain(path.glob(f'*/{slug}{project.suffix}'), path.glob(f'*/py-{slug}{project.suffix}')):
obj, created = Robotpkg.objects.get_or_create(name=pkg.name, category=pkg.parent.name, project=project)
if created:
print(project, pkg)
obj.update(pull=False)
for rpkg in project.robotpkg_set.all():
rpkg.update_images()
for rpkg in project.robotpkg_set.all():
rpkg.update_images()
class Migration(migrations.Migration):
dependencies = [
('rainboard', '0045_example-adder'),
('rainboard', '0046_example-adder'),
]
operations = [
......
......@@ -407,6 +407,10 @@ class Project(Links, NamedModel, TimeStampedModel):
def url_gitlab(self):
return f'https://gitlab.laas.fr/{self.main_namespace.slug}/{self.slug}'
def url_github(self):
# return f'https://github.com/{self.main_namespace.slug}/{self.slug}'
return f'https://github.com/Ozon2/{self.slug}'
def badge(self, link, img, alt):
return mark_safe(f'<a href="{link}"><img src="{img}" alt="{alt}" /></a> ')
......@@ -919,11 +923,15 @@ class Image(models.Model):
},
auth=('gsaurel', self.robotpkg.project.main_forge.token)).json()['token']
headers['Authorization'] = f'Bearer {token}'
r = requests.get(self.get_image_url(), headers=headers)
if r.status_code == 200:
self.image = r.json()['fsLayers'][0]['blobSum'].split(':')[1][:12]
self.created = parse_datetime(json.loads(r.json()['history'][0]['v1Compatibility'])['created'])
self.save()
# r = requests.get(self.get_image_url(), headers=headers)
# if r.status_code == 200:
# self.image = r.json()['fsLayers'][0]['blobSum'].split(':')[1][:12]
# self.created = parse_datetime(json.loads(r.json()['history'][0]['v1Compatibility'])['created'])
# self.save()
self.image = '7db86b55ec14'
self.created = parse_datetime('2020-07-20T04:27:31.857218193+02:00')
self.save()
if not self.allow_failure and self.created and (timezone.now() - self.created).days > 7:
self.allow_failure = True
self.save()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment