gitlab pipeline

Dockerfile

@@ -20,9 +20,7 @@ RUN apt-get update -qq && apt-get install -qqy \
     netcat-openbsd \
     make \
     msmtp \
- && curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \
- && echo "deb [arch=armhf] https://download.docker.com/linux/debian buster stable" >> /etc/apt/sources.list \
- && apt-get update -qq && apt-get install -qqy docker-ce
+ && curl -sSL https://get.docker.com/ | sh
 
 RUN pip3 install --no-cache-dir \
     gunicorn \
@@ -36,10 +34,8 @@ RUN pipenv install --system --deploy
 
 ADD . .
 
-CMD rm -f /opt/openrobots/etc/robotpkg.conf \
- && git clone git://git.openrobots.org/robots/robotpkg /srv/dashboard/robotpkg || true \
- && git clone git://git.openrobots.org/robots/robotpkg/robotpkg-wip /srv/dashboard/robotpkg/wip || true \
- && /srv/dashboard/robotpkg/bootstrap/bootstrap \
+CMD mkdir -p /srv/dashboard/robotpkg \
+ && git clone https://github.com/Ozon2/example-adder.git /srv/dashboard/robotpkg/example-adder || true \
  && while ! nc -z postgres 5432; do sleep 1; done \
  && ./manage.py migrate \
  && ./manage.py collectstatic --no-input \

gh/views.py

 """Views for dashboard_apps."""
 import hmac
-import git
 from hashlib import sha1
 from ipaddress import ip_address, ip_network
 from json import loads
 from pprint import pprint
 
+import git
+import requests
+from autoslug.utils import slugify
 from django.conf import settings
 from django.http import HttpRequest
 from django.http.response import (HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseRedirect,
@@ -14,12 +16,8 @@ from django.shortcuts import get_object_or_404, reverse
 from django.utils.encoding import force_bytes
 from django.views.decorators.csrf import csrf_exempt
 
-import requests
-from autoslug.utils import slugify
-
 from dashboard.middleware import ip_laas
-from rainboard.models import Forge, Namespace, Project
-
+from rainboard.models import Namespace, Project
 from . import models
 
 
@@ -37,7 +35,8 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
     branch = f'pr/{data["number"]}'
     print(f'Pull request: {event}')
 
-    namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['login']))
+    # namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['login']))
+    namespace = get_object_or_404(Namespace, slug='tpillot')
     project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
     git_repo = project.git()
 
@@ -81,8 +80,8 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
         if branch in git_repo.branches:
             git_repo.delete_head(branch, force=True)
             git_repo.delete_remote(gh_remote_name)
-            project.gitlab().branches.delete(branch)
-            print(f'Deleted branch {branch}')
+        project.gitlab().branches.delete(branch)
+        print(f'Deleted branch {branch}')
 
     return HttpResponse(rep)
 
@@ -90,67 +89,81 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
 def push(request: HttpRequest, rep: str) -> HttpResponse:
     """Someone pushed on github. Synchronise local repo & gitlab."""
     data = loads(request.body.decode())
-    namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['name']))
+    # namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['name']))
+    namespace = get_object_or_404(Namespace, slug='tpillot')
     project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
-    branch_name = data['ref'][11:]  # strip 'refs/heads/'
-    print(f'Push detected on github: {branch_name}')
+    branch = data['ref'][11:]  # strip 'refs/heads/'
     gh_remote_name = f'github/{namespace.slug}'
     gl_remote_name = f'gitlab/{namespace.slug}'
-    gh_branch_name = f'{gh_remote_name}/{branch_name}'
-    gl_branch_name = f'{gl_remote_name}/{branch_name}'
-
     git_repo = project.git()
-    gh_remote = git_repo.remote(gh_remote_name)
+    print(f'Push detected on github: {branch}')
+
+    # TODO: hard reset if necessary ?
+    if gh_remote_name in git_repo.remotes:
+        gh_remote = git_repo.remote(gh_remote_name)
+    else:
+        gh_remote = git_repo.create_remote(gh_remote_name, url=project.url_github())
     gh_remote.fetch()
 
     # The branch was deleted, delete the branch on Gitlab
     if data['after'] == "0000000000000000000000000000000000000000":
         print("Branch deleted")
-        for branch in [gh_branch_name, gl_branch_name, branch_name]:
-            if branch in git_repo.branches:
-                git_repo.delete_head(branch, force=True)
-        project.gitlab().branches.delete(branch_name)
+        if branch in git_repo.branches:
+            git_repo.delete_head(branch, force=True)
+        project.gitlab().branches.delete(branch)
         return HttpResponse(rep)
 
-    # TODO: Is that necessary ?
-    gh_ref = gh_remote.refs[branch_name]
+    gh_ref = gh_remote.refs[branch]
     if str(gh_ref.commit) != data['after']:
-        fail = f'push: wrong commit: {gh_ref.commit} vs {data["after"]}'
+        fail = f'Push: wrong commit: {gh_ref.commit} vs {data["after"]}'
         print(fail)
         return HttpResponseBadRequest(fail)
 
-    # TODO: Is that necessary ?
-    if gl_remote_name not in git_repo.remotes:
-        print(f'project {project} not available on {gl_remote_name}')
-        return HttpResponse(rep)
+    # Update branch to the last commit
+    if branch in git_repo.branches:
+        git_repo.branches[branch].commit = data['after']
+    else:
+        git_repo.create_head(branch, commit=data['after'])
 
-    # Update branches to the last commit
-    for branch in [gh_branch_name, gl_branch_name, branch_name]:
-        if branch in git_repo.branches:
-            git_repo.branches[branch].commit = data['after']
-        else:
-            git_repo.create_head(branch, commit=data['after'])
+    if gl_remote_name in git_repo.remotes:
+        gl_remote = git_repo.remote(gl_remote_name)
+    else:
+        gl_remote = git_repo.create_remote(gl_remote_name, url=project.url_gitlab())
 
     # Push the changes to gitlab
-    # TODO: force push
-    gl_remote = git_repo.remote(gl_remote_name)
+    # TODO: Force push
     gl_remote.fetch()
-    if branch_name not in gl_remote.refs or str(gl_remote.refs[branch_name].commit) != data['after']:
-        print(f'pushing {data["after"]} on {branch_name} on gitlab')
-        gl_remote.push(branch_name)
+    if branch not in gl_remote.refs or str(gl_remote.refs[branch].commit) != data['after']:
+        print(f'pushing {data["after"]} on {branch} on gitlab')
+        gl_remote.push(branch)
 
     return HttpResponse(rep)
 
 
 def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
     """Something happened on a Gitlab pipeline. Tell Github if necessary."""
-    print('pipeline')
     data = loads(request.body.decode())
-    namespace = get_object_or_404(Namespace, slug=slugify(data['project']['namespace']))
+    branch, commit, gl_status, pipeline_id = (data['object_attributes'][key] for key in ['ref', 'sha', 'status', 'id'])
+    print(f'Pipeline id #{pipeline_id} on commit {commit} for branch {branch}, status: {gl_status}')
+
+    # namespace = get_object_or_404(Namespace, slug=slugify(data['project']['namespace']))
+    namespace = get_object_or_404(Namespace, slug='tpillot')
     project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['project']['name']))
-    branch, commit, status = (data['object_attributes'][key] for key in ['ref', 'sha', 'status'])
-    # status in ['pending', 'running', 'success', 'failed']
-    print(namespace, project, branch, commit, status)
+    gh_repo = project.github()
+    ci_web_url = project.url_gitlab() + '/pipelines/' + str(pipeline_id)
+
+    # Report the status to Github
+    if gl_status in ['pending', 'success', 'failed']:
+        gh_status = gl_status if gl_status != 'failed' else 'failure'
+        if branch.startswith('pr/'):
+            pr_number = int(branch[3:])
+            pr_commit_sha = gh_repo.get_pull(pr_number).head.sha
+            gh_repo.get_commit(sha=pr_commit_sha).create_status(state=gh_status, target_url=ci_web_url,
+                                                                context='gitlab-ci')
+        else:
+            gh_commit = gh_repo.get_branch(branch).commit
+            gh_commit.create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
+
     return HttpResponse(rep)
 
 
@@ -162,7 +175,7 @@ def webhook(request: HttpRequest) -> HttpResponse:
     thx https://simpleisbetterthancomplex.com/tutorial/2016/10/31/how-to-handle-github-webhooks-using-django.html
     """
     # validate ip source
-    forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR')
+    forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR').split(', ')[0]
     networks = requests.get('https://api.github.com/meta').json()['hooks']
     if not any(ip_address(forwarded_for) in ip_network(net) for net in networks):
         print('not from github IP')
@@ -200,6 +213,8 @@ def webhook(request: HttpRequest) -> HttpResponse:
 
 @csrf_exempt
 def gl_webhook(request: HttpRequest) -> HttpResponse:
+    """Process request incoming from a gitlab webhook."""
+
     # validate ip source
     if not ip_laas(request):
         print('not from LAAS IP')

rainboard/migrations/0045_project_accept_pr_to_master.py

+# Generated by Django 3.0.8 on 2020-07-24 13:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rainboard', '0044_focal'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='accept_pr_to_master',
+            field=models.BooleanField(default=False),
+        ),
+    ]

rainboard/migrations/0045_example-adder.py → rainboard/migrations/0046_example-adder.py

@@ -3,6 +3,11 @@
 from django.db import migrations
 
 
+def add_namespace(apps, schema_editor):
+    Namespace = apps.get_model('rainboard', 'Namespace')
+    Namespace.objects.create(name='tpillot', group=False)
+
+
 def add_project(apps, schema_editor):
     Project = apps.get_model('rainboard', 'Project')
     Namespace = apps.get_model('rainboard', 'Namespace')
@@ -11,28 +16,7 @@ def add_project(apps, schema_editor):
     git_project = Project(
         name='example-adder',
         public=True,
-        main_namespace=Namespace.objects.get(name='Gepetto'),
-        main_forge=Forge.objects.get(name='Gitlab'),
-        license=None,
-        homepage=None,
-        description=None,
-        version=None,
-        updated=None,
-        tests=True,
-        docs=True,
-        from_gepetto=True,
-        cmake_name=None,
-        archived=False,
-        suffix='',
-        allow_format_failure=True,
-        has_python=True
-    )
-    git_project.save()
-
-    git_project = Project(
-        name='talos-data',
-        public=True,
-        main_namespace=Namespace.objects.get(name='Gepetto'),
+        main_namespace=Namespace.objects.get(name='tpillot'),
         main_forge=Forge.objects.get(name='Gitlab'),
         license=None,
         homepage=None,
@@ -53,9 +37,10 @@ def add_project(apps, schema_editor):
 
 class Migration(migrations.Migration):
     dependencies = [
-        ('rainboard', '0044_focal'),
+        ('rainboard', '0045_project_accept_pr_to_master'),
     ]
 
     operations = [
+        migrations.RunPython(add_namespace),
         migrations.RunPython(add_project),
     ]

rainboard/migrations/0046_robotpkg.py → rainboard/migrations/0047_robotpkg.py

@@ -7,22 +7,22 @@ path = settings.RAINBOARD_RPKG
 
 
 def add_robotpkg(apps, schema_editor):
-    for project in [Project.objects.get(slug='example-adder'), Project.objects.get(slug='talos-data')]:
+    project = Project.objects.get(slug='example-adder')
 
-        for slug in [project.slug, project.slug.replace('_', '-')]:
-            for pkg in itertools.chain(path.glob(f'*/{slug}{project.suffix}'), path.glob(f'*/py-{slug}{project.suffix}')):
-                obj, created = Robotpkg.objects.get_or_create(name=pkg.name, category=pkg.parent.name, project=project)
-                if created:
-                    print(project, pkg)
-                    obj.update(pull=False)
+    for slug in [project.slug, project.slug.replace('_', '-')]:
+        for pkg in itertools.chain(path.glob(f'*/{slug}{project.suffix}'), path.glob(f'*/py-{slug}{project.suffix}')):
+            obj, created = Robotpkg.objects.get_or_create(name=pkg.name, category=pkg.parent.name, project=project)
+            if created:
+                print(project, pkg)
+                obj.update(pull=False)
 
-        for rpkg in project.robotpkg_set.all():
-            rpkg.update_images()
+    for rpkg in project.robotpkg_set.all():
+        rpkg.update_images()
 
 
 class Migration(migrations.Migration):
     dependencies = [
-        ('rainboard', '0045_example-adder'),
+        ('rainboard', '0046_example-adder'),
     ]
 
     operations = [

rainboard/models.py

@@ -407,6 +407,10 @@ class Project(Links, NamedModel, TimeStampedModel):
     def url_gitlab(self):
         return f'https://gitlab.laas.fr/{self.main_namespace.slug}/{self.slug}'
 
+    def url_github(self):
+        # return f'https://github.com/{self.main_namespace.slug}/{self.slug}'
+        return f'https://github.com/Ozon2/{self.slug}'
+
     def badge(self, link, img, alt):
         return mark_safe(f'<a href="{link}"><img src="{img}" alt="{alt}" /></a> ')
 
@@ -919,11 +923,15 @@ class Image(models.Model):
             },
                                  auth=('gsaurel', self.robotpkg.project.main_forge.token)).json()['token']
             headers['Authorization'] = f'Bearer {token}'
-        r = requests.get(self.get_image_url(), headers=headers)
-        if r.status_code == 200:
-            self.image = r.json()['fsLayers'][0]['blobSum'].split(':')[1][:12]
-            self.created = parse_datetime(json.loads(r.json()['history'][0]['v1Compatibility'])['created'])
-            self.save()
+        # r = requests.get(self.get_image_url(), headers=headers)
+        # if r.status_code == 200:
+        #     self.image = r.json()['fsLayers'][0]['blobSum'].split(':')[1][:12]
+        #     self.created = parse_datetime(json.loads(r.json()['history'][0]['v1Compatibility'])['created'])
+        #     self.save()
+        self.image = '7db86b55ec14'
+        self.created = parse_datetime('2020-07-20T04:27:31.857218193+02:00')
+        self.save()
+
         if not self.allow_failure and self.created and (timezone.now() - self.created).days > 7:
             self.allow_failure = True
             self.save()