Sync GitHub and GitLab projects

.gitignore

 .env
 db.sqlite3
+__pycache__/
+.idea
\ No newline at end of file

.gitlab-ci.yml

@@ -21,6 +21,6 @@ build:
   image: docker
   script:
     - docker build -t ${DOCKER_TAG} .
-    - docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py migrate
-    - docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py test
+    - docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e GITHUB_PIPELINE_TOKEN -e GITLAB_PIPELINE_TOKEN -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py migrate
+    - docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e GITHUB_PIPELINE_TOKEN -e GITLAB_PIPELINE_TOKEN -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py test
     - docker push ${DOCKER_TAG}

Pipfile

@@ -16,3 +16,5 @@ requests = "*"
 GitPython = "*"
 djangorestframework = "<3.12"
 django-auth-ldap = "*"
+python-gitlab = "*"
+pygithub = "*"

dashboard/middleware.py

@@ -8,7 +8,7 @@ from rest_framework import permissions
 
 def ip_laas(request: HttpRequest) -> bool:
     """check if request comes from settings.LAAS_NETWORKS."""
-    forwarded_for = ip_address(request.META.get('HTTP_X_FORWARDED_FOR'))
+    forwarded_for = ip_address(request.META.get('HTTP_X_FORWARDED_FOR').split(', ')[0])
     return any(forwarded_for in ip_network(net) for net in settings.LAAS_NETWORKS)
 
 

dashboard/settings.py

@@ -28,6 +28,7 @@ SERVER_EMAIL = f'{EMAIL_USER}+{PROJECT}@{EMAIL_FQDN}'
 DEFAULT_FROM_EMAIL = f'{PROJECT_VERBOSE} <{EMAIL_USER}@{EMAIL_FQDN}>'
 EMAIL_BACKEND = 'django.core.mail.backends.%s' % ('filebased.EmailBackend' if DEBUG else 'smtp.EmailBackend')
 EMAIL_SUBJECT_PREFIX = f'[{PROJECT_VERBOSE}] '
+EMAIL_FILE_PATH = 'email/'
 
 ADMINS = ((os.environ.get('ADMIN_NAME',
                           f'{PROJECT_VERBOSE} webmaster'), os.environ.get('ADMIN_MAIL', f'webmaster@{DOMAIN_NAME}')), )
@@ -141,12 +142,19 @@ LOGGING = {
             'level': 'DEBUG',
             'class': 'logging.NullHandler',
         },
+        'console': {
+            'class': 'logging.StreamHandler',
+        },
     },
     'loggers': {
         'django.security.DisallowedHost': {
             'handlers': ['null'],
             'propagate': False,
         },
+        'gh': {
+            'handlers': ['console'],
+            'level': 'DEBUG',
+        },
     },
 }
 

gh/urls.py

@@ -5,6 +5,6 @@ from django.urls import path
 from . import views
 
 urlpatterns = [
-    path('webhook', views.webhook),
-    path('gl-webhook', views.gl_webhook),
+    path('webhook', views.webhook, name='webhook'),
+    path('gl-webhook', views.gl_webhook, name='gl-webhook'),
 ]

rainboard/migrations/0002_forges_groups.py

@@ -2,18 +2,23 @@
 
 import os
 
-import requests
-
 from django.db import migrations
+
 from rainboard.utils import SOURCES
 
 
 def forges(apps, schema_editor):
     Forge = apps.get_model('rainboard', 'Forge')
+    gitlab_token = os.getenv('GITLAB_TOKEN')
+    if gitlab_token is None:
+        gitlab_token = os.getenv('GITLAB_PIPELINE_TOKEN')
     Forge.objects.create(name='Gitlab', source=SOURCES.gitlab, url='https://gepgitlab.laas.fr',
-                         token=os.getenv('GITLAB_TOKEN'))
+                         token=gitlab_token)
+    github_token = os.getenv('GITHUB_TOKEN')
+    if github_token is None:
+        github_token = os.getenv('GITHUB_PIPELINE_TOKEN')
     Forge.objects.create(name='Github', source=SOURCES.github, url='https://github.com',
-                         token=os.getenv('GITHUB_TOKEN'))
+                         token=github_token)
     Forge.objects.create(name='Redmine', source=SOURCES.redmine, url='https://redmine.laas.fr',
                          token=os.getenv('REDMINE_TOKEN'))
     Forge.objects.create(name='Openrobots', source=SOURCES.redmine, url='https://git.openrobots.org',

rainboard/migrations/0045_project_accept_pr_to_master.py

+# Generated by Django 3.0.8 on 2020-07-24 13:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rainboard', '0044_focal'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='accept_pr_to_master',
+            field=models.BooleanField(default=False),
+        ),
+    ]

rainboard/migrations/0046_namespace_slug_gitlab_github.py

+# Generated by Django 3.0.8 on 2020-07-29 08:29
+
+from django.db import migrations, models
+
+
+def update_namespaces(apps, schema_editor):
+    Namespace = apps.get_model('rainboard', 'Namespace')
+    for namespace in Namespace.objects.all():
+        if namespace.slug_gitlab == '':
+            namespace.slug_gitlab = namespace.slug
+        if namespace.slug_github == '':
+            namespace.slug_github = namespace.slug
+        namespace.save()
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('rainboard', '0045_project_accept_pr_to_master'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='namespace',
+            name='slug_github',
+            field=models.CharField(default='', max_length=200),
+        ),
+        migrations.AddField(
+            model_name='namespace',
+            name='slug_gitlab',
+            field=models.CharField(default='', max_length=200),
+        ),
+        migrations.RunPython(update_namespaces),
+    ]

rainboard/models.py

@@ -16,6 +16,8 @@ from django.utils.safestring import mark_safe
 
 from autoslug import AutoSlugField
 from autoslug.utils import slugify
+from github import Github
+from gitlab import Gitlab
 from ndh.models import Links, NamedModel, TimeStampedModel
 from ndh.utils import enum_to_choices, query_sum
 
@@ -50,6 +52,15 @@ GITLAB_STATUS = {'failed': False, 'success': True, 'pending': None, 'skipped': N
 
 class Namespace(NamedModel):
     group = models.BooleanField(default=False)
+    slug_gitlab = models.CharField(max_length=200, default='')
+    slug_github = models.CharField(max_length=200, default='')
+
+    def save(self, *args, **kwargs):
+        if self.slug_gitlab == '':
+            self.slug_gitlab = self.slug
+        if self.slug_github == '':
+            self.slug_github = self.slug
+        super(Namespace, self).save(*args, **kwargs)
 
 
 class License(models.Model):
@@ -194,6 +205,7 @@ class Project(Links, NamedModel, TimeStampedModel):
     suffix = models.CharField(max_length=50, default='', blank=True)
     allow_format_failure = models.BooleanField(default=True)
     has_python = models.BooleanField(default=True)
+    accept_pr_to_master = models.BooleanField(default=False)
 
     def save(self, *args, **kwargs):
         self.name = valid_name(self.name)
@@ -209,6 +221,17 @@ class Project(Links, NamedModel, TimeStampedModel):
             return git.Repo.init(path)
         return git.Repo(str(path / '.git'))
 
+    def github(self):
+        github_forge = Forge.objects.get(slug='github')
+        gh = Github(github_forge.token)
+        return gh.get_repo(f'{self.main_namespace.slug_github}/{self.slug}')
+
+    def gitlab(self):
+        gitlab_forge = Forge.objects.get(slug='gitlab')
+        gl = Gitlab(gitlab_forge.url, private_token=gitlab_forge.token)
+        gl_repo = gl.projects.get(f'{self.main_namespace.slug_gitlab}/{self.slug}')
+        return gl_repo
+
     def main_repo(self):
         forge = self.main_forge if self.main_forge else get_default_forge(self)
         repo, created = Repo.objects.get_or_create(forge=forge,
@@ -389,7 +412,18 @@ class Project(Links, NamedModel, TimeStampedModel):
         return f'https://travis-ci.org/{self.main_namespace.slug}/{self.slug}'
 
     def url_gitlab(self):
-        return f'https://gitlab.laas.fr/{self.main_namespace.slug}/{self.slug}'
+        return f'https://gitlab.laas.fr/{self.main_namespace.slug_gitlab}/{self.slug}'
+
+    def remote_url_gitlab(self):
+        gitlab_forge = Forge.objects.get(source=SOURCES.gitlab)
+        return self.url_gitlab().replace('://', f'://gitlab-ci-token:{gitlab_forge.token}@')
+
+    def url_github(self):
+        return f'https://github.com/{self.main_namespace.slug_github}/{self.slug}'
+
+    def remote_url_github(self):
+        github_forge = Forge.objects.get(source=SOURCES.github)
+        return self.url_github().replace('://', f'://{settings.GITHUB_USER}:{github_forge.token}@')
 
     def badge(self, link, img, alt):
         return mark_safe(f'<a href="{link}"><img src="{img}" alt="{alt}" /></a> ')

rainboard/tests.py

@@ -3,6 +3,7 @@ import doctest
 from django.test import TestCase
 from django.urls import reverse
 
+from dashboard import settings
 from . import models, utils
 
 
@@ -13,24 +14,24 @@ class RainboardTests(TestCase):
         self.assertEqual(test_count, 5)
 
     def test_models(self):
-        self.assertEqual(models.License.objects.count(), 0)
-        self.assertEqual(models.Project.objects.count(), 0)
+        license_count = models.License.objects.count()
+        project_count = models.Project.objects.count()
         models.License.objects.create(name='BSD 2-Clause "Simplified" License',
                                       spdx_id='BSD-2-Clause',
                                       url='http://spdx.org/licenses/BSD-2-Clause.json')
-        models.Project.objects.create(name='Rainboard Tests',
+        models.Project.objects.create(name='Rainboard Tests 2',
                                       main_namespace=models.Namespace.objects.get(slug='gepetto'),
                                       main_forge=models.Forge.objects.get(source=utils.SOURCES.github),
                                       license=models.License.objects.first())
-        self.assertEqual(models.License.objects.count(), 1)
-        self.assertEqual(models.Project.objects.count(), 1)
+        self.assertEqual(models.License.objects.count(), license_count + 1)
+        self.assertEqual(models.Project.objects.count(), project_count + 1)
 
-        project = models.Project.objects.first()
+        project = models.Project.objects.get(name='rainboard tests 2')
 
-        self.assertEqual(project.slug, 'rainboard-tests')
+        self.assertEqual(project.slug, 'rainboard-tests-2')
         self.assertEqual(project.registry(), 'memmos.laas.fr:5000')
-        self.assertEqual(project.url_travis(), 'https://travis-ci.org/gepetto/rainboard-tests')
-        self.assertEqual(project.url_gitlab(), 'https://gitlab.laas.fr/gepetto/rainboard-tests')
+        self.assertEqual(project.url_travis(), 'https://travis-ci.org/gepetto/rainboard-tests-2')
+        self.assertEqual(project.url_gitlab(), 'https://gitlab.laas.fr/gepetto/rainboard-tests-2')
         badges = project.badges()
         for chunk in ['<img src="https://gitlab.laas', 'travis-ci', 'href="https://gepettoweb.laas']:
             self.assertIn(chunk, badges)
@@ -47,7 +48,7 @@ class RainboardTests(TestCase):
         content = response.content.decode()
         for chunk in [
                 '<title>Gepetto Packages</title>',
-                '<h1>rainboard tests</h1>',
+                '<h1>rainboard tests 2</h1>',
                 'Main forge</dt> <dd class="col-9"><a href="https://github.com">Github</a></dd>',
                 '<label class="label label-primary">BSD-2-Clause</label>',
         ]: