initial

dashboard/settings.py

+"""
+Django settings for dashboard project.
+
+Generated by 'django-admin startproject' using Django 2.0.1.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/2.0/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/2.0/ref/settings/
+"""
+
+import os
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/2.0/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'mgun706*g8lq6voa#1n*)cfk7t4m%8s@4my^al#@8tzis+#v5e'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'django.contrib.sites',
+    'rainboard',
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'dashboard.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'dashboard.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/2.0/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/2.0/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/2.0/howto/static-files/
+
+STATIC_URL = '/static/'
+SITE_ID = 1

dashboard/urls.py

+"""dashboard URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/2.0/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  path('', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  path('', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.urls import include, path
+    2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
+"""
+from django.contrib import admin
+from django.urls import path
+
+urlpatterns = [
+    path('admin/', admin.site.urls),
+]

dashboard/wsgi.py

+"""
+WSGI config for dashboard project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/2.0/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings")
+
+application = get_wsgi_application()

manage.py

+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dashboard.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)

rainboard/admin.py

+from django.contrib import admin
+
+# Register your models here.

rainboard/apps.py

+from django.apps import AppConfig
+
+
+class RainboardConfig(AppConfig):
+    name = 'rainboard'

rainboard/management/commands/populate.py

+from django.core.management.base import BaseCommand
+
+from rainboard.models import Forge, License
+
+import requests
+
+
+class Command(BaseCommand):
+    help = 'populates licenses, projets, namespaces and repos from forges'
+
+    def handle(self, *args, **options):
+        github = Forge.objects.get(name='Github')
+
+        for data in requests.get(f'{github.api_url()}/licenses', headers=github.headers()).json():
+            License.objects.create(github_key=data['key'], **{key: data[key] for key in ['name', 'spdx_id', 'url']})
+
+        for forge in Forge.objects.all():
+            forge.get_projects()

rainboard/migrations/0001_initial.py

+# Generated by Django 2.0.1 on 2018-01-05 16:21
+
+import autoslug.fields
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Branch',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('name', models.CharField(max_length=200, unique=True)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Commit',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('name', models.CharField(max_length=200, unique=True)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Forge',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, unique=True)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)),
+                ('source', models.PositiveSmallIntegerField(choices=[(1, 'github'), (2, 'gitlab'), (3, 'redmine'), (4, 'robotpkg')])),
+                ('url', models.URLField()),
+                ('token', models.CharField(blank=True, max_length=50, null=True)),
+                ('verify', models.BooleanField(default=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='License',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, unique=True)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)),
+                ('github_key', models.CharField(max_length=50)),
+                ('spdx_id', models.CharField(max_length=50)),
+                ('url', models.URLField()),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Namespace',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, unique=True)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Project',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('name', models.CharField(max_length=200, unique=True)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)),
+                ('private', models.BooleanField(default=False)),
+                ('homepage', models.URLField(blank=True, null=True)),
+                ('license', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='rainboard.License')),
+                ('main_namespace', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='rainboard.Namespace')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Repo',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('name', models.CharField(max_length=200)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name')),
+                ('homepage', models.URLField(blank=True, null=True)),
+                ('url', models.URLField(blank=True, null=True)),
+                ('default_branch', models.CharField(max_length=50)),
+                ('open_issues', models.PositiveSmallIntegerField(blank=True, null=True)),
+                ('open_pr', models.PositiveSmallIntegerField(blank=True, null=True)),
+                ('repo_id', models.PositiveIntegerField()),
+                ('forge', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Forge')),
+                ('license', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='rainboard.License')),
+                ('namespace', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Namespace')),
+                ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Project')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Robotpkg',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, unique=True)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)),
+                ('homepage', models.URLField(blank=True, null=True)),
+                ('license', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='rainboard.License')),
+                ('project', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Project')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='RobotpkgBuild',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('target', models.PositiveSmallIntegerField(choices=[(1, '12.04'), (2, '14.04'), (3, '16.04'), (4, 'dubnium')])),
+                ('passed', models.BooleanField(default=False)),
+                ('robotpkg', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Robotpkg')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='RobotpkgDependency',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, unique=True)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)),
+                ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Project')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='SystemDependency',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, unique=True)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)),
+                ('target', models.PositiveSmallIntegerField(choices=[(1, '12.04'), (2, '14.04'), (3, '16.04'), (4, 'dubnium')])),
+                ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Project')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Test',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('target', models.PositiveSmallIntegerField(choices=[(1, '12.04'), (2, '14.04'), (3, '16.04'), (4, 'dubnium')])),
+                ('passed', models.BooleanField(default=False)),
+                ('branch', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Branch')),
+                ('commit', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Commit')),
+                ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Project')),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.AddField(
+            model_name='commit',
+            name='project',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Project'),
+        ),
+        migrations.AddField(
+            model_name='branch',
+            name='commit',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Commit'),
+        ),
+        migrations.AddField(
+            model_name='branch',
+            name='repo',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Repo'),
+        ),
+    ]

rainboard/migrations/0002_forges.py

+# Generated by Django 2.0.1 on 2018-01-05 10:20
+
+import os
+
+from django.db import migrations
+
+import requests
+
+from rainboard.utils import SOURCES
+
+
+def forges(apps, schema_editor):
+    Forge = apps.get_model('rainboard', 'Forge')
+    Forge.objects.create(name='Gitlab', source=SOURCES.gitlab, url='https://eur0c.laas.fr', verify=False,
+                         token=os.getenv('GITLAB_TOKEN'))
+    Forge.objects.create(name='Github', source=SOURCES.github, url='https://github.com',
+                         token=os.getenv('GITHUB_TOKEN'))
+    Forge.objects.create(name='Redmine', source=SOURCES.redmine, url='https://redmine.laas.fr',
+                         token=os.getenv('REDMINE_TOKEN'))
+    # Forge.objects.create(name='Openrobots', source=SOURCES.redmine, url='https://git.openrobots.org',
+                         # token=os.getenv('OPENROB_TOKEN'))
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rainboard', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RunPython(forges),
+    ]

rainboard/models.py

+from django.db import models
+
+from autoslug import AutoSlugField
+from ndh.models import NamedModel, TimeStampedModel
+from ndh.utils import enum_to_choices, query_sum
+
+import requests
+
+from .utils import SOURCES, TARGETS
+
+
+class Namespace(NamedModel):
+    pass
+
+
+class License(NamedModel):
+    github_key = models.CharField(max_length=50)
+    spdx_id = models.CharField(max_length=50)
+    url = models.URLField(max_length=200)
+
+    def __str__(self):
+        return self.spdx_id or self.name
+
+
+class Project(NamedModel, TimeStampedModel):
+    private = models.BooleanField(default=False)
+    main_namespace = models.ForeignKey(Namespace, on_delete=models.SET_NULL, null=True, blank=True)
+    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
+    homepage = models.URLField(max_length=200, blank=True, null=True)
+
+
+class Forge(NamedModel):
+    source = models.PositiveSmallIntegerField(choices=enum_to_choices(SOURCES))
+    url = models.URLField(max_length=200)
+    token = models.CharField(max_length=50, blank=True, null=True)
+    verify = models.BooleanField(default=True)
+
+    def api_data(self, url=''):
+        return requests.get(self.api_url() + url, verify=self.verify, headers=self.headers()).json()
+
+    def headers(self):
+        if self.source == SOURCES.github:
+            return {'Authorization': f'token {self.token}', 'Accept': 'application/vnd.github.drax-preview+json'}
+        if self.source == SOURCES.gitlab:
+            return {'Private-Token': self.token}
+        if self.source == SOURCES.redmine:
+            return {'X-Redmine-API-Key': self.token}
+
+    def api_url(self):
+        if self.source == SOURCES.github:
+            return 'https://api.github.com'
+        if self.source == SOURCES.gitlab:
+            return f'{self.url}/api/v4'
+        return self.url
+
+    def get_projects(self):  # TODO auto
+        if self.source == SOURCES.github:
+            return self.get_projects_github()
+        if self.source == SOURCES.gitlab:
+            return self.get_projects_gitlab()
+        if self.source == SOURCES.redmine:
+            return self.get_projects_redmine()
+
+    def get_projects_github(self):
+        for namespace in Namespace.objects.all():
+            for data in self.api_data(f'/orgs/{namespace.slug}/repos'):
+                if not 'name' in data:
+                    continue
+                project, _ = Project.objects.get_or_create(name=data['name'],
+                                                           defaults={'homepage': data['homepage'],
+                                                                     'main_namespace': namespace})
+                repo, _ = Repo.objects.get_or_create(forge=self, namespace=namespace, project=project,
+                                                     defaults={'repo_id': data['id'], 'name': data['name']})
+                repo.homepage = data['homepage']
+                repo.url = data['html_url']
+                repo.repo_id = data['id']
+                repo.default_branch = data['default_branch']
+                repo.open_issues = data['open_issues']
+
+                repo_data = repo.api_data()
+                if 'license' in repo_data and repo_data['license']:
+                    license_data = repo_data['license']
+                    license, _ = License.objects.get_or_create(name=license_data['name'],
+                                                               defaults={'github_key': license_data['key']})
+                    repo.license = license
+                    if not project.license:
+                        project.license = license
+                repo.open_pr = len(repo.api_data('/pulls'))
+                repo.save()
+                project.save()
+
+    def get_projects_gitlab(self):
+        for data in self.api_data('/projects'):
+            project, created = Project.objects.get_or_create(name=data['name'])
+            namespace, _ = Namespace.objects.get_or_create(name=data['namespace']['name'])
+            if created and not 'forked_from_project' in data:
+                project.main_namespace = namespace
+
+            repo, _ = Repo.objects.get_or_create(forge=self, namespace=namespace, project=project,
+                                                 defaults={'repo_id': data['id'], 'name': data['name'],
+                                                           'url': data['web_url']})
+
+    def get_projects_redmine(self):
+        pass  # TODO
+
+
+class Repo(TimeStampedModel):
+    name = models.CharField(max_length=200)
+    slug = AutoSlugField(populate_from='name')
+    forge = models.ForeignKey(Forge, on_delete=models.CASCADE)
+    namespace = models.ForeignKey(Namespace, on_delete=models.CASCADE)
+    project = models.ForeignKey(Project, on_delete=models.CASCADE)
+    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
+    homepage = models.URLField(max_length=200, blank=True, null=True)
+    url = models.URLField(max_length=200, blank=True, null=True)
+    default_branch = models.CharField(max_length=50)
+    open_issues = models.PositiveSmallIntegerField(blank=True, null=True)
+    open_pr = models.PositiveSmallIntegerField(blank=True, null=True)
+    repo_id = models.PositiveIntegerField()
+
+    def api_url(self):
+        if self.forge.source == SOURCES.github:
+            return f'{self.forge.api_url()}/repos/{self.namespace.slug}/{self.slug}'
+        if self.forge.source == SOURCES.redmine:
+            return f'{self.forge.api_url()}/projects/{self.repo_id}.json'
+
+    def api_data(self, url=''):
+        return requests.get(self.api_url() + url, verify=self.forge.verify, headers=self.forge.headers()).json()
+
+
+class