Merge remote-tracking branch 'dashboard/master' into dashboard/

dashboard/README.md

+# Gepetto Packages Dashboard
+
+## APIs
+
+- github: https://api.github.com/orgs/stack-of-tasks/repos
+- gitlab: https://eur0c.laas.fr/api/v4/projects
+- redmine: https://redmine.laas.fr/projects.json
+
+## RUN
+
+```
+rm db.sqlite3 ; and ./manage.py migrate; and ./manage.py runserver localhost:8001
+```

dashboard/gepetto_packages/admin.py

+from django.contrib import admin
+
+# Register your models here.

dashboard/gepetto_packages/apps.py

+from django.apps import AppConfig
+
+
+class GepettoPackagesConfig(AppConfig):
+    name = 'gepetto_packages'

dashboard/gepetto_packages/migrations/0001_initial.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.7 on 2017-11-15 17:31
+from __future__ import unicode_literals
+
+import autoslug.fields
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='License',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, unique=True)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)),
+                ('github_key', models.CharField(max_length=50)),
+                ('spdx_id', models.CharField(max_length=50)),
+                ('url', models.URLField()),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Package',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('name', models.CharField(max_length=200, unique=True)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)),
+                ('homepage', models.URLField(blank=True, null=True)),
+                ('license', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='gepetto_packages.License')),
+            ],
+            options={
+                'ordering': ('name',),
+            },
+        ),
+        migrations.CreateModel(
+            name='Project',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=200, unique=True)),
+                ('slug', autoslug.fields.AutoSlugField(editable=False, populate_from='name', unique=True)),
+            ],
+            options={
+                'abstract': False,
+            },
+        ),
+        migrations.CreateModel(
+            name='Repo',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', models.DateTimeField(auto_now_add=True)),
+                ('updated', models.DateTimeField(auto_now=True)),
+                ('url', models.URLField(unique=True)),
+                ('homepage', models.URLField(blank=True, null=True)),
+                ('default_branch', models.CharField(max_length=50)),
+                ('open_issues', models.PositiveSmallIntegerField(blank=True, null=True)),
+                ('open_pr', models.PositiveSmallIntegerField(blank=True, null=True)),
+                ('repo_id', models.PositiveIntegerField()),
+                ('license', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='gepetto_packages.License')),
+                ('package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gepetto_packages.Package')),
+            ],
+            options={
+                'ordering': ('package', 'url'),
+            },
+        ),
+        migrations.AddField(
+            model_name='package',
+            name='project',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='gepetto_packages.Project'),
+        ),
+    ]

dashboard/gepetto_packages/migrations/0002_github.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.7 on 2017-11-09 15:55
+from __future__ import unicode_literals
+
+from os.path import expanduser
+
+from django.db import migrations
+
+import requests
+
+GITHUB_API = 'https://api.github.com'
+with open(expanduser('~/.githubtoken')) as f:
+    TOKEN = f.read().strip()
+HEADERS = {
+    'Authorization': f'token {TOKEN}',
+    'Accept': 'application/vnd.github.drax-preview+json',
+}
+
+PROJECTS = (
+    'Humanoid Path Planner',
+    'Stack of Tasks'
+)
+
+def github_licenses(apps, schema_editor):
+    License = apps.get_model('gepetto_packages', 'License')
+    for data in requests.get(f'{GITHUB_API}/licenses', headers=HEADERS).json():
+        License.objects.create(github_key=data['key'], **{key: data[key] for key in ['name', 'spdx_id', 'url']})
+
+
+def github_projects(apps, schema_editor):
+    Project, License, Package, Repo = (apps.get_model('gepetto_packages', model)
+                                       for model in ['Project', 'License', 'Package', 'Repo'])
+    for project_name in PROJECTS:
+        project = Project(name=project_name)
+        project.save()
+        for data in requests.get(f'{GITHUB_API}/orgs/{project.slug}/repos', headers=HEADERS).json():
+            package = Package(name=data['name'], project=project, homepage=data['homepage'])
+            package.save()
+            repo = Repo(package=package, url=data['html_url'], homepage=data['homepage'], repo_id=data['id'],
+                        default_branch=data['default_branch'], open_issues=data['open_issues'])
+            repo_api_url = f'{GITHUB_API}/repos/{project.slug}/{package.slug}'
+            repo_data = requests.get(repo_api_url, headers=HEADERS).json()
+            if 'license' in repo_data and repo_data['license']:
+                license_data = repo_data['license']
+                license, _ = License.objects.get_or_create(github_key=license_data['key'], name=license_data['name'])
+                repo.license = license
+                package.license = license
+                package.save()
+            repo.open_pr = len(requests.get(f'{repo_api_url}/pulls', headers=HEADERS).json())
+            repo.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gepetto_packages', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RunPython(github_licenses),
+        migrations.RunPython(github_projects),
+    ]

dashboard/gepetto_packages/migrations/0003_gitlab.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.7 on 2017-11-13 16:13
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+import requests
+
+GITLAB_API = 'https://eur0c.laas.fr/api/v4'
+
+def gitlab(apps, schema_editor):
+    Project, License, Package, Repo = (apps.get_model('gepetto_packages', model)
+                                       for model in ['Project', 'License', 'Package', 'Repo'])
+    for data in requests.get(f'{GITLAB_API}/projects', verify=False).json():
+        package_qs = Package.objects.filter(name=data['name'])
+        if package_qs.exists():
+            Repo.objects.create(package=package_qs.first(), url=data['web_url'], repo_id=data['id'])
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gepetto_packages', '0002_github'),
+    ]
+
+    operations = [
+        migrations.RunPython(gitlab),
+    ]

dashboard/gepetto_packages/migrations/0004_redmine.py

+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.7 on 2017-11-14 09:29
+from __future__ import unicode_literals
+
+from os.path import expanduser
+
+from django.db import migrations
+
+import requests
+
+REDMINE_APIS = [
+    'https://redmine.laas.fr',
+    # 'https://git.openrobots.org',
+]
+with open(expanduser('~/.redminetoken')) as f:
+    TOKEN = f.read().strip()
+HEADERS = {
+    'X-Redmine-API-Key': TOKEN,
+}
+PACKAGES = [
+    'openhrp3-hrp2',
+    'Pyrene Talos',
+]
+
+def redmine(apps, schema_editor):
+    Project, License, Package, Repo = (apps.get_model('gepetto_packages', model)
+                                       for model in ['Project', 'License', 'Package', 'Repo'])
+    dummy_project, _ = Project.objects.get_or_create(name='dummy')
+    for api in REDMINE_APIS:
+        for data in requests.get(f'{api}/projects.json?limit=100', headers=HEADERS).json()['projects']:
+            if data['name'] in PACKAGES:
+                Package.objects.get_or_create(name=data['identifier'], project=dummy_project)
+            package_qs = Package.objects.filter(name=data['identifier'])
+            if package_qs.exists():
+                r = Repo(package=package_qs.first(), repo_id=data['id'], open_pr=0)
+                repo_data = requests.get(f'{api}/projects/{r.repo_id}.json', headers=HEADERS).json()['project']
+                r.homepage = repo_data['homepage']
+                r.url = f'{api}/projects/{r.package.name}'
+                issues_data = requests.get(f'{api}/issues.json?project_id={r.repo_id}&status_id=open', headers=HEADERS)
+                r.open_issues = issues_data.json()['total_count']
+                r.save()
+                if r.homepage and not r.package.homepage:
+                    r.package.homepage = r.homepage
+                    r.package.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('gepetto_packages', '0003_gitlab'),
+    ]
+
+    operations = [
+        migrations.RunPython(redmine),
+    ]

dashboard/gepetto_packages/models.py

+from django.db import models
+
+from ndh.models import TimeStampedModel, NamedModel
+from ndh.utils import query_sum
+
+
+class Project(NamedModel):
+    pass
+
+
+class License(NamedModel):
+    github_key = models.CharField(max_length=50)
+    spdx_id = models.CharField(max_length=50)
+    url = models.URLField(max_length=200)
+
+    def __str__(self):
+        return self.spdx_id or self.name
+
+
+class Package(NamedModel, TimeStampedModel):
+    project = models.ForeignKey(Project)
+    homepage = models.URLField(max_length=200, blank=True, null=True)
+    license = models.ForeignKey(License, blank=True, null=True)
+
+    class Meta:
+        ordering = ('name',)
+
+    def open_pr(self):
+        return query_sum(self.repo_set, 'open_pr')
+
+    def open_issues(self):
+        return query_sum(self.repo_set, 'open_issues')
+
+
+class Repo(TimeStampedModel):
+    package = models.ForeignKey(Package)
+    url = models.URLField(max_length=200, unique=True)
+    homepage = models.URLField(max_length=200, blank=True, null=True)
+    license = models.ForeignKey(License, blank=True, null=True)
+    default_branch = models.CharField(max_length=50)
+    open_issues = models.PositiveSmallIntegerField(blank=True, null=True)
+    open_pr = models.PositiveSmallIntegerField(blank=True, null=True)
+    repo_id = models.PositiveIntegerField()
+
+    class Meta:
+        ordering = ('package', 'url')
+
+    def __str__(self):
+        return self.url

dashboard/gepetto_packages/templates/base.html

+{% load static bootstrap3 %}<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta http-equiv="X-UA-Compatible" content="IE=edge">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Gepetto Packages</title>
+    <link rel="stylesheet" href="{% bootstrap_css_url %}" crossorigin="anonymous" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" >
+
+    <!--[if lt IE 9]>
+      <script src="https://oss.maxcdn.com/html5shiv/3.7.3/html5shiv.min.js"></script>
+      <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+    <![endif]-->
+    <style>body { padding-top: 70px; }</style>
+    {% block style %}{% endblock %}
+  </head>
+  <body>
+    <nav class="navbar navbar-inverse navbar-fixed-top">
+      <div class="container-fluid">
+
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar-collapse" aria-expanded="false">
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <a class="navbar-brand" href="/">Gepetto Packages</a>
+        </div>
+      </div>
+    </nav>
+    <div class="container">
+      {% bootstrap_messages %}
+      {% block content %}
+      <h1>Hello, world!</h1>
+      {% endblock %}
+    </div>
+    <hr>
+    <script src="{% bootstrap_jquery_url %}"></script>
+    <script src="{% bootstrap_javascript_url %}" crossorigin="anonymous" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" ></script>
+    <script>{% block scripts %}{% endblock %}</script>
+  </body>
+</html>

dashboard/gepetto_packages/templates/gepetto_packages/package.html

+{% load gepetto_packages %}
+<div class="package">
+  <div class="package-name">{{ package }}</div>
+  <div class=""></div>
+  <div class="package-homepage">{{ package.homepage|domain }}</div>
+  <div class="package-issues">{{ package.open_issues }}</div>
+  <div class="package-pr">{{ package.open_pr }}</div>
+  <div class="package-license">{{ package.license|default_if_none:'?' }}</div>
+  {% for repo in package.repo_set.all %}
+  {% include 'gepetto_packages/repo.html' with repo=repo %}
+  {% endfor %}
+</div>

dashboard/gepetto_packages/templates/gepetto_packages/package_list.html

+{% extends 'base.html' %}
+
+{% block style %}
+<style>
+.package {
+  display: grid;
+  width: 100%;
+  grid-template-columns: 1fr 1fr 1fr 40px 40px 100px;
+  border-top: 1px solid;
+}
+</style>
+{% endblock %}
+
+{% block content %}
+<div class="package">
+  <div class="package-name">Name</div>
+  <div class="">Repo</div>
+  <div class="package-homepage">Homepage</div>
+  <div class="">Issues</div>
+  <div class="">PR</div>
+  <div class="package-license">License</div>
+</div>
+{% for package in package_list %}
+{% include 'gepetto_packages/package.html' with package=package %}
+{% endfor %}
+{% endblock %}

dashboard/gepetto_packages/templates/gepetto_packages/repo.html

+{% load gepetto_packages %}
+<div></div>
+<div class="repo-url">{{ repo.url|domain }}</div>
+<div class="repo-homepage">{{ repo.homepage|domain }}</div>
+<div class="repo-issues">{{ repo.open_issues|default_if_none:'?' }}</div>
+<div class="repo-pr">{{ repo.open_pr|default_if_none:'?' }}</div>
+<div class="repo-license">{{ repo.license|default_if_none:'?' }}</div>

dashboard/gepetto_packages/templatetags/gepetto_packages.py

+from django import template
+from django.utils.safestring import mark_safe
+
+register = template.Library()
+
+@register.filter
+def domain(url):
+    if url:
+        domain_name = '.'.join(url.split('/')[2].split('.')[-3:])
+        return mark_safe(f'<a href="{url}">{domain_name}</a>')
+    return '?'

dashboard/gepetto_packages/tests.py

+from django.test import TestCase
+
+# Create your tests here.

dashboard/gepetto_packages/urls.py

+from django.conf.urls import url
+from django.contrib import admin
+from django.views.generic import ListView
+
+from .models import Package
+
+urlpatterns = [
+    url(r'^admin/', admin.site.urls),
+    url(r'', ListView.as_view(model=Package)),
+]

dashboard/gepetto_packages/views.py

+from django.shortcuts import render
+
+# Create your views here.