New view showing issues and pull requests

Pipfile.lock

@@ -54,11 +54,11 @@
         },
         "django": {
             "hashes": [
-                "sha256:31a5fbbea5fc71c99e288ec0b2f00302a0a92c44b13ede80b73a6a4d6d205582",
-                "sha256:5457fc953ec560c5521b41fad9e6734a4668b7ba205832191bbdff40ec61073c"
+                "sha256:96fbe04e8ba0df289171e7f6970e0ff8b472bf4f909ed9e0e5beccbac7e1dbbe",
+                "sha256:c22b4cd8e388f8219dc121f091e53a8701f9f5bca9aa132b5254263cab516215"
             ],
             "index": "pypi",
-            "version": "==3.0.8"
+            "version": "==3.0.9"
         },
         "django-auth-ldap": {
             "hashes": [
@@ -80,7 +80,7 @@
                 "sha256:1c0a931b4245a0dcd5051ea1b244ac130a328374ce8e85254c75eb072a737201",
                 "sha256:32cc0f914ace4cef935c0d3f786dde2a52f0ea1be72153f6e356c0aa8f3925e1"
             ],
-            "markers": "python_version < '4.0' and python_full_version >= '3.6.0'",
+            "markers": "python_version >= '3.6' and python_version < '4.0'",
             "version": "==2.2.0"
         },
         "django-filter": {
@@ -101,11 +101,11 @@
         },
         "djangorestframework": {
             "hashes": [
-                "sha256:05809fc66e1c997fd9a32ea5730d9f4ba28b109b9da71fccfa5ff241201fd0a4",
-                "sha256:e782087823c47a26826ee5b6fa0c542968219263fb3976ec3c31edab23a4001f"
+                "sha256:6dd02d5a4bd2516fb93f80360673bf540c3b6641fec8766b1da2870a5aa00b32",
+                "sha256:8b1ac62c581dbc5799b03e535854b92fc4053ecfe74bad3f9c05782063d4196b"
             ],
             "index": "pypi",
-            "version": "==3.11.0"
+            "version": "==3.11.1"
         },
         "gitdb": {
             "hashes": [
@@ -177,11 +177,11 @@
         },
         "pygithub": {
             "hashes": [
-                "sha256:8375a058ec651cc0774244a3bc7395cf93617298735934cdd59e5bcd9a1df96e",
-                "sha256:d2d17d1e3f4474e070353f201164685a95b5a92f5ee0897442504e399c7bc249"
+                "sha256:371d17e855a2fd7a9ea1a9c71fd1d5d3e805369b60ce21121dd2931e5fbfc4e7",
+                "sha256:fc11a182ca3d88dde6ab2fbbd07e3441685cc8b738f7813fcbfc18be28c1d8ed"
             ],
             "index": "pypi",
-            "version": "==1.51"
+            "version": "==1.52"
         },
         "pyjwt": {
             "hashes": [
@@ -249,7 +249,7 @@
                 "sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a",
                 "sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
             "version": "==1.25.10"
         },
         "wrapt": {
@@ -276,11 +276,11 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:2dbcc8c27ca7d3cfe4fcdff7f45b27f9a8d3edfa70ff8024a71c7a8eb5f09d64",
-                "sha256:9f4fcb31d3b2c533333893b9172264e4821c1ac91839500f31bd43f2c59b3ccf"
+                "sha256:5a8f159ca8b22b9a0a1f2a28befe5ad2b703339afb58c2ffe0d7c8d7a3af5999",
+                "sha256:b70974aaa2674b05eb86a910c02ed09956a33f2dd6c71afc60f0b128a77e7f28"
             ],
             "index": "pypi",
-            "version": "==7.16.1"
+            "version": "==7.17.0"
         },
         "ipython-genutils": {
             "hashes": [

gh/tests.py

@@ -3,7 +3,6 @@ import re
 
 import git
 from hashlib import sha1
-from os import getenv
 
 from autoslug.utils import slugify
 from django.conf import settings

rainboard/filters.py

 import django_filters
 
 from . import models, utils
+from .models import Namespace, GEPETTO_SLUGS
 
 
 def filter_valid_name(queryset, name, value):
@@ -12,7 +13,18 @@ class ProjectFilter(django_filters.rest_framework.FilterSet):
 
     class Meta:
         model = models.Project
-        fields = ('name', 'from_gepetto', 'archived')
+        fields = ('name', 'main_namespace__from_gepetto', 'archived')
+
+
+class IssuePrFilter(django_filters.rest_framework.FilterSet):
+    namespace = django_filters.ModelChoiceFilter(queryset=Namespace.objects.filter(slug__in=GEPETTO_SLUGS),
+                                                 field_name='repo__namespace',
+                                                 label='namespace')
+    name = django_filters.CharFilter(field_name='repo__project__name', label='name', lookup_expr='icontains')
+
+    class Meta:
+        model = models.IssuePr
+        fields = ('name', 'namespace', 'is_issue')
 
 
 class ContributorFilter(django_filters.rest_framework.FilterSet):

rainboard/management/commands/update.py

+import re
 from datetime import timedelta
 
+import github
 from django.conf import settings
 from django.core.management import call_command
 from django.core.management.base import BaseCommand
 from django.db.models import F, Q
 from django.utils import timezone
 
-from rainboard.models import Branch, Forge, Image, Project, Repo, Robotpkg
+from rainboard.models import Branch, Forge, Image, Project, Repo, Robotpkg, IssuePr
 from rainboard.utils import SOURCES, update_robotpkg
 
+MIN_DAYS_SINCE_UPDATED = 10  # Only show issues and pull requests older than this
+SKIP_LABEL = 'skip dashboard'  # Issues and prs with this label will not be added to the dashboard
+
+
+def update_issues_pr():
+    print('\nUpdating issues and pull requests')
+    for project in Project.objects.filter(archived=False, main_namespace__from_gepetto=True):
+        try:
+            gh = project.github()
+            main_repo = project.repo_set.filter(namespace=project.main_namespace, forge__source=SOURCES.github).first()
+
+            # Create new issues and pull requests
+            for issue in gh.get_issues(state='open'):
+                days_since_updated = (timezone.now() - issue.updated_at).days
+                if main_repo is not None and days_since_updated > MIN_DAYS_SINCE_UPDATED \
+                        and SKIP_LABEL not in [label.name for label in issue.get_labels()]:
+                    url = re.sub('api\\.github\\.com/repos', 'github.com', issue.url)
+
+                    if issue.pull_request is None:
+                        IssuePr.objects.get_or_create(title=issue.title,
+                                                      repo=main_repo,
+                                                      number=issue.number,
+                                                      url=url,
+                                                      is_issue=True)
+
+                    else:
+                        IssuePr.objects.get_or_create(title=issue.title,
+                                                      repo=main_repo,
+                                                      number=issue.number,
+                                                      url=url,
+                                                      is_issue=False)
+
+        except github.UnknownObjectException:
+            print(f'Project not found: {project.main_namespace.slug}/{project.slug}')
+
+    # Update all issues and pull requests, delete closed ones
+    for issue_pr in IssuePr.objects.all():
+        issue_pr.update(SKIP_LABEL)
+
 
 class Command(BaseCommand):
     help = 'Update the DB'
@@ -59,3 +100,5 @@ class Command(BaseCommand):
         log(f'\nLook for missing images\n')
         for img in Image.objects.filter(created__lt=timezone.now() - timedelta(days=7), target__active=True):
             log(f' {img}')
+
+        update_issues_pr()

rainboard/migrations/0047_issuepr.py

+# Generated by Django 3.0.8 on 2020-08-03 15:10
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('rainboard', '0046_namespace_slug_gitlab_github'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='IssuePr',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('title', models.CharField(max_length=200)),
+                ('number', models.PositiveSmallIntegerField()),
+                ('days_since_updated', models.PositiveSmallIntegerField(blank=True, null=True)),
+                ('url', models.URLField()),
+                ('is_issue', models.BooleanField(default=True)),
+                ('repo', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Repo')),
+            ],
+            options={
+                'unique_together': {('repo', 'number', 'is_issue')},
+            },
+        ),
+    ]

rainboard/migrations/0048_namespace_from_gepetto.py

+# Generated by Django 3.0.8 on 2020-08-04 13:33
+
+from django.db import migrations, models
+
+from rainboard.models import GEPETTO_SLUGS
+
+
+def update_namespaces(apps, schema_editor):
+    Namespace = apps.get_model('rainboard', 'Namespace')
+    Namespace.objects.filter(slug__in=GEPETTO_SLUGS).update(from_gepetto=True)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ('rainboard', '0047_issuepr'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='project',
+            name='from_gepetto',
+        ),
+        migrations.AddField(
+            model_name='namespace',
+            name='from_gepetto',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.RunPython(update_namespaces),
+    ]

rainboard/models.py

@@ -48,10 +48,12 @@ CMAKE_FIELDS = {
 }
 TRAVIS_STATE = {'created': None, 'passed': True, 'started': None, 'failed': False, 'errored': False, 'canceled': False}
 GITLAB_STATUS = {'failed': False, 'success': True, 'pending': None, 'skipped': None, 'canceled': None, 'running': None}
+GEPETTO_SLUGS = ['gepetto', 'stack-of-tasks', 'humanoid-path-planner', 'loco-3d']
 
 
 class Namespace(NamedModel):
     group = models.BooleanField(default=False)
+    from_gepetto = models.BooleanField(default=False)
     slug_gitlab = models.CharField(max_length=200, default='')
     slug_github = models.CharField(max_length=200, default='')
 
@@ -199,7 +201,6 @@ class Project(Links, NamedModel, TimeStampedModel):
     updated = models.DateTimeField(blank=True, null=True)
     tests = models.BooleanField(default=True)
     docs = models.BooleanField(default=True)
-    from_gepetto = models.BooleanField(default=False)
     cmake_name = models.CharField(max_length=200, blank=True, null=True)
     archived = models.BooleanField(default=False)
     suffix = models.CharField(max_length=50, default='', blank=True)
@@ -665,6 +666,27 @@ class Repo(TimeStampedModel):
             logger.error(str(self.delete()))
 
 
+class IssuePr(models.Model):
+    title = models.CharField(max_length=200)
+    repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
+    number = models.PositiveSmallIntegerField()
+    days_since_updated = models.PositiveSmallIntegerField(blank=True, null=True)
+    url = models.URLField(max_length=200)
+    is_issue = models.BooleanField(default=True)
+
+    class Meta:
+        unique_together = ('repo', 'number', 'is_issue')
+
+    def update(self, skip_label):
+        gh = self.repo.project.github()
+        issue_pr = gh.get_issue(number=self.number) if self.is_issue else gh.get_pull(number=self.number)
+        self.days_since_updated = (timezone.now() - issue_pr.updated_at).days
+        if issue_pr.state == 'closed' or skip_label in [label.name for label in issue_pr.get_labels()]:
+            self.delete()
+        else:
+            self.save()
+
+
 class Commit(NamedModel, TimeStampedModel):
     project = models.ForeignKey(Project, on_delete=models.CASCADE)
 
@@ -988,7 +1010,7 @@ class Tag(models.Model):
 
 class GepettistQuerySet(models.QuerySet):
     def gepettist(self):
-        return self.filter(projects__from_gepetto=True, projects__archived=False)
+        return self.filter(projects__main_namespace__from_gepetto=True, projects__archived=False)
 
 
 class Contributor(models.Model):
@@ -1009,7 +1031,8 @@ class Contributor(models.Model):
         return ', '.join(str(mail) for mail in self.contributormail_set.filter(invalid=False))
 
     def contributed(self):
-        return ', '.join(str(project) for project in self.projects.filter(from_gepetto=True, archived=False))
+        return ', '.join(
+            str(project) for project in self.projects.filter(main_namespace__from_gepetto=True, archived=False))
 
 
 class ContributorName(models.Model):
@@ -1231,8 +1254,8 @@ def to_release_in_robotpkg():
 def ordered_projects():
     """ helper for gepetto/buildfarm/generate_all.py """
     fields = 'category', 'name', 'project__main_namespace__slug'
-    bad_ones = Q(from_gepetto=False) | Q(robotpkg__isnull=True) | Q(archived=True)
-    library_bad_ones = Q(library__from_gepetto=False) | Q(library__robotpkg__isnull=True)
+    bad_ones = Q(main_namespace__from_gepetto=False) | Q(robotpkg__isnull=True) | Q(archived=True)
+    library_bad_ones = Q(library__main_namespace__from_gepetto=False) | Q(library__robotpkg__isnull=True)
 
     main = Project.objects.exclude(bad_ones)
     ret = main.all().exclude(dependencies__isnull=False)

rainboard/tables.py

@@ -29,6 +29,7 @@ class ProjectTable(StrippedTable):
     pr = tables.Column(accessor='open_pr', orderable=False)
     rpkgs = tables.Column(accessor='rpkgs', orderable=False)
     badges = tables.Column(accessor='badges', orderable=False)
+    from_gepetto = tables.BooleanColumn(accessor='main_namespace__from_gepetto')
 
     class Meta:
         model = models.Project
@@ -121,3 +122,19 @@ class ContributorProjectTable(ContributorTable):
 
     class Meta:
         fields = ('names', 'mails', 'projects')
+
+
+class IssuePrTable(StrippedTable):
+    name = tables.Column(accessor='repo__project__name')
+
+    class Meta:
+        model = models.IssuePr
+        fields = ('repo__namespace', 'name', 'title', 'url', 'days_since_updated')
+        order_by = '-days_since_updated'
+
+    def render_name(self, record):
+        return record.repo.project.get_link()
+
+    def render_url(self, record):
+        rendered_name = 'issue #' if record.is_issue else 'PR #'
+        return mark_safe(f'<a href="{record.url}">{rendered_name}{record.number}</a>')

rainboard/templates/rainboard/issuepr_filter.html

+{% extends 'base.html' %}
+{% load django_tables2 bootstrap4 %}
+
+{% block content %}
+
+    <div class="d-flex justify-content-between">
+      <div>
+         <h1>Issues and Pull Requests</h1>
+      </div>
+      <div>
+          <a class="btn btn-primary" href="{% url 'rainboard:update_issues_pr'%}">Update</a>
+      </div>
+    </div>
+
+    {% if filter %}
+        <form action="" method="get" class="form form-inline">
+            {% bootstrap_form filter.form layout='inline' %}
+            {% bootstrap_button 'filter' %}
+        </form>
+    {% endif %}
+
+    {% render_table table %}
+
+{% endblock %}

rainboard/templates/rainboard/project_detail.html

@@ -40,7 +40,7 @@
               <dt class="col-3 text-right">Has tests</dt> <dd class="col-9">{{ project.tests|yesno:"✔,✘" }}</dd>
                <dt class="col-3 text-right">Has docs</dt> <dd class="col-9">{{ project.docs|yesno:"✔,✘" }}</dd>
     <dt class="col-3 text-right">Also build in Debug</dt> <dd class="col-9">{{ project.debug|yesno:"✔,✘" }}</dd>
-           <dt class="col-3 text-right">From Gepetto</dt> <dd class="col-9">{{ project.from_gepetto|yesno:"✔,✘" }}</dd>
+           <dt class="col-3 text-right">From Gepetto</dt> <dd class="col-9">{{ project.main_namespace.from_gepetto|yesno:"✔,✘" }}</dd>
                <dt class="col-3 text-right">Archived</dt> <dd class="col-9">{{ project.archived|yesno:"✔,✘" }}</dd>
            <dt class="col-3 text-right">Dependencies</dt> <dd class="col-9">{{ project.print_deps }}</dd>
    <dt class="col-3 text-right">Reverse Dependencies</dt> <dd class="col-9">{{ project.print_rdeps }}</dd>

rainboard/tests.py

@@ -5,6 +5,7 @@ from django.urls import reverse
 
 from dashboard import settings
 from . import models, utils
+from .models import IssuePr, Repo
 
 
 class RainboardTests(TestCase):
@@ -53,3 +54,26 @@ class RainboardTests(TestCase):
                 '<label class="label label-primary">BSD-2-Clause</label>',
         ]:
             self.assertIn(chunk, content)
+
+        # Test issues and pull requests view
+        repo = Repo.objects.create(name='foo',
+                                   forge=models.Forge.objects.get(source=utils.SOURCES.github),
+                                   namespace=models.Namespace.objects.get(slug='gepetto'),
+                                   project=project,
+                                   default_branch='master',
+                                   repo_id=4,
+                                   clone_url='https://github.com')
+
+        IssuePr.objects.create(title='Test issue', repo=repo, number=7, url='https://github.com', is_issue=True)
+
+        response = self.client.get(reverse('rainboard:issues_pr'), HTTP_X_FORWARDED_FOR='140.93.5.4')
+        self.assertEqual(response.status_code, 200)
+
+        content = response.content.decode()
+        for chunk in [
+                '<title>Gepetto Packages</title>', '<a class="btn btn-primary" href="/issues/update">Update</a>',
+                '<button class="btn btn-primary">filter</button>',
+                f'<a href="/project/{project.slug}/robotpkg">{project.name}</a>', '<td >Test issue</td>',
+                '<a href="https://github.com">issue #7</a>'
+        ]:
+            self.assertIn(chunk, content)

rainboard/urls.py

@@ -34,6 +34,8 @@ urlpatterns = [
     path('project/<str:slug>/images', views.ProjectImagesView.as_view(), name='project-images'),
     path('project/<str:slug>/contributors', views.ProjectContributorsView.as_view(), name='project-contributors'),
     path('project/<str:slug>/.gitlab-ci.yml', views.ProjectGitlabView.as_view(), name='project-gitlab'),
+    path('issues', views.IssuesPrView.as_view(), name='issues_pr'),
+    path('issues/update', views.update_issues_pr, name='update_issues_pr'),
     path('doc', views.json_doc, name='doc'),
     path('images', views.images_list, name='images'),
     path('docker', views.docker, name='docker'),

rainboard/views.py

-from subprocess import PIPE, run
+from subprocess import PIPE, Popen, run
 
 from django.http import Http404
-from django.http.response import HttpResponse, JsonResponse
+from django.http.response import HttpResponse, JsonResponse, HttpResponseRedirect
+from django.urls import reverse
 from django.views.generic import DetailView
 
 from django_filters.views import FilterView
@@ -34,7 +35,7 @@ class ProjectsView(SingleTableMixin, FilterView):
 
 
 class GepettoProjectsView(ProjectsView):
-    queryset = models.Project.objects.filter(from_gepetto=True, archived=False)
+    queryset = models.Project.objects.filter(main_namespace__from_gepetto=True, archived=False)
 
 
 class ProjectView(DetailView):
@@ -100,6 +101,21 @@ class ContributorsView(SingleTableMixin, DistinctMixin, FilterView):
     strict = False
 
 
+class IssuesPrView(SingleTableMixin, FilterView):
+    model = models.IssuePr
+    table_class = tables.IssuePrTable
+    filterset_class = filters.IssuePrFilter
+
+
+def update_issues_pr(request):
+    # Update issues and pull requests in a subprocess because it takes a long time to run
+    Popen([
+        'timeout', '600', './manage.py', 'shell', '-c',
+        'from rainboard.management.commands.update import update_issues_pr; update_issues_pr()'
+    ])
+    return HttpResponseRedirect(reverse('rainboard:issues_pr'))
+
+
 def json_doc(request):
     """
     Get the list of project / namespace / branch of which we want to keep the doc
@@ -129,10 +145,10 @@ def docker(request):
 def graph_svg(request):
     with open('/tmp/graph', 'w') as f:
         print('digraph { rankdir=LR;', file=f)
-        for project in models.Project.objects.filter(from_gepetto=True, archived=False):
+        for project in models.Project.objects.filter(main_namespace__from_gepetto=True, archived=False):
             print(f'{{I{project.pk} [label="{project}" URL="{project.get_absolute_url()}"];}}', file=f)
-        for dep in models.Dependency.objects.filter(project__from_gepetto=True,
-                                                    library__from_gepetto=True,
+        for dep in models.Dependency.objects.filter(project__main_namespace__from_gepetto=True,
+                                                    library__main_namespace__from_gepetto=True,
                                                     project__archived=False,
                                                     library__archived=False):
             print(f'I{dep.library.pk} -> I{dep.project.pk};', file=f)