Commit 5e724ba1 authored by Tom Pillot's avatar Tom Pillot Committed by Guilhem Saurel

New view showing issues and pull requests

parent 5ad49dc6
......@@ -54,11 +54,11 @@
},
"django": {
"hashes": [
"sha256:31a5fbbea5fc71c99e288ec0b2f00302a0a92c44b13ede80b73a6a4d6d205582",
"sha256:5457fc953ec560c5521b41fad9e6734a4668b7ba205832191bbdff40ec61073c"
"sha256:96fbe04e8ba0df289171e7f6970e0ff8b472bf4f909ed9e0e5beccbac7e1dbbe",
"sha256:c22b4cd8e388f8219dc121f091e53a8701f9f5bca9aa132b5254263cab516215"
],
"index": "pypi",
"version": "==3.0.8"
"version": "==3.0.9"
},
"django-auth-ldap": {
"hashes": [
......@@ -80,7 +80,7 @@
"sha256:1c0a931b4245a0dcd5051ea1b244ac130a328374ce8e85254c75eb072a737201",
"sha256:32cc0f914ace4cef935c0d3f786dde2a52f0ea1be72153f6e356c0aa8f3925e1"
],
"markers": "python_version < '4.0' and python_full_version >= '3.6.0'",
"markers": "python_version >= '3.6' and python_version < '4.0'",
"version": "==2.2.0"
},
"django-filter": {
......@@ -101,11 +101,11 @@
},
"djangorestframework": {
"hashes": [
"sha256:05809fc66e1c997fd9a32ea5730d9f4ba28b109b9da71fccfa5ff241201fd0a4",
"sha256:e782087823c47a26826ee5b6fa0c542968219263fb3976ec3c31edab23a4001f"
"sha256:6dd02d5a4bd2516fb93f80360673bf540c3b6641fec8766b1da2870a5aa00b32",
"sha256:8b1ac62c581dbc5799b03e535854b92fc4053ecfe74bad3f9c05782063d4196b"
],
"index": "pypi",
"version": "==3.11.0"
"version": "==3.11.1"
},
"gitdb": {
"hashes": [
......@@ -177,11 +177,11 @@
},
"pygithub": {
"hashes": [
"sha256:8375a058ec651cc0774244a3bc7395cf93617298735934cdd59e5bcd9a1df96e",
"sha256:d2d17d1e3f4474e070353f201164685a95b5a92f5ee0897442504e399c7bc249"
"sha256:371d17e855a2fd7a9ea1a9c71fd1d5d3e805369b60ce21121dd2931e5fbfc4e7",
"sha256:fc11a182ca3d88dde6ab2fbbd07e3441685cc8b738f7813fcbfc18be28c1d8ed"
],
"index": "pypi",
"version": "==1.51"
"version": "==1.52"
},
"pyjwt": {
"hashes": [
......@@ -249,7 +249,7 @@
"sha256:91056c15fa70756691db97756772bb1eb9678fa585d9184f24534b100dc60f4a",
"sha256:e7983572181f5e1522d9c98453462384ee92a0be7fac5f1413a1e35c56cc0461"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4.0'",
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
"version": "==1.25.10"
},
"wrapt": {
......@@ -276,11 +276,11 @@
},
"ipython": {
"hashes": [
"sha256:2dbcc8c27ca7d3cfe4fcdff7f45b27f9a8d3edfa70ff8024a71c7a8eb5f09d64",
"sha256:9f4fcb31d3b2c533333893b9172264e4821c1ac91839500f31bd43f2c59b3ccf"
"sha256:5a8f159ca8b22b9a0a1f2a28befe5ad2b703339afb58c2ffe0d7c8d7a3af5999",
"sha256:b70974aaa2674b05eb86a910c02ed09956a33f2dd6c71afc60f0b128a77e7f28"
],
"index": "pypi",
"version": "==7.16.1"
"version": "==7.17.0"
},
"ipython-genutils": {
"hashes": [
......
......@@ -3,7 +3,6 @@ import re
import git
from hashlib import sha1
from os import getenv
from autoslug.utils import slugify
from django.conf import settings
......
import django_filters
from . import models, utils
from .models import Namespace, GEPETTO_SLUGS
def filter_valid_name(queryset, name, value):
......@@ -12,7 +13,18 @@ class ProjectFilter(django_filters.rest_framework.FilterSet):
class Meta:
model = models.Project
fields = ('name', 'from_gepetto', 'archived')
fields = ('name', 'main_namespace__from_gepetto', 'archived')
class IssuePrFilter(django_filters.rest_framework.FilterSet):
namespace = django_filters.ModelChoiceFilter(queryset=Namespace.objects.filter(slug__in=GEPETTO_SLUGS),
field_name='repo__namespace',
label='namespace')
name = django_filters.CharFilter(field_name='repo__project__name', label='name', lookup_expr='icontains')
class Meta:
model = models.IssuePr
fields = ('name', 'namespace', 'is_issue')
class ContributorFilter(django_filters.rest_framework.FilterSet):
......
import re
from datetime import timedelta
import github
from django.conf import settings
from django.core.management import call_command
from django.core.management.base import BaseCommand
from django.db.models import F, Q
from django.utils import timezone
from rainboard.models import Branch, Forge, Image, Project, Repo, Robotpkg
from rainboard.models import Branch, Forge, Image, Project, Repo, Robotpkg, IssuePr
from rainboard.utils import SOURCES, update_robotpkg
MIN_DAYS_SINCE_UPDATED = 10 # Only show issues and pull requests older than this
SKIP_LABEL = 'skip dashboard' # Issues and prs with this label will not be added to the dashboard
def update_issues_pr():
print('\nUpdating issues and pull requests')
for project in Project.objects.filter(archived=False, main_namespace__from_gepetto=True):
try:
gh = project.github()
main_repo = project.repo_set.filter(namespace=project.main_namespace, forge__source=SOURCES.github).first()
# Create new issues and pull requests
for issue in gh.get_issues(state='open'):
days_since_updated = (timezone.now() - issue.updated_at).days
if main_repo is not None and days_since_updated > MIN_DAYS_SINCE_UPDATED \
and SKIP_LABEL not in [label.name for label in issue.get_labels()]:
url = re.sub('api\\.github\\.com/repos', 'github.com', issue.url)
if issue.pull_request is None:
IssuePr.objects.get_or_create(title=issue.title,
repo=main_repo,
number=issue.number,
url=url,
is_issue=True)
else:
IssuePr.objects.get_or_create(title=issue.title,
repo=main_repo,
number=issue.number,
url=url,
is_issue=False)
except github.UnknownObjectException:
print(f'Project not found: {project.main_namespace.slug}/{project.slug}')
# Update all issues and pull requests, delete closed ones
for issue_pr in IssuePr.objects.all():
issue_pr.update(SKIP_LABEL)
class Command(BaseCommand):
help = 'Update the DB'
......@@ -59,3 +100,5 @@ class Command(BaseCommand):
log(f'\nLook for missing images\n')
for img in Image.objects.filter(created__lt=timezone.now() - timedelta(days=7), target__active=True):
log(f' {img}')
update_issues_pr()
# Generated by Django 3.0.8 on 2020-08-03 15:10
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('rainboard', '0046_namespace_slug_gitlab_github'),
]
operations = [
migrations.CreateModel(
name='IssuePr',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('title', models.CharField(max_length=200)),
('number', models.PositiveSmallIntegerField()),
('days_since_updated', models.PositiveSmallIntegerField(blank=True, null=True)),
('url', models.URLField()),
('is_issue', models.BooleanField(default=True)),
('repo', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='rainboard.Repo')),
],
options={
'unique_together': {('repo', 'number', 'is_issue')},
},
),
]
# Generated by Django 3.0.8 on 2020-08-04 13:33
from django.db import migrations, models
from rainboard.models import GEPETTO_SLUGS
def update_namespaces(apps, schema_editor):
Namespace = apps.get_model('rainboard', 'Namespace')
Namespace.objects.filter(slug__in=GEPETTO_SLUGS).update(from_gepetto=True)
class Migration(migrations.Migration):
dependencies = [
('rainboard', '0047_issuepr'),
]
operations = [
migrations.RemoveField(
model_name='project',
name='from_gepetto',
),
migrations.AddField(
model_name='namespace',
name='from_gepetto',
field=models.BooleanField(default=False),
),
migrations.RunPython(update_namespaces),
]
......@@ -48,10 +48,12 @@ CMAKE_FIELDS = {
}
TRAVIS_STATE = {'created': None, 'passed': True, 'started': None, 'failed': False, 'errored': False, 'canceled': False}
GITLAB_STATUS = {'failed': False, 'success': True, 'pending': None, 'skipped': None, 'canceled': None, 'running': None}
GEPETTO_SLUGS = ['gepetto', 'stack-of-tasks', 'humanoid-path-planner', 'loco-3d']
class Namespace(NamedModel):
group = models.BooleanField(default=False)
from_gepetto = models.BooleanField(default=False)
slug_gitlab = models.CharField(max_length=200, default='')
slug_github = models.CharField(max_length=200, default='')
......@@ -199,7 +201,6 @@ class Project(Links, NamedModel, TimeStampedModel):
updated = models.DateTimeField(blank=True, null=True)
tests = models.BooleanField(default=True)
docs = models.BooleanField(default=True)
from_gepetto = models.BooleanField(default=False)
cmake_name = models.CharField(max_length=200, blank=True, null=True)
archived = models.BooleanField(default=False)
suffix = models.CharField(max_length=50, default='', blank=True)
......@@ -665,6 +666,27 @@ class Repo(TimeStampedModel):
logger.error(str(self.delete()))
class IssuePr(models.Model):
title = models.CharField(max_length=200)
repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
number = models.PositiveSmallIntegerField()
days_since_updated = models.PositiveSmallIntegerField(blank=True, null=True)
url = models.URLField(max_length=200)
is_issue = models.BooleanField(default=True)
class Meta:
unique_together = ('repo', 'number', 'is_issue')
def update(self, skip_label):
gh = self.repo.project.github()
issue_pr = gh.get_issue(number=self.number) if self.is_issue else gh.get_pull(number=self.number)
self.days_since_updated = (timezone.now() - issue_pr.updated_at).days
if issue_pr.state == 'closed' or skip_label in [label.name for label in issue_pr.get_labels()]:
self.delete()
else:
self.save()
class Commit(NamedModel, TimeStampedModel):
project = models.ForeignKey(Project, on_delete=models.CASCADE)
......@@ -988,7 +1010,7 @@ class Tag(models.Model):
class GepettistQuerySet(models.QuerySet):
def gepettist(self):
return self.filter(projects__from_gepetto=True, projects__archived=False)
return self.filter(projects__main_namespace__from_gepetto=True, projects__archived=False)
class Contributor(models.Model):
......@@ -1009,7 +1031,8 @@ class Contributor(models.Model):
return ', '.join(str(mail) for mail in self.contributormail_set.filter(invalid=False))
def contributed(self):
return ', '.join(str(project) for project in self.projects.filter(from_gepetto=True, archived=False))
return ', '.join(
str(project) for project in self.projects.filter(main_namespace__from_gepetto=True, archived=False))
class ContributorName(models.Model):
......@@ -1231,8 +1254,8 @@ def to_release_in_robotpkg():
def ordered_projects():
""" helper for gepetto/buildfarm/generate_all.py """
fields = 'category', 'name', 'project__main_namespace__slug'
bad_ones = Q(from_gepetto=False) | Q(robotpkg__isnull=True) | Q(archived=True)
library_bad_ones = Q(library__from_gepetto=False) | Q(library__robotpkg__isnull=True)
bad_ones = Q(main_namespace__from_gepetto=False) | Q(robotpkg__isnull=True) | Q(archived=True)
library_bad_ones = Q(library__main_namespace__from_gepetto=False) | Q(library__robotpkg__isnull=True)
main = Project.objects.exclude(bad_ones)
ret = main.all().exclude(dependencies__isnull=False)
......
......@@ -29,6 +29,7 @@ class ProjectTable(StrippedTable):
pr = tables.Column(accessor='open_pr', orderable=False)
rpkgs = tables.Column(accessor='rpkgs', orderable=False)
badges = tables.Column(accessor='badges', orderable=False)
from_gepetto = tables.BooleanColumn(accessor='main_namespace__from_gepetto')
class Meta:
model = models.Project
......@@ -121,3 +122,19 @@ class ContributorProjectTable(ContributorTable):
class Meta:
fields = ('names', 'mails', 'projects')
class IssuePrTable(StrippedTable):
name = tables.Column(accessor='repo__project__name')
class Meta:
model = models.IssuePr
fields = ('repo__namespace', 'name', 'title', 'url', 'days_since_updated')
order_by = '-days_since_updated'
def render_name(self, record):
return record.repo.project.get_link()
def render_url(self, record):
rendered_name = 'issue #' if record.is_issue else 'PR #'
return mark_safe(f'<a href="{record.url}">{rendered_name}{record.number}</a>')
{% extends 'base.html' %}
{% load django_tables2 bootstrap4 %}
{% block content %}
<div class="d-flex justify-content-between">
<div>
<h1>Issues and Pull Requests</h1>
</div>
<div>
<a class="btn btn-primary" href="{% url 'rainboard:update_issues_pr'%}">Update</a>
</div>
</div>
{% if filter %}
<form action="" method="get" class="form form-inline">
{% bootstrap_form filter.form layout='inline' %}
{% bootstrap_button 'filter' %}
</form>
{% endif %}
{% render_table table %}
{% endblock %}
......@@ -40,7 +40,7 @@
<dt class="col-3 text-right">Has tests</dt> <dd class="col-9">{{ project.tests|yesno:"✔,✘" }}</dd>
<dt class="col-3 text-right">Has docs</dt> <dd class="col-9">{{ project.docs|yesno:"✔,✘" }}</dd>
<dt class="col-3 text-right">Also build in Debug</dt> <dd class="col-9">{{ project.debug|yesno:"✔,✘" }}</dd>
<dt class="col-3 text-right">From Gepetto</dt> <dd class="col-9">{{ project.from_gepetto|yesno:"✔,✘" }}</dd>
<dt class="col-3 text-right">From Gepetto</dt> <dd class="col-9">{{ project.main_namespace.from_gepetto|yesno:"✔,✘" }}</dd>
<dt class="col-3 text-right">Archived</dt> <dd class="col-9">{{ project.archived|yesno:"✔,✘" }}</dd>
<dt class="col-3 text-right">Dependencies</dt> <dd class="col-9">{{ project.print_deps }}</dd>
<dt class="col-3 text-right">Reverse Dependencies</dt> <dd class="col-9">{{ project.print_rdeps }}</dd>
......
......@@ -5,6 +5,7 @@ from django.urls import reverse
from dashboard import settings
from . import models, utils
from .models import IssuePr, Repo
class RainboardTests(TestCase):
......@@ -53,3 +54,26 @@ class RainboardTests(TestCase):
'<label class="label label-primary">BSD-2-Clause</label>',
]:
self.assertIn(chunk, content)
# Test issues and pull requests view
repo = Repo.objects.create(name='foo',
forge=models.Forge.objects.get(source=utils.SOURCES.github),
namespace=models.Namespace.objects.get(slug='gepetto'),
project=project,
default_branch='master',
repo_id=4,
clone_url='https://github.com')
IssuePr.objects.create(title='Test issue', repo=repo, number=7, url='https://github.com', is_issue=True)
response = self.client.get(reverse('rainboard:issues_pr'), HTTP_X_FORWARDED_FOR='140.93.5.4')
self.assertEqual(response.status_code, 200)
content = response.content.decode()
for chunk in [
'<title>Gepetto Packages</title>', '<a class="btn btn-primary" href="/issues/update">Update</a>',
'<button class="btn btn-primary">filter</button>',
f'<a href="/project/{project.slug}/robotpkg">{project.name}</a>', '<td >Test issue</td>',
'<a href="https://github.com">issue #7</a>'
]:
self.assertIn(chunk, content)
......@@ -34,6 +34,8 @@ urlpatterns = [
path('project/<str:slug>/images', views.ProjectImagesView.as_view(), name='project-images'),
path('project/<str:slug>/contributors', views.ProjectContributorsView.as_view(), name='project-contributors'),
path('project/<str:slug>/.gitlab-ci.yml', views.ProjectGitlabView.as_view(), name='project-gitlab'),
path('issues', views.IssuesPrView.as_view(), name='issues_pr'),
path('issues/update', views.update_issues_pr, name='update_issues_pr'),
path('doc', views.json_doc, name='doc'),
path('images', views.images_list, name='images'),
path('docker', views.docker, name='docker'),
......
from subprocess import PIPE, run
from subprocess import PIPE, Popen, run
from django.http import Http404
from django.http.response import HttpResponse, JsonResponse
from django.http.response import HttpResponse, JsonResponse, HttpResponseRedirect
from django.urls import reverse
from django.views.generic import DetailView
from django_filters.views import FilterView
......@@ -34,7 +35,7 @@ class ProjectsView(SingleTableMixin, FilterView):
class GepettoProjectsView(ProjectsView):
queryset = models.Project.objects.filter(from_gepetto=True, archived=False)
queryset = models.Project.objects.filter(main_namespace__from_gepetto=True, archived=False)
class ProjectView(DetailView):
......@@ -100,6 +101,21 @@ class ContributorsView(SingleTableMixin, DistinctMixin, FilterView):
strict = False
class IssuesPrView(SingleTableMixin, FilterView):
model = models.IssuePr
table_class = tables.IssuePrTable
filterset_class = filters.IssuePrFilter
def update_issues_pr(request):
# Update issues and pull requests in a subprocess because it takes a long time to run
Popen([
'timeout', '600', './manage.py', 'shell', '-c',
'from rainboard.management.commands.update import update_issues_pr; update_issues_pr()'
])
return HttpResponseRedirect(reverse('rainboard:issues_pr'))
def json_doc(request):
"""
Get the list of project / namespace / branch of which we want to keep the doc
......@@ -129,10 +145,10 @@ def docker(request):
def graph_svg(request):
with open('/tmp/graph', 'w') as f:
print('digraph { rankdir=LR;', file=f)
for project in models.Project.objects.filter(from_gepetto=True, archived=False):
for project in models.Project.objects.filter(main_namespace__from_gepetto=True, archived=False):
print(f'{{I{project.pk} [label="{project}" URL="{project.get_absolute_url()}"];}}', file=f)
for dep in models.Dependency.objects.filter(project__from_gepetto=True,
library__from_gepetto=True,
for dep in models.Dependency.objects.filter(project__main_namespace__from_gepetto=True,
library__main_namespace__from_gepetto=True,
project__archived=False,
library__archived=False):
print(f'I{dep.library.pk} -> I{dep.project.pk};', file=f)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment