Commit 983969a7 authored by Tom Pillot's avatar Tom Pillot
Browse files

Add tests, search bar and update button in issues view

parent 4f0b8b86
Pipeline #10514 failed with stage
in 2 minutes and 32 seconds
import django_filters
from . import models, utils
from .models import Namespace, GEPETTO_SLUGS
def filter_valid_name(queryset, name, value):
......@@ -12,18 +13,17 @@ class ProjectFilter(django_filters.rest_framework.FilterSet):
class Meta:
model = models.Project
fields = ('name', 'from_gepetto', 'archived')
fields = ('name', 'main_namespace__from_gepetto', 'archived')
class IssuePrFilter(django_filters.rest_framework.FilterSet):
namespace = django_filters.CharFilter(field_name='repo__namespace__name', label='namespace',
lookup_expr='icontains')
namespace = django_filters.ModelChoiceFilter(queryset=Namespace.objects.filter(slug__in=GEPETTO_SLUGS),
field_name='repo__namespace', label='namespace')
name = django_filters.CharFilter(field_name='repo__project__name', label='name', lookup_expr='icontains')
is_issue = django_filters.BooleanFilter(field_name='is_issue', label='issues')
class Meta:
model = models.IssuePr
fields = ('namespace', 'name', 'is_issue')
fields = ('name', 'namespace', 'is_issue')
class ContributorFilter(django_filters.rest_framework.FilterSet):
......
import re
from datetime import timedelta
from datetime import timedelta, datetime
import github
from django.conf import settings
......@@ -10,9 +10,49 @@ from django.db.models import F, Q
from django.utils import timezone
from github import Github
from rainboard.models import Branch, Forge, Image, Project, Repo, Robotpkg, IssuePr
from rainboard.models import Branch, Forge, Image, Project, Repo, Robotpkg, IssuePr, GEPETTO_SLUGS
from rainboard.utils import SOURCES, update_robotpkg
MIN_DAYS_SINCE_UPDATED = 10 # Only show issues and pull requests older than this
SKIP_LABEL = 'skip dashboard' # Issues and prs with this label will not be added to the dashboard
def update_issues_pr():
print('\nUpdating issues and pull requests')
now = datetime.now()
for project in Project.objects.filter(archived=False, main_namespace__slug__in=GEPETTO_SLUGS):
try:
gh = project.github()
main_repo = project.repo_set.filter(namespace=project.main_namespace, forge__source=SOURCES.github).first()
# Create new issues and pull requests
for issue in gh.get_issues(state='open'):
days_since_updated = (now - issue.updated_at).days
if main_repo is not None and days_since_updated > MIN_DAYS_SINCE_UPDATED \
and SKIP_LABEL not in [label.name for label in issue.get_labels()]:
url = re.sub('api\\.github\\.com/repos', 'github.com', issue.url)
if issue.pull_request is None:
IssuePr.objects.get_or_create(title=issue.title,
repo=main_repo,
number=issue.number,
url=url,
is_issue=True)
else:
IssuePr.objects.get_or_create(title=issue.title,
repo=main_repo,
number=issue.number,
url=url,
is_issue=False)
except github.UnknownObjectException:
print(f'Project not found: {project.main_namespace.slug_github}/{project.slug}')
# Update all issues and pull requests, delete closed ones
for issue_pr in IssuePr.objects.all():
issue_pr.update(SKIP_LABEL)
class Command(BaseCommand):
help = 'Update the DB'
......@@ -64,34 +104,4 @@ class Command(BaseCommand):
# for img in Image.objects.filter(created__lt=timezone.now() - timedelta(days=7), target__active=True):
# log(f' {img}')
log('\nUpdating issues and pull requests\n')
for project in Project.objects.filter(archived=False, main_namespace__slug='gepetto'):
try:
gh = project.github()
main_repo = project.repo_set.filter(namespace=project.main_namespace, forge__source=SOURCES.github).first()
# Create new issues
for issue in gh.get_issues(state='open'):
url = re.sub('api\\.github\\.com/repos', 'github.com', issue.url)
obj, created = IssuePr.objects.get_or_create(title=issue.title,
repo=main_repo,
number=issue.number,
url=url,
is_issue=True)
# Create new pull requests
for pr in gh.get_pulls(state='open'):
url = re.sub('api\\.github\\.com/repos', 'github.com', pr.url)
url = re.sub('pulls/', 'pull/', url)
obj, created = IssuePr.objects.get_or_create(title=pr.title,
repo=main_repo,
number=pr.number,
url=url,
is_issue=False)
except github.UnknownObjectException as e:
log(f'\nProject not found: {project.main_namespace.slug_github}/{project.slug}\n')
# Update all issues and pull requests, delete closed ones
for issue_pr in IssuePr.objects.all():
issue_pr.update()
update_issues_pr()
# Generated by Django 3.0.7 on 2020-07-21 08:59
from django.db import migrations
def add_namespace(apps, schema_editor):
Namespace = apps.get_model('rainboard', 'Namespace')
Namespace.objects.create(name='tpillot', slug_gitlab='tpillot', slug_github='ozon2')
def add_project(apps, schema_editor):
Project = apps.get_model('rainboard', 'Project')
Namespace = apps.get_model('rainboard', 'Namespace')
Forge = apps.get_model('rainboard', 'Forge')
Project.objects.create(
name='example-adder',
main_namespace=Namespace.objects.get(name='tpillot'),
main_forge=Forge.objects.get(name='Gitlab'),
from_gepetto=True
)
class Migration(migrations.Migration):
dependencies = [
('rainboard', '0046_namespace_slug_gitlab_github'),
]
operations = [
migrations.RunPython(add_namespace),
migrations.RunPython(add_project),
]
......@@ -7,7 +7,7 @@ import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('rainboard', '0047_example-adder'),
('rainboard', '0046_namespace_slug_gitlab_github'),
]
operations = [
......
# Generated by Django 3.0.8 on 2020-08-04 13:33
from django.db import migrations, models
from rainboard.models import GEPETTO_SLUGS
def update_namespaces(apps, schema_editor):
Namespace = apps.get_model('rainboard', 'Namespace')
Namespace.objects.filter(slug__in=GEPETTO_SLUGS).update(from_gepetto=True)
class Migration(migrations.Migration):
dependencies = [
('rainboard', '0047_issuepr'),
]
operations = [
migrations.RemoveField(
model_name='project',
name='from_gepetto',
),
migrations.AddField(
model_name='namespace',
name='from_gepetto',
field=models.BooleanField(default=False),
),
migrations.RunPython(update_namespaces),
]
......@@ -48,12 +48,14 @@ CMAKE_FIELDS = {
}
TRAVIS_STATE = {'created': None, 'passed': True, 'started': None, 'failed': False, 'errored': False, 'canceled': False}
GITLAB_STATUS = {'failed': False, 'success': True, 'pending': None, 'skipped': None, 'canceled': None, 'running': None}
GEPETTO_SLUGS = ['gepetto', 'stack-of-tasks', 'humanoid-path-planner', 'loco-3d']
class Namespace(NamedModel):
group = models.BooleanField(default=False)
slug_gitlab = models.CharField(max_length=200, default='')
slug_github = models.CharField(max_length=200, default='')
from_gepetto = models.BooleanField(default=False)
def save(self, *args, **kwargs):
if self.slug_gitlab == '':
......@@ -199,7 +201,6 @@ class Project(Links, NamedModel, TimeStampedModel):
updated = models.DateTimeField(blank=True, null=True)
tests = models.BooleanField(default=True)
docs = models.BooleanField(default=True)
from_gepetto = models.BooleanField(default=False)
cmake_name = models.CharField(max_length=200, blank=True, null=True)
archived = models.BooleanField(default=False)
suffix = models.CharField(max_length=50, default='', blank=True)
......@@ -676,11 +677,11 @@ class IssuePr(models.Model):
class Meta:
unique_together = ('repo', 'number', 'is_issue')
def update(self):
def update(self, skip_label):
gh = self.repo.project.github()
issue_pr = gh.get_issue(number=self.number) if self.is_issue else gh.get_pull(number=self.number)
self.days_since_updated = (datetime.now() - issue_pr.updated_at).days
if issue_pr.state == 'closed':
if issue_pr.state == 'closed' or skip_label in [label.name for label in issue_pr.get_labels()]:
self.delete()
else:
self.save()
......@@ -1015,7 +1016,7 @@ class Tag(models.Model):
class GepettistQuerySet(models.QuerySet):
def gepettist(self):
return self.filter(projects__from_gepetto=True, projects__archived=False)
return self.filter(projects__main_namespace__from_gepetto=True, projects__archived=False)
class Contributor(models.Model):
......@@ -1036,7 +1037,7 @@ class Contributor(models.Model):
return ', '.join(str(mail) for mail in self.contributormail_set.filter(invalid=False))
def contributed(self):
return ', '.join(str(project) for project in self.projects.filter(from_gepetto=True, archived=False))
return ', '.join(str(project) for project in self.projects.filter(main_namespace__from_gepetto=True, archived=False))
class ContributorName(models.Model):
......@@ -1258,8 +1259,8 @@ def to_release_in_robotpkg():
def ordered_projects():
""" helper for gepetto/buildfarm/generate_all.py """
fields = 'category', 'name', 'project__main_namespace__slug'
bad_ones = Q(from_gepetto=False) | Q(robotpkg__isnull=True) | Q(archived=True)
library_bad_ones = Q(library__from_gepetto=False) | Q(library__robotpkg__isnull=True)
bad_ones = Q(main_namespace__from_gepetto=False) | Q(robotpkg__isnull=True) | Q(archived=True)
library_bad_ones = Q(library__main_namespace__from_gepetto=False) | Q(library__robotpkg__isnull=True)
main = Project.objects.exclude(bad_ones)
ret = main.all().exclude(dependencies__isnull=False)
......
......@@ -29,6 +29,7 @@ class ProjectTable(StrippedTable):
pr = tables.Column(accessor='open_pr', orderable=False)
rpkgs = tables.Column(accessor='rpkgs', orderable=False)
badges = tables.Column(accessor='badges', orderable=False)
from_gepetto = tables.BooleanColumn(accessor='main_namespace__from_gepetto')
class Meta:
model = models.Project
......@@ -124,11 +125,11 @@ class ContributorProjectTable(ContributorTable):
class IssuePrTable(StrippedTable):
name = tables.Column(accessor='repo.project.name')
name = tables.Column(accessor='repo__project__name')
class Meta:
model = models.IssuePr
fields = ('repo.namespace', 'name', 'title', 'url', 'days_since_updated')
fields = ('repo__namespace', 'name', 'title', 'url', 'days_since_updated')
order_by = '-days_since_updated'
def render_name(self, record):
......
......@@ -3,15 +3,22 @@
{% block content %}
<h1>Issues and Pull Requests</h1>
<div class="d-flex justify-content-between">
<div>
<h1>Issues and Pull Requests</h1>
</div>
<div>
<a class="btn btn-primary" href="{% url 'rainboard:update_issues_pr'%}">Update</a>
</div>
</div>
{% if filter %}
<form action="" method="get" class="form form-inline">
{% bootstrap_form filter.form layout='inline' %}
{% bootstrap_button 'filter' %}
</form>
{% endif %}
{% if filter %}
<form action="" method="get" class="form form-inline">
{% bootstrap_form filter.form layout='inline' %}
{% bootstrap_button 'filter' %}
</form>
{% endif %}
{% render_table table %}
{% render_table table %}
{% endblock %}
......@@ -40,7 +40,7 @@
<dt class="col-3 text-right">Has tests</dt> <dd class="col-9">{{ project.tests|yesno:"✔,✘" }}</dd>
<dt class="col-3 text-right">Has docs</dt> <dd class="col-9">{{ project.docs|yesno:"✔,✘" }}</dd>
<dt class="col-3 text-right">Also build in Debug</dt> <dd class="col-9">{{ project.debug|yesno:"✔,✘" }}</dd>
<dt class="col-3 text-right">From Gepetto</dt> <dd class="col-9">{{ project.from_gepetto|yesno:"✔,✘" }}</dd>
<dt class="col-3 text-right">From Gepetto</dt> <dd class="col-9">{{ project.main_namespace.from_gepetto|yesno:"✔,✘" }}</dd>
<dt class="col-3 text-right">Archived</dt> <dd class="col-9">{{ project.archived|yesno:"✔,✘" }}</dd>
<dt class="col-3 text-right">Dependencies</dt> <dd class="col-9">{{ project.print_deps }}</dd>
<dt class="col-3 text-right">Reverse Dependencies</dt> <dd class="col-9">{{ project.print_rdeps }}</dd>
......
......@@ -5,6 +5,7 @@ from django.urls import reverse
from dashboard import settings
from . import models, utils
from .models import IssuePr, Repo
class RainboardTests(TestCase):
......@@ -55,3 +56,32 @@ class RainboardTests(TestCase):
'<label class="label label-primary">BSD-2-Clause</label>',
]:
self.assertIn(chunk, content)
# Test issues and pull requests view
repo = Repo.objects.create(name='foo',
forge=models.Forge.objects.get(source=utils.SOURCES.github),
namespace=models.Namespace.objects.get(slug='gepetto'),
project=project,
default_branch='master',
repo_id=4,
clone_url='https://github.com')
IssuePr.objects.create(title='Test issue',
repo=repo,
number=7,
url='https://github.com',
is_issue=True)
response = self.client.get(reverse('rainboard:issues_pr'))
self.assertEqual(response.status_code, 200)
content = response.content.decode()
for chunk in [
'<title>Gepetto Packages</title>',
'<a class="btn btn-primary" href="/issues/update">Update</a>',
'<button class="btn btn-primary">filter</button>',
'<a href="/project/rainboard-tests-2/robotpkg">rainboard tests 2</a>',
'<td >Test issue</td>',
'<a href="https://github.com">issue #7</a>'
]:
self.assertIn(chunk, content)
......@@ -35,6 +35,7 @@ urlpatterns = [
path('project/<str:slug>/contributors', views.ProjectContributorsView.as_view(), name='project-contributors'),
path('project/<str:slug>/.gitlab-ci.yml', views.ProjectGitlabView.as_view(), name='project-gitlab'),
path('issues', views.IssuesPrView.as_view(), name='issues_pr'),
path('issues/update', views.update_issues_pr, name='update_issues_pr'),
path('doc', views.json_doc, name='doc'),
path('images', views.images_list, name='images'),
path('docker', views.docker, name='docker'),
......
import subprocess
from subprocess import PIPE, run
from django.http import Http404
from django.http.response import HttpResponse, JsonResponse
from django.http.response import HttpResponse, JsonResponse, HttpResponseRedirect
from django.urls import reverse
from django.views.generic import DetailView
from django_filters.views import FilterView
......@@ -34,7 +36,7 @@ class ProjectsView(SingleTableMixin, FilterView):
class GepettoProjectsView(ProjectsView):
queryset = models.Project.objects.filter(from_gepetto=True, archived=False)
queryset = models.Project.objects.filter(main_namespace__from_gepetto=True, archived=False)
class ProjectView(DetailView):
......@@ -106,6 +108,13 @@ class IssuesPrView(SingleTableMixin, FilterView):
filterset_class = filters.IssuePrFilter
def update_issues_pr(request):
# Update issues and pull requests in a subprocess because it takes a long time to run
subprocess.Popen(['timeout', '600', './manage.py', 'shell', '-c',
'from rainboard.management.commands.update import update_issues_pr; update_issues_pr()'])
return HttpResponseRedirect(reverse('rainboard:issues_pr'))
def json_doc(request):
"""
Get the list of project / namespace / branch of which we want to keep the doc
......@@ -135,10 +144,10 @@ def docker(request):
def graph_svg(request):
with open('/tmp/graph', 'w') as f:
print('digraph { rankdir=LR;', file=f)
for project in models.Project.objects.filter(from_gepetto=True, archived=False):
for project in models.Project.objects.filter(main_namespace__from_gepetto=True, archived=False):
print(f'{{I{project.pk} [label="{project}" URL="{project.get_absolute_url()}"];}}', file=f)
for dep in models.Dependency.objects.filter(project__from_gepetto=True,
library__from_gepetto=True,
for dep in models.Dependency.objects.filter(project__main_namespace__from_gepetto=True,
library__main_namespace__from_gepetto=True,
project__archived=False,
library__archived=False):
print(f'I{dep.library.pk} -> I{dep.project.pk};', file=f)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment