update.py 4.35 KB
Newer Older
1
import re
2
3
from datetime import timedelta

4
import github
Guilhem Saurel's avatar
Guilhem Saurel committed
5
from django.conf import settings
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
6
from django.core.management import call_command
Guilhem Saurel's avatar
Guilhem Saurel committed
7
from django.core.management.base import BaseCommand
Guilhem Saurel's avatar
Guilhem Saurel committed
8
from django.db.models import F, Q
9
from django.utils import timezone
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
10

11
from rainboard.models import Branch, Forge, Image, Project, Repo, Robotpkg, IssuePr
12
from rainboard.utils import SOURCES, update_robotpkg
Guilhem Saurel's avatar
Guilhem Saurel committed
13

14
15
16
17
18
19
20
21
22
23
24
25
26
MIN_DAYS_SINCE_UPDATED = 10  # Only show issues and pull requests older than this
SKIP_LABEL = 'skip dashboard'  # Issues and prs with this label will not be added to the dashboard


def update_issues_pr():
    print('\nUpdating issues and pull requests')
    for project in Project.objects.filter(archived=False, main_namespace__from_gepetto=True):
        try:
            gh = project.github()
            main_repo = project.repo_set.filter(namespace=project.main_namespace, forge__source=SOURCES.github).first()

            # Create new issues and pull requests
            for issue in gh.get_issues(state='open'):
Tom Pillot's avatar
Tom Pillot committed
27
                days_since_updated = (timezone.now() - timezone.make_aware(issue.updated_at)).days
28
29
30
31
32
                if main_repo is not None and days_since_updated > MIN_DAYS_SINCE_UPDATED \
                        and SKIP_LABEL not in [label.name for label in issue.get_labels()]:
                    url = re.sub('api\\.github\\.com/repos', 'github.com', issue.url)

                    if issue.pull_request is None:
Guilhem Saurel's avatar
Guilhem Saurel committed
33
                        IssuePr.objects.get_or_create(repo=main_repo,
34
35
                                                      number=issue.number,
                                                      url=url,
Guilhem Saurel's avatar
Guilhem Saurel committed
36
37
                                                      is_issue=True,
                                                      defaults={'title': issue.title})
38
39

                    else:
Guilhem Saurel's avatar
Guilhem Saurel committed
40
                        IssuePr.objects.get_or_create(repo=main_repo,
41
42
                                                      number=issue.number,
                                                      url=url,
Guilhem Saurel's avatar
Guilhem Saurel committed
43
44
                                                      is_issue=False,
                                                      defaults={'title': issue.title})
45
46
47
48
49
50
51
52

        except github.UnknownObjectException:
            print(f'Project not found: {project.main_namespace.slug}/{project.slug}')

    # Update all issues and pull requests, delete closed ones
    for issue_pr in IssuePr.objects.all():
        issue_pr.update(SKIP_LABEL)

Guilhem Saurel's avatar
Guilhem Saurel committed
53
54
55
56
57

class Command(BaseCommand):
    help = 'Update the DB'

    def handle(self, *args, **options):
Guilhem Saurel's avatar
Guilhem Saurel committed
58
        def log(message):
Guilhem Saurel's avatar
Guilhem Saurel committed
59
            self.stdout.write(message)
Guilhem Saurel's avatar
Guilhem Saurel committed
60

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
61
        log('updating forges')
62
        for forge in Forge.objects.order_by('source'):
Guilhem Saurel's avatar
Guilhem Saurel committed
63
            log(f' updating {forge}')
64
65
            forge.get_projects()

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
66
        log('\nUpdating all repos\n')
67
        for repo in Repo.objects.filter(project__archived=False, project__main_namespace__from_gepetto=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
68
            log(f' {repo}')
Guilhem Saurel's avatar
Guilhem Saurel committed
69
70
            repo.update()

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
71
        log('\nUpdating all branches\n')
72
        for branch in Branch.objects.filter(project__archived=False, project__main_namespace__from_gepetto=True):
Guilhem Saurel's avatar
details    
Guilhem Saurel committed
73
            log(f' {branch.project} - {branch}')
Guilhem Saurel's avatar
Guilhem Saurel committed
74
75
            branch.update(pull=False)

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
76
        log('\nPulling Robotpkg\n')
77
        update_robotpkg(settings.RAINBOARD_RPKG)
Guilhem Saurel's avatar
Guilhem Saurel committed
78

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
79
        log('\nUpdating gepetto projects\n')
80
        for project in Project.objects.filter(archived=False, main_namespace__from_gepetto=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
81
            log(f' {project}')
82
            project.update(only_main_branches=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
83

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
84
        log('\nUpdating Robotpkg\n')
85
        for robotpkg in Robotpkg.objects.filter(project__archived=False, project__main_namespace__from_gepetto=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
86
            log(f' {robotpkg}')
Guilhem Saurel's avatar
Guilhem Saurel committed
87
88
            robotpkg.update(pull=False)

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
89
        log('\nUpdating keep doc\n')
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
90
91
92
        Branch.objects.filter(Q(name__endswith='master') | Q(name__endswith='devel'),
                              repo__namespace=F('project__main_namespace'),
                              repo__forge__source=SOURCES.gitlab).update(keep_doc=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
93

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
94
        log('\nDelete perso\n')
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
95
        call_command('delete_perso')
96

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
97
        log('\nclean obsolete Images\n')
Guilhem Saurel's avatar
Guilhem Saurel committed
98
99
        Image.objects.filter(robotpkg__project__archived=True).delete()

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
100
        log('\nLook for missing images\n')
101
        for img in Image.objects.filter(created__lt=timezone.now() - timedelta(days=7), target__active=True):
102
            log(f' {img}')
103
104

        update_issues_pr()