models.py 51.9 KB
Newer Older
1
import json
2
import logging
Guilhem Saurel's avatar
Guilhem Saurel committed
3
import re
Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
4
from subprocess import check_output
5

Guilhem Saurel's avatar
Guilhem Saurel committed
6
7
import git
import requests
Guilhem Saurel's avatar
Guilhem Saurel committed
8
from django.conf import settings
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
9
from django.db import models
Guilhem Saurel's avatar
Guilhem Saurel committed
10
from django.db.models import Q
11
from django.db.utils import DataError
12
from django.template.loader import get_template
13
from django.utils import timezone
Guilhem Saurel's avatar
Guilhem Saurel committed
14
15
from django.utils.dateparse import parse_datetime
from django.utils.safestring import mark_safe
Guilhem Saurel's avatar
Guilhem Saurel committed
16
17

from autoslug import AutoSlugField
18
from autoslug.utils import slugify
Tom Pillot's avatar
Tom Pillot committed
19
20
from github import Github
from gitlab import Gitlab
Guilhem Saurel's avatar
Guilhem Saurel committed
21
from ndh.models import Links, NamedModel, TimeStampedModel
Guilhem Saurel's avatar
Guilhem Saurel committed
22
from ndh.utils import enum_to_choices, query_sum
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
23

Guilhem Saurel's avatar
Guilhem Saurel committed
24
from .utils import SOURCES, api_next, invalid_mail, slugify_with_dots, valid_name
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
25

26
27
logger = logging.getLogger('rainboard.models')

Guilhem Saurel's avatar
Guilhem Saurel committed
28
MAIN_BRANCHES = ['master', 'devel']
Guilhem Saurel's avatar
Guilhem Saurel committed
29
RPKG_URL = 'http://robotpkg.openrobots.org'
Guilhem Saurel's avatar
Guilhem Saurel committed
30
DOC_URL = 'https://gepettoweb.laas.fr/doc'
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
31
32
33
RPKG_LICENSES = {
    'gnu-lgpl-v3': 'LGPL-3.0',
    'gnu-lgpl-v2': 'LGPL-2.0',
Guilhem Saurel's avatar
Guilhem Saurel committed
34
    'gnu-lgpl-v2.1': 'LGPL-2.1',
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
35
36
37
38
39
40
    'mit': 'MIT',
    'gnu-gpl-v3': 'GPL-3.0',
    '2-clause-bsd': 'BSD-2-Clause',
    'eclipse': 'EPL-1.0',
    'modified-bsd': 'BSD-3-Clause'
}
Guilhem Saurel's avatar
Guilhem Saurel committed
41
RPKG_FIELDS = ['PKGBASE', 'PKGVERSION', 'MASTER_SITES', 'MASTER_REPOSITORY', 'MAINTAINER', 'COMMENT', 'HOMEPAGE']
Guilhem Saurel's avatar
Guilhem Saurel committed
42
43
44
45
46
47
48
CMAKE_FIELDS = {
    'NAME': 'cmake_name',
    'DESCRIPTION': 'description',
    'URL': 'homepage',
    'VERSION': 'version',
    'SUFFIX': 'suffix'
}
Guilhem Saurel's avatar
Guilhem Saurel committed
49
TRAVIS_STATE = {'created': None, 'passed': True, 'started': None, 'failed': False, 'errored': False, 'canceled': False}
Guilhem Saurel's avatar
Guilhem Saurel committed
50
GITLAB_STATUS = {'failed': False, 'success': True, 'pending': None, 'skipped': None, 'canceled': None, 'running': None}
Guilhem Saurel's avatar
Guilhem Saurel committed
51

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
52
53

class Namespace(NamedModel):
54
    group = models.BooleanField(default=False)
Tom Pillot's avatar
Tom Pillot committed
55
56
57
58
59
60
61
62
63
    slug_gitlab = models.CharField(max_length=200, default='')
    slug_github = models.CharField(max_length=200, default='')

    def save(self, *args, **kwargs):
        if self.slug_gitlab == '':
            self.slug_gitlab = self.slug
        if self.slug_github == '':
            self.slug_github = self.slug
        super(Namespace, self).save(*args, **kwargs)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
64
65


Guilhem Saurel's avatar
Guilhem Saurel committed
66
67
68
class License(models.Model):
    name = models.CharField(max_length=200)
    spdx_id = models.CharField(max_length=50, unique=True)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
69
70
71
72
73
74
    url = models.URLField(max_length=200)

    def __str__(self):
        return self.spdx_id or self.name


Guilhem Saurel's avatar
Guilhem Saurel committed
75
class Forge(Links, NamedModel):
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
76
77
78
79
80
    source = models.PositiveSmallIntegerField(choices=enum_to_choices(SOURCES))
    url = models.URLField(max_length=200)
    token = models.CharField(max_length=50, blank=True, null=True)
    verify = models.BooleanField(default=True)

Guilhem Saurel's avatar
Guilhem Saurel committed
81
82
83
    def get_absolute_url(self):
        return self.url

Guilhem Saurel's avatar
Guilhem Saurel committed
84
    def api_req(self, url='', name=None, page=1):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
85
        logger.debug(f'requesting api {self} {url}, page {page}')
86
87
88
89
90
        try:
            return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
        except requests.exceptions.ConnectionError:
            logger.error(f'requesting api {self} {url}, page {page} - SECOND TRY')
            return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
Guilhem Saurel's avatar
Guilhem Saurel committed
91
92
93
94
95
96

    def api_data(self, url=''):
        req = self.api_req(url)
        return req.json() if req.status_code == 200 else []  # TODO

    def api_list(self, url='', name=None):
Guilhem Saurel's avatar
Guilhem Saurel committed
97
98
        page = 1
        while page:
Guilhem Saurel's avatar
Guilhem Saurel committed
99
            req = self.api_req(url, name, page)
Guilhem Saurel's avatar
Guilhem Saurel committed
100
101
102
103
104
            if req.status_code != 200:
                return []  # TODO
            data = req.json()
            if name is not None:
                data = data[name]
Guilhem Saurel's avatar
Guilhem Saurel committed
105
            yield from data
Guilhem Saurel's avatar
Guilhem Saurel committed
106
            page = api_next(self.source, req)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
107
108

    def headers(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
109
        return {
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
110
111
112
113
114
115
116
117
118
119
120
121
122
123
            SOURCES.github: {
                'Authorization': f'token {self.token}',
                'Accept': 'application/vnd.github.drax-preview+json'
            },
            SOURCES.gitlab: {
                'Private-Token': self.token
            },
            SOURCES.redmine: {
                'X-Redmine-API-Key': self.token
            },
            SOURCES.travis: {
                'Authorization': f'token {self.token}',
                'TRAVIS-API-Version': '3'
            },
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
124
        }[self.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
125
126

    def api_url(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
127
128
129
130
        return {
            SOURCES.github: 'https://api.github.com',
            SOURCES.gitlab: f'{self.url}/api/v4',
            SOURCES.redmine: self.url,
Guilhem Saurel's avatar
travis    
Guilhem Saurel committed
131
            SOURCES.travis: 'https://api.travis-ci.org',
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
132
        }[self.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
133

134
135
    def get_namespaces_github(self):
        for namespace in Namespace.objects.filter(group=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
136
            for data in self.api_list(f'/orgs/{namespace.slug}/members'):
Guilhem Saurel's avatar
Guilhem Saurel committed
137
138
139
140
141
                Namespace.objects.get_or_create(slug=data['login'].lower(),
                                                defaults={
                                                    'name': data['login'],
                                                    'group': False
                                                })
142

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
143
144
    def get_namespaces_gitlab(self):
        for data in self.api_list('/namespaces'):
145
            Namespace.objects.get_or_create(slug=slugify(data['path']),
Guilhem Saurel's avatar
Guilhem Saurel committed
146
147
148
149
                                            defaults={
                                                'name': data['name'],
                                                'group': data['kind'] == 'group'
                                            })
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
150
        for data in self.api_list('/users'):
151
            Namespace.objects.get_or_create(slug=slugify(data['username']), defaults={'name': data['name']})
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
152

Guilhem Saurel's avatar
Guilhem Saurel committed
153
154
155
    def get_namespaces_redmine(self):
        pass  # TODO

Guilhem Saurel's avatar
Guilhem Saurel committed
156
157
158
    def get_namespaces_travis(self):
        pass

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
159
160
161
    def get_projects(self):
        getattr(self, f'get_namespaces_{self.get_source_display()}')()
        return getattr(self, f'get_projects_{self.get_source_display()}')()
162

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
163
    def get_projects_github(self):
164
        for org in Namespace.objects.filter(group=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
165
            for data in self.api_list(f'/orgs/{org.slug}/repos'):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
166
                update_github(self, org, data)
167
        for user in Namespace.objects.filter(group=False):
Guilhem Saurel's avatar
Guilhem Saurel committed
168
            for data in self.api_list(f'/users/{user.slug}/repos'):
Guilhem Saurel's avatar
Guilhem Saurel committed
169
                if Project.objects.filter(name=valid_name(data['name'])).exists():
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
170
                    update_github(self, user, data)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
171
172

    def get_projects_gitlab(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
173
        for data in self.api_list('/projects'):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
174
            update_gitlab(self, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
175
176

        for orphan in Project.objects.filter(main_namespace=None):
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
177
            repo = orphan.repo_set.filter(forge__source=SOURCES.gitlab).first()
Guilhem Saurel's avatar
Guilhem Saurel committed
178
179
            if repo:
                update_gitlab(self, self.api_data(f'/projects/{repo.forked_from}'))
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
180
181
182
183

    def get_projects_redmine(self):
        pass  # TODO

Guilhem Saurel's avatar
Guilhem Saurel committed
184
185
186
187
188
189
    def get_projects_travis(self):
        for namespace in Namespace.objects.all():
            for repository in self.api_list(f'/owner/{namespace.slug}/repos', 'repositories'):
                if repository['active']:
                    update_travis(namespace, repository)

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
190

Guilhem Saurel's avatar
Guilhem Saurel committed
191
class Project(Links, NamedModel, TimeStampedModel):
Guilhem Saurel's avatar
Guilhem Saurel committed
192
    public = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
193
194
195
196
    main_namespace = models.ForeignKey(Namespace, on_delete=models.SET_NULL, null=True, blank=True)
    main_forge = models.ForeignKey(Forge, on_delete=models.SET_NULL, null=True, blank=True)
    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    homepage = models.URLField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
197
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
198
    version = models.CharField(max_length=20, blank=True, null=True)
199
    updated = models.DateTimeField(blank=True, null=True)
200
201
    tests = models.BooleanField(default=True)
    docs = models.BooleanField(default=True)
202
    from_gepetto = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
203
    cmake_name = models.CharField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
204
    archived = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
205
    suffix = models.CharField(max_length=50, default='', blank=True)
Guilhem Saurel's avatar
oops    
Guilhem Saurel committed
206
    allow_format_failure = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
207
    has_python = models.BooleanField(default=True)
Tom Pillot's avatar
Tom Pillot committed
208
    accept_pr_to_master = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
209

210
211
212
213
    def save(self, *args, **kwargs):
        self.name = valid_name(self.name)
        super().save(*args, **kwargs)

Guilhem Saurel's avatar
Guilhem Saurel committed
214
    def git_path(self):
215
        return settings.RAINBOARD_GITS / self.main_namespace.slug / self.slug.strip()  # workaround SafeText TypeError
Guilhem Saurel's avatar
Guilhem Saurel committed
216

Guilhem Saurel's avatar
Guilhem Saurel committed
217
    def git(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
218
        path = self.git_path()
Guilhem Saurel's avatar
Guilhem Saurel committed
219
220
221
222
223
        if not path.exists():
            logger.info(f'Creating repo for {self.main_namespace.slug}/{self.slug}')
            return git.Repo.init(path)
        return git.Repo(str(path / '.git'))

Tom Pillot's avatar
Tom Pillot committed
224
225
226
227
228
229
230
231
232
233
234
    def github(self):
        github_forge = Forge.objects.get(slug='github')
        gh = Github(github_forge.token)
        return gh.get_repo(f'{self.main_namespace.slug_github}/{self.slug}')

    def gitlab(self):
        gitlab_forge = Forge.objects.get(slug='gitlab')
        gl = Gitlab(gitlab_forge.url, private_token=gitlab_forge.token)
        gl_repo = gl.projects.get(f'{self.main_namespace.slug_gitlab}/{self.slug}')
        return gl_repo

235
    def main_repo(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
236
        forge = self.main_forge if self.main_forge else get_default_forge(self)
Guilhem Saurel's avatar
Guilhem Saurel committed
237
238
239
240
241
242
243
244
        repo, created = Repo.objects.get_or_create(forge=forge,
                                                   namespace=self.main_namespace,
                                                   project=self,
                                                   defaults={
                                                       'name': self.name,
                                                       'default_branch': 'master',
                                                       'repo_id': 0
                                                   })
Guilhem Saurel's avatar
Guilhem Saurel committed
245
        if created:
246
            repo.api_update()
Guilhem Saurel's avatar
Guilhem Saurel committed
247
248
        return repo

Guilhem Saurel's avatar
Guilhem Saurel committed
249
    def update_branches(self, main=True, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
250
251
252
        branches = [b[2:] for b in self.git().git.branch('-a', '--no-color').split('\n')]
        if main:
            branches = [b for b in branches if b.endswith('master') or b.endswith('devel')]
Guilhem Saurel's avatar
Guilhem Saurel committed
253
        for branch in branches:
Guilhem Saurel's avatar
Guilhem Saurel committed
254
            logger.info(f'update branch {branch}')
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
255
256
            if branch.startswith('remotes/'):
                branch = branch[8:]
257
            if branch.count('/') < 2:
Guilhem Saurel's avatar
Guilhem Saurel committed
258
259
                if branch != 'master':
                    logger.error(f'wrong branch "{branch}" in {self.git_path()}')
260
                continue
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
261
            forge, namespace, name = branch.split('/', maxsplit=2)
262
            namespace, _ = Namespace.objects.get_or_create(slug=slugify(namespace), defaults={'name': namespace})
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
263
            forge = Forge.objects.get(slug=forge)
Guilhem Saurel's avatar
Guilhem Saurel committed
264
265
266
267
268
269
270
271
            repo, created = Repo.objects.get_or_create(forge=forge,
                                                       namespace=namespace,
                                                       project=self,
                                                       defaults={
                                                           'name': self.name,
                                                           'default_branch': 'master',
                                                           'repo_id': 0
                                                       })
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
272
273
            if created:
                repo.api_update()
274
275
276
            instance, bcreated = Branch.objects.get_or_create(name=branch, project=self, repo=repo)
            if bcreated:
                instance.update(pull=pull)
277

Guilhem Saurel's avatar
update    
Guilhem Saurel committed
278
279
280
    def checkout(self):
        self.main_branch().git().checkout()

Guilhem Saurel's avatar
Guilhem Saurel committed
281
    def main_branch(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
282
        return self.main_repo().main_branch()
Guilhem Saurel's avatar
Guilhem Saurel committed
283

Guilhem Saurel's avatar
Guilhem Saurel committed
284
285
286
287
288
289
290
    def cmake(self):
        filename = self.git_path() / 'CMakeLists.txt'
        if not filename.exists():
            return
        with filename.open() as f:
            content = f.read()
        for key, value in CMAKE_FIELDS.items():
Guilhem Saurel's avatar
Guilhem Saurel committed
291
            search = re.search(r'set\s*\(\s*project_%s\s+([^)]+)*\)' % key, content, re.I)
Guilhem Saurel's avatar
Guilhem Saurel committed
292
            if search:
293
                try:
294
                    old = getattr(self, value)
295
                    new = search.groups()[0].strip(''' \r\n\t'"''').replace('_', '-')
296
297
298
                    if old != new:
                        setattr(self, value, new)
                        self.save()
299
                except DataError:
300
                    setattr(self, value, old)
301
        for dependency in re.findall(r'ADD_[A-Z]+_DEPENDENCY\s*\(["\']?([^ "\')]+).*["\']?\)', content, re.I):
Guilhem Saurel's avatar
Guilhem Saurel committed
302
            project = Project.objects.filter(name=valid_name(dependency))
303
            if project.exists():
Guilhem Saurel's avatar
oops    
Guilhem Saurel committed
304
305
                dependency, _ = Dependency.objects.get_or_create(project=self, library=project.first())
                if not dependency.cmake:
306
307
                    dependency.cmake = True
                    dependency.save()
Guilhem Saurel's avatar
Guilhem Saurel committed
308

309
    def ros(self):
310
311
        try:
            filename = self.git_path() / 'package.xml'
312
        except TypeError:
313
            return
314
315
316
317
318
        if not filename.exists():
            return
        with filename.open() as f:
            content = f.read()
        for dependency in re.findall(r'<run_depend>(\w+).*</run_depend>', content, re.I):
Guilhem Saurel's avatar
Guilhem Saurel committed
319
            project = Project.objects.filter(name=valid_name(dependency))
320
321
322
323
324
325
            if project.exists():
                dependency, _ = Dependency.objects.get_or_create(project=self, library=project.first())
                if not dependency.ros:
                    dependency.ros = True
                    dependency.save()

326
327
328
329
330
331
    def repos(self):
        return self.repo_set.count()

    def rpkgs(self):
        return self.robotpkg_set.count()

Guilhem Saurel's avatar
Guilhem Saurel committed
332
333
334
335
    def update_tags(self):
        for tag in self.git().tags:
            Tag.objects.get_or_create(name=str(tag), project=self)

Guilhem Saurel's avatar
Guilhem Saurel committed
336
    def update_repo(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
337
338
        branch = str(self.main_branch()).split('/', maxsplit=2)[2]
        self.git().head.commit = self.git().remotes[self.main_repo().git_remote()].refs[branch].commit
Guilhem Saurel's avatar
Guilhem Saurel committed
339

Guilhem Saurel's avatar
cijob    
Guilhem Saurel committed
340
341
342
343
    def ci_jobs(self):
        if self.main_forge.source == SOURCES.gitlab:
            self.main_repo().get_jobs_gitlab()

344
    def update(self, only_main_branches=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
345
346
        if self.main_namespace is None:
            return
347
        self.update_branches(main=only_main_branches)
Guilhem Saurel's avatar
Guilhem Saurel committed
348
        self.update_tags()
Guilhem Saurel's avatar
Guilhem Saurel committed
349
        self.update_repo()
350
        tag = self.tag_set.filter(name__startswith='v').last()  # TODO: implement SQL ordering for semver
Guilhem Saurel's avatar
Guilhem Saurel committed
351
352
        if tag is not None:
            self.version = tag.name[1:]
353
        robotpkg = self.robotpkg_set.order_by('-updated').first()
Guilhem Saurel's avatar
Guilhem Saurel committed
354
        branch = self.branch_set.order_by('-updated').first()
Guilhem Saurel's avatar
Guilhem Saurel committed
355
356
357
        branch_updated = branch is not None and branch.updated is not None
        robotpkg_updated = robotpkg is not None and robotpkg.updated is not None
        if branch_updated or robotpkg_updated:
Guilhem Saurel's avatar
Guilhem Saurel committed
358
            if not robotpkg_updated:
Guilhem Saurel's avatar
Guilhem Saurel committed
359
                self.updated = branch.updated
Guilhem Saurel's avatar
Guilhem Saurel committed
360
            elif not branch_updated:
Guilhem Saurel's avatar
Guilhem Saurel committed
361
362
363
                self.updated = robotpkg.updated
            else:
                self.updated = max(branch.updated, robotpkg.updated)
Guilhem Saurel's avatar
cijob    
Guilhem Saurel committed
364
        self.ci_jobs()
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
365
        self.checkout()
366
        self.cmake()
Guilhem Saurel's avatar
Guilhem Saurel committed
367
        self.ros()
368
369
        self.save()

Guilhem Saurel's avatar
Guilhem Saurel committed
370
    def commits_since(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
371
372
373
374
375
        try:
            commits = self.git().git.rev_list(f'v{self.version}..{self.main_branch()}')
            return len(commits.split('\n')) if commits else 0
        except git.exc.GitCommandError:
            pass
Guilhem Saurel's avatar
Guilhem Saurel committed
376

377
378
379
380
381
382
    def open_issues(self):
        return query_sum(self.repo_set, 'open_issues')

    def open_pr(self):
        return query_sum(self.repo_set, 'open_pr')

383
384
385
    def gitlabciyml(self):
        return get_template('rainboard/gitlab-ci.yml').render({'project': self})

Guilhem Saurel's avatar
Guilhem Saurel committed
386
387
388
389
390
391
392
393
394
    def contributors(self, update=False):
        if update:
            for guy in self.git().git.shortlog('-nse').split('\n'):
                name, mail = guy[7:-1].split(' <')
                contributor = get_contributor(name, mail)
                contributor.projects.add(self)
                contributor.save()
        return self.contributor_set.all()

395
396
397
    def registry(self):
        return settings.PUBLIC_REGISTRY if self.public else settings.PRIVATE_REGISTRY

Guilhem Saurel's avatar
Guilhem Saurel committed
398
    def doc_coverage_image(self):
399
400
        images = Image.objects.filter(robotpkg__project=self, target__main=True)
        return images.order_by('-py3', '-debug').first()  # 18.04 / Python 3 / Debug is the preferred image
Guilhem Saurel's avatar
Guilhem Saurel committed
401

402
    def print_deps(self):
403
        return mark_safe(', '.join(d.library.get_link() for d in self.dependencies.all()))
404
405

    def print_rdeps(self):
406
        return mark_safe(', '.join(d.project.get_link() for d in self.rdeps.all()))
407

Guilhem Saurel's avatar
Guilhem Saurel committed
408
409
410
    def ordered_robotpkg(self):
        return self.robotpkg_set.order_by('name')

Guilhem Saurel's avatar
Guilhem Saurel committed
411
412
413
414
    def url_travis(self):
        return f'https://travis-ci.org/{self.main_namespace.slug}/{self.slug}'

    def url_gitlab(self):
Tom Pillot's avatar
Tom Pillot committed
415
416
417
418
419
420
421
422
423
424
425
426
        return f'https://gitlab.laas.fr/{self.main_namespace.slug_gitlab}/{self.slug}'

    def remote_url_gitlab(self):
        gitlab_forge = Forge.objects.get(source=SOURCES.gitlab)
        return self.url_gitlab().replace('://', f'://gitlab-ci-token:{gitlab_forge.token}@')

    def url_github(self):
        return f'https://github.com/{self.main_namespace.slug_github}/{self.slug}'

    def remote_url_github(self):
        github_forge = Forge.objects.get(source=SOURCES.github)
        return self.url_github().replace('://', f'://{settings.GITHUB_USER}:{github_forge.token}@')
Guilhem Saurel's avatar
Guilhem Saurel committed
427
428
429
430
431
432
433
434
435
436
437
438

    def badge(self, link, img, alt):
        return mark_safe(f'<a href="{link}"><img src="{img}" alt="{alt}" /></a> ')

    def badge_travis(self):
        return self.badge(self.url_travis(), f'{self.url_travis()}.svg?branch=master', 'Building Status')

    def badge_gitlab(self):
        return self.badge(self.url_gitlab(), f'{self.url_gitlab()}/badges/master/pipeline.svg', 'Pipeline Status')

    def badge_coverage(self):
        return self.badge(f'{DOC_URL}/{self.main_namespace.slug}/{self.slug}/master/coverage',
Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
439
                          f'{self.url_gitlab()}/badges/master/coverage.svg?job=doc-coverage"', 'Coverage Report')
Guilhem Saurel's avatar
Guilhem Saurel committed
440
441

    def badges(self):
Guilhem Saurel's avatar
safe ""    
Guilhem Saurel committed
442
        travis = self.badge_travis() if self.public else mark_safe('')
443
        return travis + self.badge_gitlab() + self.badge_coverage()
Guilhem Saurel's avatar
Guilhem Saurel committed
444

Guilhem Saurel's avatar
Guilhem Saurel committed
445
446
447
448
449
450
451
    def cron(self):
        """ generate a cron-style interval description to run CI monthly on master """
        hour, day = (self.pk // 30) % 24, self.pk % 30 + 1
        return f'0 {hour} {day} * *'

    def pipeline_schedules(self):
        """ provides a link to gitlab's CI schedules page showing then cron rule to use with this project """
Guilhem Saurel's avatar
oops    
Guilhem Saurel committed
452
        repo = self.repo_set.filter(forge__source=SOURCES.gitlab, namespace__group=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
453
454
        if repo.exists():
            link = repo.first().url + '/pipeline_schedules'
455
            return mark_safe(f'<a href="{link}">{self.cron()}</a>')
Guilhem Saurel's avatar
Guilhem Saurel committed
456

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
457

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
458
459
class Repo(TimeStampedModel):
    name = models.CharField(max_length=200)
Guilhem Saurel's avatar
Guilhem Saurel committed
460
    slug = AutoSlugField(populate_from='name', slugify=slugify_with_dots)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
461
462
463
464
465
466
467
468
469
470
    forge = models.ForeignKey(Forge, on_delete=models.CASCADE)
    namespace = models.ForeignKey(Namespace, on_delete=models.CASCADE)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    homepage = models.URLField(max_length=200, blank=True, null=True)
    url = models.URLField(max_length=200, blank=True, null=True)
    default_branch = models.CharField(max_length=50)
    open_issues = models.PositiveSmallIntegerField(blank=True, null=True)
    open_pr = models.PositiveSmallIntegerField(blank=True, null=True)
    repo_id = models.PositiveIntegerField()
Guilhem Saurel's avatar
Guilhem Saurel committed
471
    forked_from = models.PositiveIntegerField(blank=True, null=True)
472
    clone_url = models.URLField(max_length=200)
Guilhem Saurel's avatar
travis    
Guilhem Saurel committed
473
    travis_id = models.PositiveIntegerField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
474
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
475
    archived = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
476
477
478

    def __str__(self):
        return self.name
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
479
480

    def api_url(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
481
        api_url = self.forge.api_url()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
482
        return {
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
483
484
485
            SOURCES.github: f'{api_url}/repos/{self.namespace.slug}/{self.slug}',
            SOURCES.redmine: f'{api_url}/projects/{self.repo_id}.json',
            SOURCES.gitlab: f'{api_url}/projects/{self.repo_id}',
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
486
        }[self.forge.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
487

Guilhem Saurel's avatar
Guilhem Saurel committed
488
    def api_req(self, url='', name=None, page=1):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
489
        logger.debug(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}')
490
        try:
Guilhem Saurel's avatar
Guilhem Saurel committed
491
492
493
            return requests.get(self.api_url() + url, {'page': page},
                                verify=self.forge.verify,
                                headers=self.forge.headers())
494
495
        except requests.exceptions.ConnectionError:
            logger.error(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page} - SECOND TRY')
Guilhem Saurel's avatar
Guilhem Saurel committed
496
497
498
            return requests.get(self.api_url() + url, {'page': page},
                                verify=self.forge.verify,
                                headers=self.forge.headers())
Guilhem Saurel's avatar
Guilhem Saurel committed
499
500
501
502
503
504

    def api_data(self, url=''):
        req = self.api_req(url)
        return req.json() if req.status_code == 200 else []  # TODO

    def api_list(self, url='', name=None):
Guilhem Saurel's avatar
Guilhem Saurel committed
505
506
        page = 1
        while page:
Guilhem Saurel's avatar
Guilhem Saurel committed
507
            req = self.api_req(url, name, page)
Guilhem Saurel's avatar
Guilhem Saurel committed
508
509
510
511
            if req.status_code != 200:
                return []  # TODO
            data = req.json()
            if name is not None:
Guilhem Saurel's avatar
Guilhem Saurel committed
512
513
514
515
                if name in data:
                    data = data[name]
                else:
                    return []  # TODO
Guilhem Saurel's avatar
Guilhem Saurel committed
516
            yield from data
Guilhem Saurel's avatar
Guilhem Saurel committed
517
            page = api_next(self.forge.source, req)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
518

Guilhem Saurel's avatar
Guilhem Saurel committed
519
    def api_update(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
520
        data = self.api_data()
Guilhem Saurel's avatar
Guilhem Saurel committed
521
        if data:
Guilhem Saurel's avatar
Guilhem Saurel committed
522
523
524
525
526
527
528
529
            if data['archived']:
                if self.project.main_repo() == self:
                    self.project.archived = True
                    self.project.save()
                self.archived = True
                self.save()
            else:
                return getattr(self, f'api_update_{self.forge.get_source_display()}')(data)
Guilhem Saurel's avatar
Guilhem Saurel committed
530
531

    def api_update_gitlab(self, data):
532
        update_gitlab(self.forge, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
533

Guilhem Saurel's avatar
Guilhem Saurel committed
534
    def api_update_github(self, data):
535
        update_github(self.forge, self.namespace, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
536

Guilhem Saurel's avatar
git    
Guilhem Saurel committed
537
538
539
    def get_clone_url(self):
        if self.forge.source == SOURCES.gitlab:
            return self.clone_url.replace('://', f'://gitlab-ci-token:{self.forge.token}@')
540
541
        if self.forge.source == SOURCES.github:
            return self.clone_url.replace('://', f'://{settings.GITHUB_USER}:{self.forge.token}@')
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
542
543
        return self.clone_url

544
545
546
    def git_remote(self):
        return f'{self.forge.slug}/{self.namespace.slug}'

Guilhem Saurel's avatar
git    
Guilhem Saurel committed
547
    def git(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
548
        git_repo = self.project.git()
549
        remote = self.git_remote()
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
550
        try:
Guilhem Saurel's avatar
Guilhem Saurel committed
551
            return git_repo.remote(remote)
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
552
        except ValueError:
Guilhem Saurel's avatar
Guilhem Saurel committed
553
            logger.info(f'Creating remote {remote}')
Guilhem Saurel's avatar
Guilhem Saurel committed
554
            return git_repo.create_remote(remote, self.get_clone_url())
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
555

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
556
    def fetch(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
557
558
        git_repo = self.git()
        logger.debug(f'fetching {self.forge} / {self.namespace} / {self.project}')
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
559
        try:
Guilhem Saurel's avatar
Guilhem Saurel committed
560
            git_repo.fetch()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
561
        except git.exc.GitCommandError:
Guilhem Saurel's avatar
Guilhem Saurel committed
562
            logger.warning(f'fetching {self.forge} / {self.namespace} / {self.project} - SECOND TRY')
563
564
565
566
567
            try:
                git_repo.fetch()
            except git.exc.GitCommandError:
                return False
        return True
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
568

Guilhem Saurel's avatar
Guilhem Saurel committed
569
570
571
    def main_branch(self):
        return self.project.branch_set.get(name=f'{self.git_remote()}/{self.default_branch}')

Guilhem Saurel's avatar
Guilhem Saurel committed
572
    def ahead(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
573
574
        main_branch = self.main_branch()
        return main_branch.ahead if main_branch is not None else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
575
576

    def behind(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
577
578
        main_branch = self.main_branch()
        return main_branch.behind if main_branch is not None else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
579

Guilhem Saurel's avatar
Guilhem Saurel committed
580
581
582
583
    def get_builds(self):
        return getattr(self, f'get_builds_{self.forge.get_source_display()}')()

    def get_builds_gitlab(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
584
        for pipeline in self.api_list('/pipelines'):
Guilhem Saurel's avatar
Guilhem Saurel committed
585
586
587
            pid, ref = pipeline['id'], pipeline['ref']
            if self.project.tag_set.filter(name=ref).exists():
                continue
Guilhem Saurel's avatar
Guilhem Saurel committed
588
            data = self.api_data(f'/pipelines/{pid}')
Guilhem Saurel's avatar
Guilhem Saurel committed
589
590
591
592
            branch_name = f'{self.forge.slug}/{self.namespace.slug}/{ref}'
            branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
            if created:
                branch.update()
Guilhem Saurel's avatar
Guilhem Saurel committed
593
594
595
596
597
598
599
            ci_build, created = CIBuild.objects.get_or_create(repo=self,
                                                              build_id=pid,
                                                              defaults={
                                                                  'passed': GITLAB_STATUS[pipeline['status']],
                                                                  'started': parse_datetime(data['created_at']),
                                                                  'branch': branch,
                                                              })
Guilhem Saurel's avatar
cijob    
Guilhem Saurel committed
600
601
602
603
604
605
            if not created and ci_build.passed != GITLAB_STATUS[pipeline['status']]:
                ci_build.passed = GITLAB_STATUS[pipeline['status']]
                ci_build.save()

    def get_jobs_gitlab(self):
        for data in self.api_list('/jobs'):
Guilhem Saurel's avatar
oops    
Guilhem Saurel committed
606
            branch_name = f'{self.forge.slug}/{self.namespace.slug}/{data["ref"]}'
Guilhem Saurel's avatar
cijob    
Guilhem Saurel committed
607
608
609
            branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
            if created:
                branch.update()
Guilhem Saurel's avatar
Guilhem Saurel committed
610
611
612
613
614
615
616
            ci_job, created = CIJob.objects.get_or_create(repo=self,
                                                          job_id=data['id'],
                                                          defaults={
                                                              'passed': GITLAB_STATUS[data['status']],
                                                              'started': parse_datetime(data['created_at']),
                                                              'branch': branch,
                                                          })
Guilhem Saurel's avatar
cijob    
Guilhem Saurel committed
617
618
619
620
621
622
623
624
            if not created and ci_job.passed != GITLAB_STATUS[data['status']]:
                ci_job.passed = GITLAB_STATUS[data['status']]
                ci_job.save()
            if self == self.project.main_repo() and data['name'].startswith('robotpkg-'):
                py3 = '-py3' in data['name']
                debug = '-debug' in data['name']
                target = next(target for target in Target.objects.all() if target.name in data['name']).name
                robotpkg = data['name'][9:-(2 + len(target) + (5 if debug else 7) + (3 if py3 else 0))]  # shame.
Guilhem Saurel's avatar
Guilhem Saurel committed
625
626
627
628
                images = Image.objects.filter(robotpkg__name=robotpkg, target__name=target, debug=debug, py3=py3)
                if not images.exists():
                    continue
                image = images.first()
Guilhem Saurel's avatar
cijob    
Guilhem Saurel committed
629
630
631
                if image.allow_failure and GITLAB_STATUS[data['status']]:
                    image.allow_failure = False
                    image.save()
632
                    print('  success', data['web_url'])
Guilhem Saurel's avatar
Guilhem Saurel committed
633
634
635
636

    def get_builds_github(self):
        if self.travis_id is not None:
            travis = Forge.objects.get(source=SOURCES.travis)
Guilhem Saurel's avatar
Guilhem Saurel committed
637
            for build in travis.api_list(f'/repo/{self.travis_id}/builds', name='builds'):
638
                if build['branch'] is None or self.project.tag_set.filter(name=build['branch']['name']).exists():
Guilhem Saurel's avatar
Guilhem Saurel committed
639
640
641
642
                    continue
                branch_name = f'{self.forge.slug}/{self.namespace.slug}/{build["branch"]["name"]}'
                branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
                if created:
Guilhem Saurel's avatar
Guilhem Saurel committed
643
                    branch.update()
Guilhem Saurel's avatar
Guilhem Saurel committed
644
                started = build['started_at'] if build['started_at'] is not None else build['finished_at']
Guilhem Saurel's avatar
Guilhem Saurel committed
645
646
647
648
649
650
651
                CIBuild.objects.get_or_create(repo=self,
                                              build_id=build['id'],
                                              defaults={
                                                  'passed': TRAVIS_STATE[build['state']],
                                                  'started': parse_datetime(started),
                                                  'branch': branch,
                                              })
Guilhem Saurel's avatar
Guilhem Saurel committed
652
653

    def update(self, pull=True):
654
        ok = True
655
656
        if self.project.main_namespace is None:
            return
Guilhem Saurel's avatar
Guilhem Saurel committed
657
658
        self.project.update_tags()
        if pull:
659
660
661
662
663
664
665
            ok = self.fetch()
        if ok:
            self.api_update()
            self.get_builds()
        else:
            logger.error(f'fetching {self.forge} / {self.namespace} / {self.project} - NOT FOUND - DELETING')
            logger.error(str(self.delete()))
Guilhem Saurel's avatar
Guilhem Saurel committed
666

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
667
668
669
670

class Commit(NamedModel, TimeStampedModel):
    project = models.ForeignKey(Project, on_delete=models.CASCADE)

Guilhem Saurel's avatar
Guilhem Saurel committed
671
672
673
    class Meta:
        unique_together = ('project', 'name')

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
674

Guilhem Saurel's avatar
Guilhem Saurel committed
675
676
677
678
679
class Branch(TimeStampedModel):
    name = models.CharField(max_length=200)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    ahead = models.PositiveSmallIntegerField(blank=True, null=True)
    behind = models.PositiveSmallIntegerField(blank=True, null=True)
680
    updated = models.DateTimeField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
681
    repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
682
    deleted = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
683
    keep_doc = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
684
685

    def __str__(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
686
687
688
        return self.name

    class Meta:
Guilhem Saurel's avatar
Guilhem Saurel committed
689
        unique_together = ('project', 'name', 'repo')
Guilhem Saurel's avatar
Guilhem Saurel committed
690
691

    def get_ahead(self, branch='master'):
Guilhem Saurel's avatar
Guilhem Saurel committed
692
693
        commits = self.project.git().git.rev_list(f'{branch}..{self}')
        return len(commits.split('\n')) if commits else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
694
695

    def get_behind(self, branch='master'):
Guilhem Saurel's avatar
Guilhem Saurel committed
696
697
        commits = self.project.git().git.rev_list(f'{self}..{branch}')
        return len(commits.split('\n')) if commits else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
698

699
700
701
    def git(self):
        git_repo = self.project.git()
        if self.name not in git_repo.branches:
Guilhem Saurel's avatar
Guilhem Saurel committed
702
            remote = self.repo.git()
703
704
705
706
707
            _, _, branch = self.name.split('/', maxsplit=2)
            git_repo.create_head(self.name, remote.refs[branch]).set_tracking_branch(remote.refs[branch])
        return git_repo.branches[self.name]

    def update(self, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
708
709
710
711
712
713
714
        if self.deleted:
            return
        try:
            if pull:
                self.repo.fetch()
                if self.repo != self.project.main_repo():
                    self.project.main_repo().fetch()
Guilhem Saurel's avatar
details    
Guilhem Saurel committed
715
716
            try:
                main_branch = self.project.main_branch()
Guilhem Saurel's avatar
Guilhem Saurel committed
717
718
                self.ahead = self.get_ahead(main_branch)
                self.behind = self.get_behind(main_branch)
Guilhem Saurel's avatar
typo    
Guilhem Saurel committed
719
            except Branch.DoesNotExist:
Guilhem Saurel's avatar
details    
Guilhem Saurel committed
720
                pass
Guilhem Saurel's avatar
Guilhem Saurel committed
721
722
723
            self.updated = self.git().commit.authored_datetime
        except (git.exc.GitCommandError, IndexError):
            self.deleted = True
Guilhem Saurel's avatar
Guilhem Saurel committed
724
        self.save()
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
725

Guilhem Saurel's avatar
Guilhem Saurel committed
726
727
728
729
730
731
732
    def ci(self):
        build = self.cibuild_set.last()
        if build is None:
            return ''
        status = {True: '✓', False: '✗', None: '?'}[build.passed]
        return mark_safe(f'<a href="{build.url()}">{status}</a>')

733
    def forge(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
734
        return self.repo.forge
735
736

    def namespace(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
737
        return self.repo.namespace
738

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
739

Guilhem Saurel's avatar
Guilhem Saurel committed
740
class TargetQuerySet(models.QuerySet):
Guilhem Saurel's avatar
Guilhem Saurel committed
741
742
743
    def active(self):
        return self.filter(active=True)

Guilhem Saurel's avatar
Guilhem Saurel committed
744
745
746
    def main(self):
        return self.get(main=True)

Guilhem Saurel's avatar
Guilhem Saurel committed
747

Guilhem Saurel's avatar
Guilhem Saurel committed
748
class Target(NamedModel):
Guilhem Saurel's avatar
Guilhem Saurel committed
749
    active = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
750
    main = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
751

Guilhem Saurel's avatar
Guilhem Saurel committed
752
    objects = TargetQuerySet.as_manager()
Guilhem Saurel's avatar
Guilhem Saurel committed
753
754
755


# class Test(TimeStampedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
756
757
758
759
760
761
762
#     project = models.ForeignKey(Project, on_delete=models.CASCADE)
#     branch = models.ForeignKey(Branch, on_delete=models.CASCADE)
#     commit = models.ForeignKey(Commit, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
#     passed = models.BooleanField(default=False)
#     TODO: travis vs gitlab-ci ?
#     TODO: deploy binary, doc, coverage, lint
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
763

Guilhem Saurel's avatar
Guilhem Saurel committed
764
# class SystemDependency(NamedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
765
766
#     project = models.ForeignKey(Project, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
767
768


Guilhem Saurel's avatar
Guilhem Saurel committed
769
class Robotpkg(NamedModel):
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
770
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
771
    category = models.CharField(max_length=50)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
772

Guilhem Saurel's avatar
Guilhem Saurel committed
773
    pkgbase = models.CharField(max_length=50, default='')
774
    pkgversion = models.CharField(max_length=50, default='')
Guilhem Saurel's avatar
Guilhem Saurel committed
775
776
777
778
    master_sites = models.CharField(max_length=200, default='')
    master_repository = models.CharField(max_length=200, default='')
    maintainer = models.CharField(max_length=200, default='')
    comment = models.TextField()
779
    homepage = models.URLField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
780
781

    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
782
    public = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
783
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
784
785
    updated = models.DateTimeField(blank=True, null=True)

Guilhem Saurel's avatar
same_py    
Guilhem Saurel committed
786
787
    same_py = models.BooleanField(default=True)

Guilhem Saurel's avatar
Guilhem Saurel committed
788
789
790
791
792
793
794
795
    def main_page(self):
        if self.category != 'wip':
            return f'{RPKG_URL}/robotpkg/{self.category}/{self.name}'

    def build_page(self):
        path = '-wip/wip' if self.category == 'wip' else f'/{self.category}'
        return f'{RPKG_URL}/rbulk/robotpkg{path}/{self.name}'

796
    def update_images(self):
797
        py3s = [False, True] if self.name.startswith('py-') else [False]
Guilhem Saurel's avatar
Guilhem Saurel committed
798
        debugs = [False, True]
Guilhem Saurel's avatar
Guilhem Saurel committed
799
        for target in Target.objects.active():
800
801
802
            for py3 in py3s:
                for debug in debugs:
                    Image.objects.get_or_create(robotpkg=self, target=target, py3=py3, debug=debug)[0].update()
Guilhem Saurel's avatar
Guilhem Saurel committed
803

804
    def update(self, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
805
806
807
808
809
810
        path = settings.RAINBOARD_RPKG
        repo = git.Repo(str(path / 'wip' / '.git' if self.category == 'wip' else path / '.git'))
        if pull:
            repo.remotes.origin.pull()

        cwd = path / self.category / self.name
Guilhem Saurel's avatar
Guilhem Saurel committed
811
812
813
        if not cwd.is_dir():
            logger.warning(f'deleted {self}: {self.delete()}')
            return
Guilhem Saurel's avatar
Guilhem Saurel committed
814
815
816
817
818
819
820
821
822
823
824
825
826
        for field in RPKG_FIELDS:
            cmd = ['make', 'show-var', f'VARNAME={field}']
            self.__dict__[field.lower()] = check_output(cmd, cwd=cwd).decode().strip()

        repo_path = self.name if self.category == 'wip' else f'{self.category}/{self.name}'
        last_commit = next(repo.iter_commits(paths=repo_path, max_count=1))
        self.updated = last_commit.authored_datetime

        license = check_output(['make', 'show-var', f'VARNAME=LICENSE'], cwd=cwd).decode().strip()
        if license in RPKG_LICENSES:
            self.license = License.objects.get(spdx_id=RPKG_LICENSES[license])
        else:
            logger.warning(f'Unknown robotpkg license: {license}')
Guilhem Saurel's avatar
Guilhem Saurel committed
827
        self.public = not bool(check_output(['make', 'show-var', f'VARNAME=RESTRICTED'], cwd=cwd).decode().strip())
Guilhem Saurel's avatar
Guilhem Saurel committed
828
829
830
        with (cwd / 'DESCR').open() as f:
            self.description = f.read().strip()

831
        self.update_images()
Guilhem Saurel's avatar
Guilhem Saurel committed
832
        self.save()
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
833

834
    def valid_images(self):
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
        def is_valid(_base_image, _image):
            """The image is valid if it has less than one different parameter from the base image."""
            if _image == self.project.doc_coverage_image():
                #  There is no need to keep this image because it is already tested by doc-coverage
                return False

            base_param = (_base_image.target.name, _base_image.py3, _base_image.debug)
            param = (_image.target.name, _image.py3, _image.debug)

            diff = 0  # Number of different parameters between the base image and the current image
            for i in range(len(base_param)):
                if base_param[i] != param[i]:
                    diff += 1
            return diff <= 1

        # 18.04 / Python 3 / Release is the preferred base image
        base_images = Image.objects.filter(robotpkg__project=self.project, target__main=True)
        base_image = base_images.order_by('-py3', 'debug').first()

        images = self.image_set.filter(created__isnull=False, target__active=True).order_by('target__name')
        return (image for image in images if is_valid(base_image, image))
856

857
    def without_py(self):
Guilhem Saurel's avatar
same_py    
Guilhem Saurel committed
858
        if 'py-' in self.name and self.same_py:
859
860
            return Robotpkg.objects.filter(name=self.name.replace('py-', '')).first()

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
861

Guilhem Saurel's avatar
Guilhem Saurel committed
862
# class RobotpkgBuild(TimeStampedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
863
864
865
#     robotpkg = models.ForeignKey(Robotpkg, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
#     passed = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
866
867


868
869
class Image(models.Model):
    robotpkg = models.ForeignKey(Robotpkg, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
870
    target = models.ForeignKey(Target, on_delete=models.CASCADE)
871
872
    created = models.DateTimeField(blank=True, null=True)
    image = models.CharField(max_length=12, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
873
    py3 = models.BooleanField(default=False)
874
    debug = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
875
    allow_failure = models.BooleanField(default=False)
876
877

    class Meta:
878
        unique_together = ('robotpkg', 'target', 'py3', 'debug')
879
880

    def __str__(self):
881
882
883
884
885
886
        if self.py3:
            py = '-py3'
        elif self.robotpkg.name.startswith('py-'):
            py = '-py2'
        else:
            py = ''
Guilhem Saurel's avatar
Guilhem Saurel committed
887
        return f'{self.robotpkg}{py}:{self.target}'
888
889

    def get_build_args(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
890
891
892
        ret = {
            'TARGET': self.target,
            'ROBOTPKG': self.robotpkg,
Guilhem Saurel's avatar
typo    
Guilhem Saurel committed
893
            'CATEGORY': self.robotpkg.category,
Guilhem Saurel's avatar
Guilhem Saurel committed
894
895
            'REGISTRY': self.robotpkg.project.registry(),
        }
Guilhem Saurel's avatar
Guilhem Saurel committed
896
        if not self.robotpkg.project.public:
Guilhem Saurel's avatar
Guilhem Saurel committed
897
898
899
            ret['IMAGE'] = 'robotpkg-jrl-py3' if self.py3 else 'robotpkg-jrl'
        elif self.py3:
            ret['IMAGE'] = 'robotpkg-py3'
Guilhem Saurel's avatar
Guilhem Saurel committed
900
        return ret
901
902
903

    def get_image_name(self):
        project = self.robotpkg.project
904
        return f'{project.registry()}/{project.main_namespace.slug}/{project.slug}/{self}'.lower()
905

906
907
908
    def get_image_url(self):
        project = self.robotpkg.project
        manifest = str(self).replace(':', '/manifests/')
909
        return f'https://{project.registry()}/v2/{project.main_namespace.slug}/{project.slug}/{manifest}'
910

Guilhem Saurel's avatar
Guilhem Saurel committed
911
    def get_job_name(self):
912
913
        mode = 'debug' if self.debug else 'release'
        return f'robotpkg-{self}-{mode}'.replace(':', '-')
Guilhem Saurel's avatar
Guilhem Saurel committed
914

915
916
917
    def build(self):
        args = self.get_build_args()
        build_args = sum((['--build-arg', f'{key}={value}'] for key, value in args.items()), list())
918
919
920
921
922
923
924
        return ['docker', 'build', '-t', self.get_image_name()] + build_args + ['.']

    def pull(self):
        return ['docker', 'pull', self.get_image_name()]

    def push(self):
        return ['docker', 'push', self.get_image_name()]
925