models.py 48.8 KB
Newer Older
1
import json
2
import logging
Guilhem Saurel's avatar
Guilhem Saurel committed
3
import re
Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
4
from subprocess import check_output
5

Guilhem Saurel's avatar
Guilhem Saurel committed
6
from django.conf import settings
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
7
from django.db import models
Guilhem Saurel's avatar
Guilhem Saurel committed
8
from django.db.models import Q
Guilhem Saurel's avatar
Guilhem Saurel committed
9
from django.db.models.functions import Length
10
from django.db.utils import DataError
11
from django.template.loader import get_template
12
from django.utils import timezone
Guilhem Saurel's avatar
Guilhem Saurel committed
13
14
from django.utils.dateparse import parse_datetime
from django.utils.safestring import mark_safe
Guilhem Saurel's avatar
Guilhem Saurel committed
15

Guilhem Saurel's avatar
Guilhem Saurel committed
16
import requests
17
18

import git
Guilhem Saurel's avatar
Guilhem Saurel committed
19
from autoslug import AutoSlugField
20
from autoslug.utils import slugify
Guilhem Saurel's avatar
Guilhem Saurel committed
21
from ndh.models import Links, NamedModel, TimeStampedModel
Guilhem Saurel's avatar
Guilhem Saurel committed
22
from ndh.utils import enum_to_choices, query_sum
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
23

Guilhem Saurel's avatar
Guilhem Saurel committed
24
from .utils import SOURCES, api_next, invalid_mail, slugify_with_dots, valid_name
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
25

26
27
logger = logging.getLogger('rainboard.models')

Guilhem Saurel's avatar
Guilhem Saurel committed
28
MAIN_BRANCHES = ['master', 'devel']
Guilhem Saurel's avatar
Guilhem Saurel committed
29
RPKG_URL = 'http://robotpkg.openrobots.org'
Guilhem Saurel's avatar
Guilhem Saurel committed
30
DOC_URL = 'http://projects.laas.fr/gepetto/doc'
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
31
32
33
RPKG_LICENSES = {
    'gnu-lgpl-v3': 'LGPL-3.0',
    'gnu-lgpl-v2': 'LGPL-2.0',
Guilhem Saurel's avatar
Guilhem Saurel committed
34
    'gnu-lgpl-v2.1': 'LGPL-2.1',
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
35
36
37
38
39
40
    'mit': 'MIT',
    'gnu-gpl-v3': 'GPL-3.0',
    '2-clause-bsd': 'BSD-2-Clause',
    'eclipse': 'EPL-1.0',
    'modified-bsd': 'BSD-3-Clause'
}
Guilhem Saurel's avatar
Guilhem Saurel committed
41
RPKG_FIELDS = ['PKGBASE', 'PKGVERSION', 'MASTER_SITES', 'MASTER_REPOSITORY', 'MAINTAINER', 'COMMENT', 'HOMEPAGE']
Guilhem Saurel's avatar
Guilhem Saurel committed
42
43
44
45
46
47
48
CMAKE_FIELDS = {
    'NAME': 'cmake_name',
    'DESCRIPTION': 'description',
    'URL': 'homepage',
    'VERSION': 'version',
    'SUFFIX': 'suffix'
}
Guilhem Saurel's avatar
Guilhem Saurel committed
49
TRAVIS_STATE = {'created': None, 'passed': True, 'started': None, 'failed': False, 'errored': False, 'canceled': False}
Guilhem Saurel's avatar
Guilhem Saurel committed
50
GITLAB_STATUS = {'failed': False, 'success': True, 'pending': None, 'skipped': None, 'canceled': None, 'running': None}
Guilhem Saurel's avatar
Guilhem Saurel committed
51

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
52
53

class Namespace(NamedModel):
54
    group = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
55
56


Guilhem Saurel's avatar
Guilhem Saurel committed
57
58
59
class License(models.Model):
    name = models.CharField(max_length=200)
    spdx_id = models.CharField(max_length=50, unique=True)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
60
61
62
63
64
65
    url = models.URLField(max_length=200)

    def __str__(self):
        return self.spdx_id or self.name


Guilhem Saurel's avatar
Guilhem Saurel committed
66
class Forge(Links, NamedModel):
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
67
68
69
70
71
    source = models.PositiveSmallIntegerField(choices=enum_to_choices(SOURCES))
    url = models.URLField(max_length=200)
    token = models.CharField(max_length=50, blank=True, null=True)
    verify = models.BooleanField(default=True)

Guilhem Saurel's avatar
Guilhem Saurel committed
72
73
74
    def get_absolute_url(self):
        return self.url

Guilhem Saurel's avatar
Guilhem Saurel committed
75
    def api_req(self, url='', name=None, page=1):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
76
        logger.debug(f'requesting api {self} {url}, page {page}')
77
78
79
80
81
        try:
            return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
        except requests.exceptions.ConnectionError:
            logger.error(f'requesting api {self} {url}, page {page} - SECOND TRY')
            return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
Guilhem Saurel's avatar
Guilhem Saurel committed
82
83
84
85
86
87

    def api_data(self, url=''):
        req = self.api_req(url)
        return req.json() if req.status_code == 200 else []  # TODO

    def api_list(self, url='', name=None):
Guilhem Saurel's avatar
Guilhem Saurel committed
88
89
        page = 1
        while page:
Guilhem Saurel's avatar
Guilhem Saurel committed
90
            req = self.api_req(url, name, page)
Guilhem Saurel's avatar
Guilhem Saurel committed
91
92
93
94
95
            if req.status_code != 200:
                return []  # TODO
            data = req.json()
            if name is not None:
                data = data[name]
Guilhem Saurel's avatar
Guilhem Saurel committed
96
            yield from data
Guilhem Saurel's avatar
Guilhem Saurel committed
97
            page = api_next(self.source, req)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
98
99

    def headers(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
100
        return {
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
101
102
103
104
105
106
107
108
109
110
111
112
113
114
            SOURCES.github: {
                'Authorization': f'token {self.token}',
                'Accept': 'application/vnd.github.drax-preview+json'
            },
            SOURCES.gitlab: {
                'Private-Token': self.token
            },
            SOURCES.redmine: {
                'X-Redmine-API-Key': self.token
            },
            SOURCES.travis: {
                'Authorization': f'token {self.token}',
                'TRAVIS-API-Version': '3'
            },
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
115
        }[self.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
116
117

    def api_url(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
118
119
120
121
        return {
            SOURCES.github: 'https://api.github.com',
            SOURCES.gitlab: f'{self.url}/api/v4',
            SOURCES.redmine: self.url,
Guilhem Saurel's avatar
travis    
Guilhem Saurel committed
122
            SOURCES.travis: 'https://api.travis-ci.org',
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
123
        }[self.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
124

125
126
    def get_namespaces_github(self):
        for namespace in Namespace.objects.filter(group=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
127
            for data in self.api_list(f'/orgs/{namespace.slug}/members'):
Guilhem Saurel's avatar
Guilhem Saurel committed
128
129
130
131
132
                Namespace.objects.get_or_create(slug=data['login'].lower(),
                                                defaults={
                                                    'name': data['login'],
                                                    'group': False
                                                })
133

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
134
135
    def get_namespaces_gitlab(self):
        for data in self.api_list('/namespaces'):
136
            Namespace.objects.get_or_create(slug=slugify(data['path']),
Guilhem Saurel's avatar
Guilhem Saurel committed
137
138
139
140
                                            defaults={
                                                'name': data['name'],
                                                'group': data['kind'] == 'group'
                                            })
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
141
        for data in self.api_list('/users'):
142
            Namespace.objects.get_or_create(slug=slugify(data['username']), defaults={'name': data['name']})
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
143

Guilhem Saurel's avatar
Guilhem Saurel committed
144
145
146
    def get_namespaces_redmine(self):
        pass  # TODO

Guilhem Saurel's avatar
Guilhem Saurel committed
147
148
149
    def get_namespaces_travis(self):
        pass

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
150
151
152
    def get_projects(self):
        getattr(self, f'get_namespaces_{self.get_source_display()}')()
        return getattr(self, f'get_projects_{self.get_source_display()}')()
153

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
154
    def get_projects_github(self):
155
        for org in Namespace.objects.filter(group=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
156
            for data in self.api_list(f'/orgs/{org.slug}/repos'):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
157
                update_github(self, org, data)
158
        for user in Namespace.objects.filter(group=False):
Guilhem Saurel's avatar
Guilhem Saurel committed
159
            for data in self.api_list(f'/users/{user.slug}/repos'):
Guilhem Saurel's avatar
Guilhem Saurel committed
160
                if Project.objects.filter(name=valid_name(data['name'])).exists():
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
161
                    update_github(self, user, data)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
162
163

    def get_projects_gitlab(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
164
        for data in self.api_list('/projects'):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
165
            update_gitlab(self, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
166
167

        for orphan in Project.objects.filter(main_namespace=None):
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
168
            repo = orphan.repo_set.filter(forge__source=SOURCES.gitlab).first()
Guilhem Saurel's avatar
Guilhem Saurel committed
169
170
            if repo:
                update_gitlab(self, self.api_data(f'/projects/{repo.forked_from}'))
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
171
172
173
174

    def get_projects_redmine(self):
        pass  # TODO

Guilhem Saurel's avatar
Guilhem Saurel committed
175
176
177
178
179
180
    def get_projects_travis(self):
        for namespace in Namespace.objects.all():
            for repository in self.api_list(f'/owner/{namespace.slug}/repos', 'repositories'):
                if repository['active']:
                    update_travis(namespace, repository)

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
181

Guilhem Saurel's avatar
Guilhem Saurel committed
182
class Project(Links, NamedModel, TimeStampedModel):
Guilhem Saurel's avatar
Guilhem Saurel committed
183
    public = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
184
185
186
187
    main_namespace = models.ForeignKey(Namespace, on_delete=models.SET_NULL, null=True, blank=True)
    main_forge = models.ForeignKey(Forge, on_delete=models.SET_NULL, null=True, blank=True)
    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    homepage = models.URLField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
188
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
189
    version = models.CharField(max_length=20, blank=True, null=True)
190
    updated = models.DateTimeField(blank=True, null=True)
191
192
    tests = models.BooleanField(default=True)
    docs = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
193
    from_gepetto = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
194
    cmake_name = models.CharField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
195
    archived = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
196
    suffix = models.CharField(max_length=50, default='', blank=True)
Guilhem Saurel's avatar
oops    
Guilhem Saurel committed
197
    allow_format_failure = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
198
    has_python = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
199

200
201
202
203
    def save(self, *args, **kwargs):
        self.name = valid_name(self.name)
        super().save(*args, **kwargs)

Guilhem Saurel's avatar
Guilhem Saurel committed
204
    def git_path(self):
205
        return settings.RAINBOARD_GITS / self.main_namespace.slug / self.slug.strip()  # workaround SafeText TypeError
Guilhem Saurel's avatar
Guilhem Saurel committed
206

Guilhem Saurel's avatar
Guilhem Saurel committed
207
    def git(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
208
        path = self.git_path()
Guilhem Saurel's avatar
Guilhem Saurel committed
209
210
211
212
213
        if not path.exists():
            logger.info(f'Creating repo for {self.main_namespace.slug}/{self.slug}')
            return git.Repo.init(path)
        return git.Repo(str(path / '.git'))

214
    def main_repo(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
215
        forge = self.main_forge if self.main_forge else get_default_forge(self)
Guilhem Saurel's avatar
Guilhem Saurel committed
216
217
218
219
220
221
222
223
        repo, created = Repo.objects.get_or_create(forge=forge,
                                                   namespace=self.main_namespace,
                                                   project=self,
                                                   defaults={
                                                       'name': self.name,
                                                       'default_branch': 'master',
                                                       'repo_id': 0
                                                   })
Guilhem Saurel's avatar
Guilhem Saurel committed
224
        if created:
225
            repo.api_update()
Guilhem Saurel's avatar
Guilhem Saurel committed
226
227
        return repo

Guilhem Saurel's avatar
Guilhem Saurel committed
228
    def update_branches(self, main=True, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
229
230
231
        branches = [b[2:] for b in self.git().git.branch('-a', '--no-color').split('\n')]
        if main:
            branches = [b for b in branches if b.endswith('master') or b.endswith('devel')]
Guilhem Saurel's avatar
Guilhem Saurel committed
232
        for branch in branches:
Guilhem Saurel's avatar
Guilhem Saurel committed
233
            logger.info(f'update branch {branch}')
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
234
235
            if branch.startswith('remotes/'):
                branch = branch[8:]
236
            if branch.count('/') < 2:
Guilhem Saurel's avatar
Guilhem Saurel committed
237
238
                if branch != 'master':
                    logger.error(f'wrong branch "{branch}" in {self.git_path()}')
239
                continue
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
240
241
242
            forge, namespace, name = branch.split('/', maxsplit=2)
            namespace, _ = Namespace.objects.get_or_create(slug=namespace)
            forge = Forge.objects.get(slug=forge)
Guilhem Saurel's avatar
Guilhem Saurel committed
243
244
245
246
247
248
249
250
            repo, created = Repo.objects.get_or_create(forge=forge,
                                                       namespace=namespace,
                                                       project=self,
                                                       defaults={
                                                           'name': self.name,
                                                           'default_branch': 'master',
                                                           'repo_id': 0
                                                       })
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
251
252
            if created:
                repo.api_update()
253
254
255
            instance, bcreated = Branch.objects.get_or_create(name=branch, project=self, repo=repo)
            if bcreated:
                instance.update(pull=pull)
256

Guilhem Saurel's avatar
update    
Guilhem Saurel committed
257
258
259
    def checkout(self):
        self.main_branch().git().checkout()

Guilhem Saurel's avatar
Guilhem Saurel committed
260
    def main_branch(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
261
        return self.main_repo().main_branch()
Guilhem Saurel's avatar
Guilhem Saurel committed
262

Guilhem Saurel's avatar
Guilhem Saurel committed
263
264
265
266
267
268
269
    def cmake(self):
        filename = self.git_path() / 'CMakeLists.txt'
        if not filename.exists():
            return
        with filename.open() as f:
            content = f.read()
        for key, value in CMAKE_FIELDS.items():
Guilhem Saurel's avatar
Guilhem Saurel committed
270
            search = re.search(r'set\s*\(\s*project_%s\s+([^)]+)*\)' % key, content, re.I)
Guilhem Saurel's avatar
Guilhem Saurel committed
271
            if search:
272
                try:
273
                    old = getattr(self, value)
274
                    new = search.groups()[0].strip(''' \r\n\t'"''').replace('_', '-')
275
276
277
                    if old != new:
                        setattr(self, value, new)
                        self.save()
278
                except DataError:
279
                    setattr(self, value, old)
280
        for dependency in re.findall(r'ADD_[A-Z]+_DEPENDENCY\s*\(["\']?([^ "\')]+).*["\']?\)', content, re.I):
Guilhem Saurel's avatar
Guilhem Saurel committed
281
            project = Project.objects.filter(name=valid_name(dependency))
282
            if project.exists():
Guilhem Saurel's avatar
oops    
Guilhem Saurel committed
283
284
                dependency, _ = Dependency.objects.get_or_create(project=self, library=project.first())
                if not dependency.cmake:
285
286
                    dependency.cmake = True
                    dependency.save()
Guilhem Saurel's avatar
Guilhem Saurel committed
287

288
    def ros(self):
289
290
        try:
            filename = self.git_path() / 'package.xml'
291
        except TypeError:
292
            return
293
294
295
296
297
        if not filename.exists():
            return
        with filename.open() as f:
            content = f.read()
        for dependency in re.findall(r'<run_depend>(\w+).*</run_depend>', content, re.I):
Guilhem Saurel's avatar
Guilhem Saurel committed
298
            project = Project.objects.filter(name=valid_name(dependency))
299
300
301
302
303
304
            if project.exists():
                dependency, _ = Dependency.objects.get_or_create(project=self, library=project.first())
                if not dependency.ros:
                    dependency.ros = True
                    dependency.save()

305
306
307
308
309
310
    def repos(self):
        return self.repo_set.count()

    def rpkgs(self):
        return self.robotpkg_set.count()

Guilhem Saurel's avatar
Guilhem Saurel committed
311
312
313
314
    def update_tags(self):
        for tag in self.git().tags:
            Tag.objects.get_or_create(name=str(tag), project=self)

Guilhem Saurel's avatar
Guilhem Saurel committed
315
    def update_repo(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
316
317
        branch = str(self.main_branch()).split('/', maxsplit=2)[2]
        self.git().head.commit = self.git().remotes[self.main_repo().git_remote()].refs[branch].commit
Guilhem Saurel's avatar
Guilhem Saurel committed
318

Guilhem Saurel's avatar
cijob    
Guilhem Saurel committed
319
320
321
322
    def ci_jobs(self):
        if self.main_forge.source == SOURCES.gitlab:
            self.main_repo().get_jobs_gitlab()

323
    def update(self, only_main_branches=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
324
325
        if self.main_namespace is None:
            return
326
        self.update_branches(main=only_main_branches)
Guilhem Saurel's avatar
Guilhem Saurel committed
327
        self.update_tags()
Guilhem Saurel's avatar
Guilhem Saurel committed
328
        self.update_repo()
329
        tag = self.tag_set.filter(name__startswith='v').last()  # TODO: implement SQL ordering for semver
Guilhem Saurel's avatar
Guilhem Saurel committed
330
331
        if tag is not None:
            self.version = tag.name[1:]
332
        robotpkg = self.robotpkg_set.order_by('-updated').first()
Guilhem Saurel's avatar
Guilhem Saurel committed
333
        branch = self.branch_set.order_by('-updated').first()
Guilhem Saurel's avatar
Guilhem Saurel committed
334
335
336
        branch_updated = branch is not None and branch.updated is not None
        robotpkg_updated = robotpkg is not None and robotpkg.updated is not None
        if branch_updated or robotpkg_updated:
Guilhem Saurel's avatar
Guilhem Saurel committed
337
            if not robotpkg_updated:
Guilhem Saurel's avatar
Guilhem Saurel committed
338
                self.updated = branch.updated
Guilhem Saurel's avatar
Guilhem Saurel committed
339
            elif not branch_updated:
Guilhem Saurel's avatar
Guilhem Saurel committed
340
341
342
                self.updated = robotpkg.updated
            else:
                self.updated = max(branch.updated, robotpkg.updated)
Guilhem Saurel's avatar
cijob    
Guilhem Saurel committed
343
        self.ci_jobs()
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
344
        self.checkout()
345
        self.cmake()
Guilhem Saurel's avatar
Guilhem Saurel committed
346
        self.ros()
347
348
        self.save()

Guilhem Saurel's avatar
Guilhem Saurel committed
349
    def commits_since(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
350
351
352
353
354
        try:
            commits = self.git().git.rev_list(f'v{self.version}..{self.main_branch()}')
            return len(commits.split('\n')) if commits else 0
        except git.exc.GitCommandError:
            pass
Guilhem Saurel's avatar
Guilhem Saurel committed
355

356
357
358
359
360
361
    def open_issues(self):
        return query_sum(self.repo_set, 'open_issues')

    def open_pr(self):
        return query_sum(self.repo_set, 'open_pr')

362
363
364
    def gitlabciyml(self):
        return get_template('rainboard/gitlab-ci.yml').render({'project': self})

Guilhem Saurel's avatar
Guilhem Saurel committed
365
366
367
368
369
370
371
372
373
    def contributors(self, update=False):
        if update:
            for guy in self.git().git.shortlog('-nse').split('\n'):
                name, mail = guy[7:-1].split(' <')
                contributor = get_contributor(name, mail)
                contributor.projects.add(self)
                contributor.save()
        return self.contributor_set.all()

374
375
376
    def registry(self):
        return settings.PUBLIC_REGISTRY if self.public else settings.PRIVATE_REGISTRY

Guilhem Saurel's avatar
Guilhem Saurel committed
377
    def doc_coverage_image(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
378
        images = Image.objects.filter(robotpkg__project=self, py3=False, target__main=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
379
380
        return images.order_by(Length('robotpkg__name').desc()).first()

381
    def print_deps(self):
382
        return mark_safe(', '.join(d.library.get_link() for d in self.dependencies.all()))
383
384

    def print_rdeps(self):
385
        return mark_safe(', '.join(d.project.get_link() for d in self.rdeps.all()))
386

Guilhem Saurel's avatar
Guilhem Saurel committed
387
388
389
    def ordered_robotpkg(self):
        return self.robotpkg_set.order_by('name')

Guilhem Saurel's avatar
Guilhem Saurel committed
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
    def url_travis(self):
        return f'https://travis-ci.org/{self.main_namespace.slug}/{self.slug}'

    def url_gitlab(self):
        return f'https://gepgitlab.laas.fr/{self.main_namespace.slug}/{self.slug}'

    def badge(self, link, img, alt):
        return mark_safe(f'<a href="{link}"><img src="{img}" alt="{alt}" /></a> ')

    def badge_travis(self):
        return self.badge(self.url_travis(), f'{self.url_travis()}.svg?branch=master', 'Building Status')

    def badge_gitlab(self):
        return self.badge(self.url_gitlab(), f'{self.url_gitlab()}/badges/master/pipeline.svg', 'Pipeline Status')

    def badge_coverage(self):
        return self.badge(f'{DOC_URL}/{self.main_namespace.slug}/{self.slug}/master/coverage',
Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
407
                          f'{self.url_gitlab()}/badges/master/coverage.svg?job=doc-coverage"', 'Coverage Report')
Guilhem Saurel's avatar
Guilhem Saurel committed
408
409

    def badges(self):
Guilhem Saurel's avatar
safe ""    
Guilhem Saurel committed
410
        travis = self.badge_travis() if self.public else mark_safe('')
411
        return travis + self.badge_gitlab() + self.badge_coverage()
Guilhem Saurel's avatar
Guilhem Saurel committed
412

Guilhem Saurel's avatar
Guilhem Saurel committed
413
414
415
416
417
418
419
    def cron(self):
        """ generate a cron-style interval description to run CI monthly on master """
        hour, day = (self.pk // 30) % 24, self.pk % 30 + 1
        return f'0 {hour} {day} * *'

    def pipeline_schedules(self):
        """ provides a link to gitlab's CI schedules page showing then cron rule to use with this project """
Guilhem Saurel's avatar
oops    
Guilhem Saurel committed
420
        repo = self.repo_set.filter(forge__source=SOURCES.gitlab, namespace__group=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
421
422
        if repo.exists():
            link = repo.first().url + '/pipeline_schedules'
423
            return mark_safe(f'<a href="{link}">{self.cron()}</a>')
Guilhem Saurel's avatar
Guilhem Saurel committed
424

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
425

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
426
427
class Repo(TimeStampedModel):
    name = models.CharField(max_length=200)
Guilhem Saurel's avatar
Guilhem Saurel committed
428
    slug = AutoSlugField(populate_from='name', slugify=slugify_with_dots)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
429
430
431
432
433
434
435
436
437
438
    forge = models.ForeignKey(Forge, on_delete=models.CASCADE)
    namespace = models.ForeignKey(Namespace, on_delete=models.CASCADE)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    homepage = models.URLField(max_length=200, blank=True, null=True)
    url = models.URLField(max_length=200, blank=True, null=True)
    default_branch = models.CharField(max_length=50)
    open_issues = models.PositiveSmallIntegerField(blank=True, null=True)
    open_pr = models.PositiveSmallIntegerField(blank=True, null=True)
    repo_id = models.PositiveIntegerField()
Guilhem Saurel's avatar
Guilhem Saurel committed
439
    forked_from = models.PositiveIntegerField(blank=True, null=True)
440
    clone_url = models.URLField(max_length=200)
Guilhem Saurel's avatar
travis    
Guilhem Saurel committed
441
    travis_id = models.PositiveIntegerField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
442
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
443
    archived = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
444
445
446

    def __str__(self):
        return self.name
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
447
448

    def api_url(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
449
        api_url = self.forge.api_url()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
450
        return {
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
451
452
453
            SOURCES.github: f'{api_url}/repos/{self.namespace.slug}/{self.slug}',
            SOURCES.redmine: f'{api_url}/projects/{self.repo_id}.json',
            SOURCES.gitlab: f'{api_url}/projects/{self.repo_id}',
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
454
        }[self.forge.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
455

Guilhem Saurel's avatar
Guilhem Saurel committed
456
    def api_req(self, url='', name=None, page=1):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
457
        logger.debug(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}')
458
        try:
Guilhem Saurel's avatar
Guilhem Saurel committed
459
460
461
            return requests.get(self.api_url() + url, {'page': page},
                                verify=self.forge.verify,
                                headers=self.forge.headers())
462
463
        except requests.exceptions.ConnectionError:
            logger.error(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page} - SECOND TRY')
Guilhem Saurel's avatar
Guilhem Saurel committed
464
465
466
            return requests.get(self.api_url() + url, {'page': page},
                                verify=self.forge.verify,
                                headers=self.forge.headers())
Guilhem Saurel's avatar
Guilhem Saurel committed
467
468
469
470
471
472

    def api_data(self, url=''):
        req = self.api_req(url)
        return req.json() if req.status_code == 200 else []  # TODO

    def api_list(self, url='', name=None):
Guilhem Saurel's avatar
Guilhem Saurel committed
473
474
        page = 1
        while page:
Guilhem Saurel's avatar
Guilhem Saurel committed
475
            req = self.api_req(url, name, page)
Guilhem Saurel's avatar
Guilhem Saurel committed
476
477
478
479
            if req.status_code != 200:
                return []  # TODO
            data = req.json()
            if name is not None:
Guilhem Saurel's avatar
Guilhem Saurel committed
480
481
482
483
                if name in data:
                    data = data[name]
                else:
                    return []  # TODO
Guilhem Saurel's avatar
Guilhem Saurel committed
484
            yield from data
Guilhem Saurel's avatar
Guilhem Saurel committed
485
            page = api_next(self.forge.source, req)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
486

Guilhem Saurel's avatar
Guilhem Saurel committed
487
    def api_update(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
488
        data = self.api_data()
Guilhem Saurel's avatar
Guilhem Saurel committed
489
        if data:
Guilhem Saurel's avatar
Guilhem Saurel committed
490
491
492
493
494
495
496
497
            if data['archived']:
                if self.project.main_repo() == self:
                    self.project.archived = True
                    self.project.save()
                self.archived = True
                self.save()
            else:
                return getattr(self, f'api_update_{self.forge.get_source_display()}')(data)
Guilhem Saurel's avatar
Guilhem Saurel committed
498
499

    def api_update_gitlab(self, data):
500
        update_gitlab(self.forge, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
501

Guilhem Saurel's avatar
Guilhem Saurel committed
502
    def api_update_github(self, data):
503
        update_github(self.forge, self.namespace, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
504

Guilhem Saurel's avatar
git    
Guilhem Saurel committed
505
506
507
    def get_clone_url(self):
        if self.forge.source == SOURCES.gitlab:
            return self.clone_url.replace('://', f'://gitlab-ci-token:{self.forge.token}@')
508
509
        if self.forge.source == SOURCES.github:
            return self.clone_url.replace('://', f'://{settings.GITHUB_USER}:{self.forge.token}@')
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
510
511
        return self.clone_url

512
513
514
    def git_remote(self):
        return f'{self.forge.slug}/{self.namespace.slug}'

Guilhem Saurel's avatar
git    
Guilhem Saurel committed
515
    def git(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
516
        git_repo = self.project.git()
517
        remote = self.git_remote()
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
518
        try:
Guilhem Saurel's avatar
Guilhem Saurel committed
519
            return git_repo.remote(remote)
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
520
        except ValueError:
Guilhem Saurel's avatar
Guilhem Saurel committed
521
            logger.info(f'Creating remote {remote}')
Guilhem Saurel's avatar
Guilhem Saurel committed
522
            return git_repo.create_remote(remote, self.get_clone_url())
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
523

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
524
    def fetch(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
525
526
        git_repo = self.git()
        logger.debug(f'fetching {self.forge} / {self.namespace} / {self.project}')
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
527
        try:
Guilhem Saurel's avatar
Guilhem Saurel committed
528
            git_repo.fetch()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
529
        except git.exc.GitCommandError:
Guilhem Saurel's avatar
Guilhem Saurel committed
530
            logger.warning(f'fetching {self.forge} / {self.namespace} / {self.project} - SECOND TRY')
531
532
533
534
535
            try:
                git_repo.fetch()
            except git.exc.GitCommandError:
                return False
        return True
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
536

Guilhem Saurel's avatar
Guilhem Saurel committed
537
538
539
    def main_branch(self):
        return self.project.branch_set.get(name=f'{self.git_remote()}/{self.default_branch}')

Guilhem Saurel's avatar
Guilhem Saurel committed
540
    def ahead(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
541
542
        main_branch = self.main_branch()
        return main_branch.ahead if main_branch is not None else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
543
544

    def behind(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
545
546
        main_branch = self.main_branch()
        return main_branch.behind if main_branch is not None else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
547

Guilhem Saurel's avatar
Guilhem Saurel committed
548
549
550
551
    def get_builds(self):
        return getattr(self, f'get_builds_{self.forge.get_source_display()}')()

    def get_builds_gitlab(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
552
        for pipeline in self.api_list('/pipelines'):
Guilhem Saurel's avatar
Guilhem Saurel committed
553
554
555
            pid, ref = pipeline['id'], pipeline['ref']
            if self.project.tag_set.filter(name=ref).exists():
                continue
Guilhem Saurel's avatar
Guilhem Saurel committed
556
            data = self.api_data(f'/pipelines/{pid}')
Guilhem Saurel's avatar
Guilhem Saurel committed
557
558
559
560
            branch_name = f'{self.forge.slug}/{self.namespace.slug}/{ref}'
            branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
            if created:
                branch.update()
Guilhem Saurel's avatar
Guilhem Saurel committed
561
562
563
564
565
566
567
            ci_build, created = CIBuild.objects.get_or_create(repo=self,
                                                              build_id=pid,
                                                              defaults={
                                                                  'passed': GITLAB_STATUS[pipeline['status']],
                                                                  'started': parse_datetime(data['created_at']),
                                                                  'branch': branch,
                                                              })
Guilhem Saurel's avatar
cijob    
Guilhem Saurel committed
568
569
570
571
572
573
            if not created and ci_build.passed != GITLAB_STATUS[pipeline['status']]:
                ci_build.passed = GITLAB_STATUS[pipeline['status']]
                ci_build.save()

    def get_jobs_gitlab(self):
        for data in self.api_list('/jobs'):
Guilhem Saurel's avatar
oops    
Guilhem Saurel committed
574
            branch_name = f'{self.forge.slug}/{self.namespace.slug}/{data["ref"]}'
Guilhem Saurel's avatar
cijob    
Guilhem Saurel committed
575
576
577
            branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
            if created:
                branch.update()
Guilhem Saurel's avatar
Guilhem Saurel committed
578
579
580
581
582
583
584
            ci_job, created = CIJob.objects.get_or_create(repo=self,
                                                          job_id=data['id'],
                                                          defaults={
                                                              'passed': GITLAB_STATUS[data['status']],
                                                              'started': parse_datetime(data['created_at']),
                                                              'branch': branch,
                                                          })
Guilhem Saurel's avatar
cijob    
Guilhem Saurel committed
585
586
587
588
589
590
591
592
            if not created and ci_job.passed != GITLAB_STATUS[data['status']]:
                ci_job.passed = GITLAB_STATUS[data['status']]
                ci_job.save()
            if self == self.project.main_repo() and data['name'].startswith('robotpkg-'):
                py3 = '-py3' in data['name']
                debug = '-debug' in data['name']
                target = next(target for target in Target.objects.all() if target.name in data['name']).name
                robotpkg = data['name'][9:-(2 + len(target) + (5 if debug else 7) + (3 if py3 else 0))]  # shame.
Guilhem Saurel's avatar
Guilhem Saurel committed
593
594
595
596
                images = Image.objects.filter(robotpkg__name=robotpkg, target__name=target, debug=debug, py3=py3)
                if not images.exists():
                    continue
                image = images.first()
Guilhem Saurel's avatar
cijob    
Guilhem Saurel committed
597
598
599
                if image.allow_failure and GITLAB_STATUS[data['status']]:
                    image.allow_failure = False
                    image.save()
600
                    print('  success', data['web_url'])
Guilhem Saurel's avatar
Guilhem Saurel committed
601
602
603
604

    def get_builds_github(self):
        if self.travis_id is not None:
            travis = Forge.objects.get(source=SOURCES.travis)
Guilhem Saurel's avatar
Guilhem Saurel committed
605
            for build in travis.api_list(f'/repo/{self.travis_id}/builds', name='builds'):
606
                if build['branch'] is None or self.project.tag_set.filter(name=build['branch']['name']).exists():
Guilhem Saurel's avatar
Guilhem Saurel committed
607
608
609
610
                    continue
                branch_name = f'{self.forge.slug}/{self.namespace.slug}/{build["branch"]["name"]}'
                branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
                if created:
Guilhem Saurel's avatar
Guilhem Saurel committed
611
                    branch.update()
Guilhem Saurel's avatar
Guilhem Saurel committed
612
                started = build['started_at'] if build['started_at'] is not None else build['finished_at']
Guilhem Saurel's avatar
Guilhem Saurel committed
613
614
615
616
617
618
619
                CIBuild.objects.get_or_create(repo=self,
                                              build_id=build['id'],
                                              defaults={
                                                  'passed': TRAVIS_STATE[build['state']],
                                                  'started': parse_datetime(started),
                                                  'branch': branch,
                                              })
Guilhem Saurel's avatar
Guilhem Saurel committed
620
621

    def update(self, pull=True):
622
        ok = True
623
624
        if self.project.main_namespace is None:
            return
Guilhem Saurel's avatar
Guilhem Saurel committed
625
626
        self.project.update_tags()
        if pull:
627
628
629
630
631
632
633
            ok = self.fetch()
        if ok:
            self.api_update()
            self.get_builds()
        else:
            logger.error(f'fetching {self.forge} / {self.namespace} / {self.project} - NOT FOUND - DELETING')
            logger.error(str(self.delete()))
Guilhem Saurel's avatar
Guilhem Saurel committed
634

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
635
636
637
638

class Commit(NamedModel, TimeStampedModel):
    project = models.ForeignKey(Project, on_delete=models.CASCADE)

Guilhem Saurel's avatar
Guilhem Saurel committed
639
640
641
    class Meta:
        unique_together = ('project', 'name')

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
642

Guilhem Saurel's avatar
Guilhem Saurel committed
643
644
645
646
647
class Branch(TimeStampedModel):
    name = models.CharField(max_length=200)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    ahead = models.PositiveSmallIntegerField(blank=True, null=True)
    behind = models.PositiveSmallIntegerField(blank=True, null=True)
648
    updated = models.DateTimeField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
649
    repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
650
    deleted = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
651
    keep_doc = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
652
653

    def __str__(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
654
655
656
        return self.name

    class Meta:
Guilhem Saurel's avatar
Guilhem Saurel committed
657
        unique_together = ('project', 'name', 'repo')
Guilhem Saurel's avatar
Guilhem Saurel committed
658
659

    def get_ahead(self, branch='master'):
Guilhem Saurel's avatar
Guilhem Saurel committed
660
661
        commits = self.project.git().git.rev_list(f'{branch}..{self}')
        return len(commits.split('\n')) if commits else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
662
663

    def get_behind(self, branch='master'):
Guilhem Saurel's avatar
Guilhem Saurel committed
664
665
        commits = self.project.git().git.rev_list(f'{self}..{branch}')
        return len(commits.split('\n')) if commits else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
666

667
668
669
    def git(self):
        git_repo = self.project.git()
        if self.name not in git_repo.branches:
Guilhem Saurel's avatar
Guilhem Saurel committed
670
            remote = self.repo.git()
671
672
673
674
675
            _, _, branch = self.name.split('/', maxsplit=2)
            git_repo.create_head(self.name, remote.refs[branch]).set_tracking_branch(remote.refs[branch])
        return git_repo.branches[self.name]

    def update(self, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
676
677
678
679
680
681
682
        if self.deleted:
            return
        try:
            if pull:
                self.repo.fetch()
                if self.repo != self.project.main_repo():
                    self.project.main_repo().fetch()
Guilhem Saurel's avatar
details    
Guilhem Saurel committed
683
684
            try:
                main_branch = self.project.main_branch()
Guilhem Saurel's avatar
Guilhem Saurel committed
685
686
                self.ahead = self.get_ahead(main_branch)
                self.behind = self.get_behind(main_branch)
Guilhem Saurel's avatar
typo    
Guilhem Saurel committed
687
            except Branch.DoesNotExist:
Guilhem Saurel's avatar
details    
Guilhem Saurel committed
688
                pass
Guilhem Saurel's avatar
Guilhem Saurel committed
689
690
691
            self.updated = self.git().commit.authored_datetime
        except (git.exc.GitCommandError, IndexError):
            self.deleted = True
Guilhem Saurel's avatar
Guilhem Saurel committed
692
        self.save()
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
693

Guilhem Saurel's avatar
Guilhem Saurel committed
694
695
696
697
698
699
700
    def ci(self):
        build = self.cibuild_set.last()
        if build is None:
            return ''
        status = {True: '✓', False: '✗', None: '?'}[build.passed]
        return mark_safe(f'<a href="{build.url()}">{status}</a>')

701
    def forge(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
702
        return self.repo.forge
703
704

    def namespace(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
705
        return self.repo.namespace
706

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
707

Guilhem Saurel's avatar
Guilhem Saurel committed
708
class TargetQuerySet(models.QuerySet):
Guilhem Saurel's avatar
Guilhem Saurel committed
709
710
711
    def active(self):
        return self.filter(active=True)

Guilhem Saurel's avatar
Guilhem Saurel committed
712
713
714
    def main(self):
        return self.get(main=True)

Guilhem Saurel's avatar
Guilhem Saurel committed
715

Guilhem Saurel's avatar
Guilhem Saurel committed
716
class Target(NamedModel):
Guilhem Saurel's avatar
Guilhem Saurel committed
717
    active = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
718
    main = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
719

Guilhem Saurel's avatar
Guilhem Saurel committed
720
    objects = TargetQuerySet.as_manager()
Guilhem Saurel's avatar
Guilhem Saurel committed
721
722
723


# class Test(TimeStampedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
724
725
726
727
728
729
730
#     project = models.ForeignKey(Project, on_delete=models.CASCADE)
#     branch = models.ForeignKey(Branch, on_delete=models.CASCADE)
#     commit = models.ForeignKey(Commit, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
#     passed = models.BooleanField(default=False)
#     TODO: travis vs gitlab-ci ?
#     TODO: deploy binary, doc, coverage, lint
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
731

Guilhem Saurel's avatar
Guilhem Saurel committed
732
# class SystemDependency(NamedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
733
734
#     project = models.ForeignKey(Project, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
735
736


Guilhem Saurel's avatar
Guilhem Saurel committed
737
class Robotpkg(NamedModel):
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
738
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
739
    category = models.CharField(max_length=50)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
740

Guilhem Saurel's avatar
Guilhem Saurel committed
741
    pkgbase = models.CharField(max_length=50, default='')
742
    pkgversion = models.CharField(max_length=50, default='')
Guilhem Saurel's avatar
Guilhem Saurel committed
743
744
745
746
    master_sites = models.CharField(max_length=200, default='')
    master_repository = models.CharField(max_length=200, default='')
    maintainer = models.CharField(max_length=200, default='')
    comment = models.TextField()
747
    homepage = models.URLField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
748
749

    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
750
    public = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
751
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
752
753
    updated = models.DateTimeField(blank=True, null=True)

Guilhem Saurel's avatar
same_py    
Guilhem Saurel committed
754
755
    same_py = models.BooleanField(default=True)

Guilhem Saurel's avatar
Guilhem Saurel committed
756
757
758
759
760
761
762
763
    def main_page(self):
        if self.category != 'wip':
            return f'{RPKG_URL}/robotpkg/{self.category}/{self.name}'

    def build_page(self):
        path = '-wip/wip' if self.category == 'wip' else f'/{self.category}'
        return f'{RPKG_URL}/rbulk/robotpkg{path}/{self.name}'

764
    def update_images(self):
765
        py3s = [False, True] if self.name.startswith('py-') else [False]
Guilhem Saurel's avatar
Guilhem Saurel committed
766
        debugs = [False, True]
Guilhem Saurel's avatar
Guilhem Saurel committed
767
        for target in Target.objects.active():
768
769
770
            for py3 in py3s:
                for debug in debugs:
                    Image.objects.get_or_create(robotpkg=self, target=target, py3=py3, debug=debug)[0].update()
Guilhem Saurel's avatar
Guilhem Saurel committed
771

772
    def update(self, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
773
774
775
776
777
778
        path = settings.RAINBOARD_RPKG
        repo = git.Repo(str(path / 'wip' / '.git' if self.category == 'wip' else path / '.git'))
        if pull:
            repo.remotes.origin.pull()

        cwd = path / self.category / self.name
Guilhem Saurel's avatar
Guilhem Saurel committed
779
780
781
        if not cwd.is_dir():
            logger.warning(f'deleted {self}: {self.delete()}')
            return
Guilhem Saurel's avatar
Guilhem Saurel committed
782
783
784
785
786
787
788
789
790
791
792
793
794
        for field in RPKG_FIELDS:
            cmd = ['make', 'show-var', f'VARNAME={field}']
            self.__dict__[field.lower()] = check_output(cmd, cwd=cwd).decode().strip()

        repo_path = self.name if self.category == 'wip' else f'{self.category}/{self.name}'
        last_commit = next(repo.iter_commits(paths=repo_path, max_count=1))
        self.updated = last_commit.authored_datetime

        license = check_output(['make', 'show-var', f'VARNAME=LICENSE'], cwd=cwd).decode().strip()
        if license in RPKG_LICENSES:
            self.license = License.objects.get(spdx_id=RPKG_LICENSES[license])
        else:
            logger.warning(f'Unknown robotpkg license: {license}')
Guilhem Saurel's avatar
Guilhem Saurel committed
795
        self.public = not bool(check_output(['make', 'show-var', f'VARNAME=RESTRICTED'], cwd=cwd).decode().strip())
Guilhem Saurel's avatar
Guilhem Saurel committed
796
797
798
        with (cwd / 'DESCR').open() as f:
            self.description = f.read().strip()

799
        self.update_images()
Guilhem Saurel's avatar
Guilhem Saurel committed
800
        self.save()
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
801

802
    def valid_images(self):
803
        return self.image_set.filter(created__isnull=False, target__active=True).order_by('target__name')
804

805
    def without_py(self):
Guilhem Saurel's avatar
same_py    
Guilhem Saurel committed
806
        if 'py-' in self.name and self.same_py:
807
808
            return Robotpkg.objects.filter(name=self.name.replace('py-', '')).first()

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
809

Guilhem Saurel's avatar
Guilhem Saurel committed
810
# class RobotpkgBuild(TimeStampedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
811
812
813
#     robotpkg = models.ForeignKey(Robotpkg, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
#     passed = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
814
815


816
817
class Image(models.Model):
    robotpkg = models.ForeignKey(Robotpkg, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
818
    target = models.ForeignKey(Target, on_delete=models.CASCADE)
819
820
    created = models.DateTimeField(blank=True, null=True)
    image = models.CharField(max_length=12, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
821
    py3 = models.BooleanField(default=False)
822
    debug = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
823
    allow_failure = models.BooleanField(default=False)
824
825

    class Meta:
826
        unique_together = ('robotpkg', 'target', 'py3', 'debug')
827
828

    def __str__(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
829
830
        py = '-py3' if self.py3 else ''
        return f'{self.robotpkg}{py}:{self.target}'
831
832

    def get_build_args(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
833
834
835
        ret = {
            'TARGET': self.target,
            'ROBOTPKG': self.robotpkg,
Guilhem Saurel's avatar
typo    
Guilhem Saurel committed
836
            'CATEGORY': self.robotpkg.category,
Guilhem Saurel's avatar
Guilhem Saurel committed
837
838
            'REGISTRY': self.robotpkg.project.registry(),
        }
Guilhem Saurel's avatar
Guilhem Saurel committed
839
        if not self.robotpkg.project.public:
Guilhem Saurel's avatar
Guilhem Saurel committed
840
841
842
            ret['IMAGE'] = 'robotpkg-jrl-py3' if self.py3 else 'robotpkg-jrl'
        elif self.py3:
            ret['IMAGE'] = 'robotpkg-py3'
Guilhem Saurel's avatar
Guilhem Saurel committed
843
        return ret
844
845
846

    def get_image_name(self):
        project = self.robotpkg.project
847
        return f'{project.registry()}/{project.main_namespace.slug}/{project.slug}/{self}'.lower()
848

849
850
851
    def get_image_url(self):
        project = self.robotpkg.project
        manifest = str(self).replace(':', '/manifests/')
852
        return f'https://{project.registry()}/v2/{project.main_namespace.slug}/{project.slug}/{manifest}'
853

Guilhem Saurel's avatar
Guilhem Saurel committed
854
    def get_job_name(self):
855
856
        mode = 'debug' if self.debug else 'release'
        return f'robotpkg-{self}-{mode}'.replace(':', '-')
Guilhem Saurel's avatar
Guilhem Saurel committed
857

858
859
860
    def build(self):
        args = self.get_build_args()
        build_args = sum((['--build-arg', f'{key}={value}'] for key, value in args.items()), list())
861
862
863
864
865
866
867
        return ['docker', 'build', '-t', self.get_image_name()] + build_args + ['.']

    def pull(self):
        return ['docker', 'pull', self.get_image_name()]

    def push(self):
        return ['docker', 'push', self.get_image_name()]
868

Guilhem Saurel's avatar
Guilhem Saurel committed
869
    def update(self, pull=False):
870
871
872
        headers = {}
        if not self.robotpkg.project.public:
            image_name = self.get_image_name().split('/', maxsplit=1)[1].split(':')[0]
Guilhem Saurel's avatar
Guilhem Saurel committed
873
874
875
876
877
878
879
            token = requests.get(f'{self.robotpkg.project.main_forge.url}/jwt/auth', {
                'client_id': 'docker',
                'offline_token': True,
                'service': 'container_registry',
                'scope': f'repository:{image_name}:push,pull'
            },
                                 auth=('gsaurel', self.robotpkg.project.main_forge.token)).json()['token']
880
881
            headers['Authorization'] = f'Bearer {token}'
        r = requests.get(self.get_image_url(), headers=headers)
882
883
884
885
        if r.status_code == 200:
            self.image = r.json()['fsLayers'][0]['blobSum'].split(':')[1][:12]
            self.created = parse_datetime(json.loads(r.json()['history'][0]['v1Compatibility'])['created'])
            self.save()
Guilhem Saurel's avatar
oops    
Guilhem Saurel committed
886
        if not self.allow_failure and self.created and (timezone.now() - self.created).days > 7:
887
888
            self.allow_failure = True
            self.save()
889
890


Guilhem Saurel's avatar
Guilhem Saurel committed
891
892
893
894
895
896
897
898
class CIBuild(models.Model):
    repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
    passed = models.NullBooleanField()
    build_id = models.PositiveIntegerField()
    started = models.DateTimeField()
    branch = models.ForeignKey(Branch, on_delete=models.CASCADE)

    class Meta:
Guilhem Saurel's avatar
Guilhem Saurel committed
899
        ordering = ('-started', )
Guilhem Saurel's avatar
Guilhem Saurel committed
900
901
902
903
904
905
906
907

    def url(self):
        if self.repo.forge.source == SOURCES.github:
            return f'https://travis-ci.org/{self.repo.namespace.slug}/{self.repo.slug}/builds/{self.build_id}'
        if self.repo.forge.source == SOURCES.gitlab:
            return f'{self.repo.forge.url}/{self.repo.namespace.slug}/{self.repo.slug}/pipelines/{self.build_id}'