models.py 39.6 KB
Newer Older
1
import json
2
import logging
Guilhem Saurel's avatar
Guilhem Saurel committed
3
import re
Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
4
from subprocess import check_output
5

Guilhem Saurel's avatar
Guilhem Saurel committed
6
from django.conf import settings
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
7
from django.db import models
Guilhem Saurel's avatar
Guilhem Saurel committed
8
from django.db.models import Q
Guilhem Saurel's avatar
Guilhem Saurel committed
9
from django.db.models.functions import Length
10
from django.db.utils import DataError
11
from django.template.loader import get_template
Guilhem Saurel's avatar
Guilhem Saurel committed
12
13
from django.utils.dateparse import parse_datetime
from django.utils.safestring import mark_safe
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
14

Guilhem Saurel's avatar
Guilhem Saurel committed
15
import requests
Guilhem Saurel's avatar
Guilhem Saurel committed
16
17

import git
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
18
from autoslug import AutoSlugField
Guilhem Saurel's avatar
Guilhem Saurel committed
19
from ndh.models import Links, NamedModel, TimeStampedModel
Guilhem Saurel's avatar
Guilhem Saurel committed
20
from ndh.utils import enum_to_choices, query_sum
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
21

Guilhem Saurel's avatar
Guilhem Saurel committed
22
from .utils import SOURCES, api_next, invalid_mail, slugify_with_dots, valid_name
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
23

24
25
logger = logging.getLogger('rainboard.models')

Guilhem Saurel's avatar
Guilhem Saurel committed
26
MAIN_BRANCHES = ['master', 'devel']
Guilhem Saurel's avatar
Guilhem Saurel committed
27
RPKG_URL = 'http://robotpkg.openrobots.org'
Guilhem Saurel's avatar
Guilhem Saurel committed
28
DOC_URL = 'http://projects.laas.fr/gepetto/doc'
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
29
30
31
32
33
34
35
36
37
RPKG_LICENSES = {
    'gnu-lgpl-v3': 'LGPL-3.0',
    'gnu-lgpl-v2': 'LGPL-2.0',
    'mit': 'MIT',
    'gnu-gpl-v3': 'GPL-3.0',
    '2-clause-bsd': 'BSD-2-Clause',
    'eclipse': 'EPL-1.0',
    'modified-bsd': 'BSD-3-Clause'
}
Guilhem Saurel's avatar
Guilhem Saurel committed
38
RPKG_FIELDS = ['PKGBASE', 'PKGVERSION', 'MASTER_SITES', 'MASTER_REPOSITORY', 'MAINTAINER', 'COMMENT', 'HOMEPAGE']
Guilhem Saurel's avatar
Guilhem Saurel committed
39
CMAKE_FIELDS = {'NAME': 'cmake_name', 'DESCRIPTION': 'description', 'URL': 'homepage', 'VERSION': 'version'}
Guilhem Saurel's avatar
Guilhem Saurel committed
40
TRAVIS_STATE = {'created': None, 'passed': True, 'started': None, 'failed': False, 'errored': False, 'canceled': False}
Guilhem Saurel's avatar
Guilhem Saurel committed
41
GITLAB_STATUS = {'failed': False, 'success': True, 'pending': None, 'skipped': None, 'canceled': None, 'running': None}
Guilhem Saurel's avatar
Guilhem Saurel committed
42

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
43
44

class Namespace(NamedModel):
45
    group = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
46
47


Guilhem Saurel's avatar
Guilhem Saurel committed
48
49
50
class License(models.Model):
    name = models.CharField(max_length=200)
    spdx_id = models.CharField(max_length=50, unique=True)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
51
52
53
54
55
56
    url = models.URLField(max_length=200)

    def __str__(self):
        return self.spdx_id or self.name


Guilhem Saurel's avatar
Guilhem Saurel committed
57
class Forge(Links, NamedModel):
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
58
59
60
61
62
    source = models.PositiveSmallIntegerField(choices=enum_to_choices(SOURCES))
    url = models.URLField(max_length=200)
    token = models.CharField(max_length=50, blank=True, null=True)
    verify = models.BooleanField(default=True)

Guilhem Saurel's avatar
Guilhem Saurel committed
63
64
65
    def get_absolute_url(self):
        return self.url

Guilhem Saurel's avatar
Guilhem Saurel committed
66
    def api_req(self, url='', name=None, page=1):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
67
        logger.debug(f'requesting api {self} {url}, page {page}')
68
69
70
71
72
        try:
            return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
        except requests.exceptions.ConnectionError:
            logger.error(f'requesting api {self} {url}, page {page} - SECOND TRY')
            return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
Guilhem Saurel's avatar
Guilhem Saurel committed
73
74
75
76
77
78

    def api_data(self, url=''):
        req = self.api_req(url)
        return req.json() if req.status_code == 200 else []  # TODO

    def api_list(self, url='', name=None):
Guilhem Saurel's avatar
Guilhem Saurel committed
79
80
        page = 1
        while page:
Guilhem Saurel's avatar
Guilhem Saurel committed
81
            req = self.api_req(url, name, page)
Guilhem Saurel's avatar
Guilhem Saurel committed
82
83
84
85
86
            if req.status_code != 200:
                return []  # TODO
            data = req.json()
            if name is not None:
                data = data[name]
Guilhem Saurel's avatar
Guilhem Saurel committed
87
            yield from data
Guilhem Saurel's avatar
Guilhem Saurel committed
88
            page = api_next(self.source, req)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
89
90

    def headers(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
91
        return {
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
92
93
94
95
96
97
98
99
100
101
102
103
104
105
            SOURCES.github: {
                'Authorization': f'token {self.token}',
                'Accept': 'application/vnd.github.drax-preview+json'
            },
            SOURCES.gitlab: {
                'Private-Token': self.token
            },
            SOURCES.redmine: {
                'X-Redmine-API-Key': self.token
            },
            SOURCES.travis: {
                'Authorization': f'token {self.token}',
                'TRAVIS-API-Version': '3'
            },
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
106
        }[self.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
107
108

    def api_url(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
109
110
111
112
        return {
            SOURCES.github: 'https://api.github.com',
            SOURCES.gitlab: f'{self.url}/api/v4',
            SOURCES.redmine: self.url,
Guilhem Saurel's avatar
travis    
Guilhem Saurel committed
113
            SOURCES.travis: 'https://api.travis-ci.org',
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
114
        }[self.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
115

116
117
    def get_namespaces_github(self):
        for namespace in Namespace.objects.filter(group=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
118
            for data in self.api_list(f'/orgs/{namespace.slug}/members'):
Guilhem Saurel's avatar
Guilhem Saurel committed
119
120
                Namespace.objects.get_or_create(slug=data['login'].lower(),
                                                defaults={'name': data['login'], 'group': False})
121

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
122
123
    def get_namespaces_gitlab(self):
        for data in self.api_list('/namespaces'):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
124
125
126
127
128
            Namespace.objects.get_or_create(
                slug=data['path'], defaults={
                    'name': data['name'],
                    'group': data['kind'] == 'group'
                })
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
129
130
131
        for data in self.api_list('/users'):
            Namespace.objects.get_or_create(slug=data['username'], defaults={'name': data['name']})

Guilhem Saurel's avatar
Guilhem Saurel committed
132
133
134
    def get_namespaces_redmine(self):
        pass  # TODO

Guilhem Saurel's avatar
Guilhem Saurel committed
135
136
137
    def get_namespaces_travis(self):
        pass

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
138
139
140
    def get_projects(self):
        getattr(self, f'get_namespaces_{self.get_source_display()}')()
        return getattr(self, f'get_projects_{self.get_source_display()}')()
141

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
142
    def get_projects_github(self):
143
        for org in Namespace.objects.filter(group=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
144
            for data in self.api_list(f'/orgs/{org.slug}/repos'):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
145
                update_github(self, org, data)
146
        for user in Namespace.objects.filter(group=False):
Guilhem Saurel's avatar
Guilhem Saurel committed
147
            for data in self.api_list(f'/users/{user.slug}/repos'):
Guilhem Saurel's avatar
Guilhem Saurel committed
148
                if Project.objects.filter(name=valid_name(data['name'])).exists():
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
149
                    update_github(self, user, data)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
150
151

    def get_projects_gitlab(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
152
        for data in self.api_list('/projects'):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
153
            update_gitlab(self, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
154
155

        for orphan in Project.objects.filter(main_namespace=None):
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
156
            repo = orphan.repo_set.filter(forge__source=SOURCES.gitlab).first()
Guilhem Saurel's avatar
Guilhem Saurel committed
157
158
            if repo:
                update_gitlab(self, self.api_data(f'/projects/{repo.forked_from}'))
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
159
160
161
162

    def get_projects_redmine(self):
        pass  # TODO

Guilhem Saurel's avatar
Guilhem Saurel committed
163
164
165
166
167
168
    def get_projects_travis(self):
        for namespace in Namespace.objects.all():
            for repository in self.api_list(f'/owner/{namespace.slug}/repos', 'repositories'):
                if repository['active']:
                    update_travis(namespace, repository)

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
169

Guilhem Saurel's avatar
Guilhem Saurel committed
170
class Project(Links, NamedModel, TimeStampedModel):
Guilhem Saurel's avatar
Guilhem Saurel committed
171
    public = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
172
173
174
175
    main_namespace = models.ForeignKey(Namespace, on_delete=models.SET_NULL, null=True, blank=True)
    main_forge = models.ForeignKey(Forge, on_delete=models.SET_NULL, null=True, blank=True)
    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    homepage = models.URLField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
176
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
177
    version = models.CharField(max_length=20, blank=True, null=True)
178
    updated = models.DateTimeField(blank=True, null=True)
179
180
    tests = models.BooleanField(default=True)
    docs = models.BooleanField(default=True)
181
    debug = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
182
    from_gepetto = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
183
    cmake_name = models.CharField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
184

185
186
187
188
    def save(self, *args, **kwargs):
        self.name = valid_name(self.name)
        super().save(*args, **kwargs)

Guilhem Saurel's avatar
Guilhem Saurel committed
189
    def git_path(self):
190
        return settings.RAINBOARD_GITS / self.main_namespace.slug / self.slug.strip()  # workaround SafeText TypeError
Guilhem Saurel's avatar
Guilhem Saurel committed
191

Guilhem Saurel's avatar
Guilhem Saurel committed
192
    def git(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
193
        path = self.git_path()
Guilhem Saurel's avatar
Guilhem Saurel committed
194
195
196
197
198
        if not path.exists():
            logger.info(f'Creating repo for {self.main_namespace.slug}/{self.slug}')
            return git.Repo.init(path)
        return git.Repo(str(path / '.git'))

199
    def main_repo(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
200
        forge = self.main_forge if self.main_forge else get_default_forge(self)
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
201
202
203
204
205
206
207
208
209
        repo, created = Repo.objects.get_or_create(
            forge=forge,
            namespace=self.main_namespace,
            project=self,
            defaults={
                'name': self.name,
                'default_branch': 'master',
                'repo_id': 0
            })
Guilhem Saurel's avatar
Guilhem Saurel committed
210
        if created:
211
            repo.api_update()
Guilhem Saurel's avatar
Guilhem Saurel committed
212
213
        return repo

Guilhem Saurel's avatar
Guilhem Saurel committed
214
    def update_branches(self, main=True, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
215
216
217
        branches = [b[2:] for b in self.git().git.branch('-a', '--no-color').split('\n')]
        if main:
            branches = [b for b in branches if b.endswith('master') or b.endswith('devel')]
Guilhem Saurel's avatar
Guilhem Saurel committed
218
        for branch in branches:
Guilhem Saurel's avatar
Guilhem Saurel committed
219
            logger.info(f'update branch {branch}')
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
220
221
            if branch.startswith('remotes/'):
                branch = branch[8:]
222
223
224
            if branch.count('/') < 2:
                logger.error(f'wrong branch "{branch}" in {self.git_path()}')
                continue
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
            forge, namespace, name = branch.split('/', maxsplit=2)
            namespace, _ = Namespace.objects.get_or_create(slug=namespace)
            forge = Forge.objects.get(slug=forge)
            repo, created = Repo.objects.get_or_create(
                forge=forge,
                namespace=namespace,
                project=self,
                defaults={
                    'name': self.name,
                    'default_branch': 'master',
                    'repo_id': 0
                })
            if created:
                repo.api_update()
            instance, bcreated = Branch.objects.get_or_create(name=branch, project=self, repo=repo)
Guilhem Saurel's avatar
Guilhem Saurel committed
240
            if bcreated:
Guilhem Saurel's avatar
Guilhem Saurel committed
241
                instance.update(pull=pull)
242

Guilhem Saurel's avatar
update    
Guilhem Saurel committed
243
244
245
    def checkout(self):
        self.main_branch().git().checkout()

Guilhem Saurel's avatar
Guilhem Saurel committed
246
    def main_branch(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
247
        return self.main_repo().main_branch()
Guilhem Saurel's avatar
Guilhem Saurel committed
248

Guilhem Saurel's avatar
Guilhem Saurel committed
249
250
251
252
253
254
255
    def cmake(self):
        filename = self.git_path() / 'CMakeLists.txt'
        if not filename.exists():
            return
        with filename.open() as f:
            content = f.read()
        for key, value in CMAKE_FIELDS.items():
Guilhem Saurel's avatar
Guilhem Saurel committed
256
            search = re.search(fr'set\s*\(\s*project_{key}\s+([^)]+)*\)', content, re.I)
Guilhem Saurel's avatar
Guilhem Saurel committed
257
            if search:
258
                try:
259
                    old = getattr(self, value)
260
                    new = search.groups()[0].strip(''' \r\n\t'"''').replace('_', '-')
261
262
263
                    if old != new:
                        setattr(self, value, new)
                        self.save()
264
                except DataError:
265
                    setattr(self, value, old)
266
        for dependency in re.findall(r'ADD_[A-Z]+_DEPENDENCY\s*\(["\']?([^ "\')]+).*["\']?\)', content, re.I):
Guilhem Saurel's avatar
Guilhem Saurel committed
267
            project = Project.objects.filter(name=valid_name(dependency))
268
            if project.exists():
Guilhem Saurel's avatar
oops    
Guilhem Saurel committed
269
270
                dependency, _ = Dependency.objects.get_or_create(project=self, library=project.first())
                if not dependency.cmake:
271
272
                    dependency.cmake = True
                    dependency.save()
Guilhem Saurel's avatar
Guilhem Saurel committed
273

274
    def ros(self):
275
276
        try:
            filename = self.git_path() / 'package.xml'
277
        except TypeError:
278
            return
279
280
281
282
283
        if not filename.exists():
            return
        with filename.open() as f:
            content = f.read()
        for dependency in re.findall(r'<run_depend>(\w+).*</run_depend>', content, re.I):
Guilhem Saurel's avatar
Guilhem Saurel committed
284
            project = Project.objects.filter(name=valid_name(dependency))
285
286
287
288
289
290
            if project.exists():
                dependency, _ = Dependency.objects.get_or_create(project=self, library=project.first())
                if not dependency.ros:
                    dependency.ros = True
                    dependency.save()

291
292
293
294
295
296
    def repos(self):
        return self.repo_set.count()

    def rpkgs(self):
        return self.robotpkg_set.count()

Guilhem Saurel's avatar
Guilhem Saurel committed
297
298
299
300
    def update_tags(self):
        for tag in self.git().tags:
            Tag.objects.get_or_create(name=str(tag), project=self)

301
    def update(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
302
303
        if self.main_namespace is None:
            return
Guilhem Saurel's avatar
Guilhem Saurel committed
304
        self.update_branches()
Guilhem Saurel's avatar
Guilhem Saurel committed
305
        self.update_tags()
306
        tag = self.tag_set.filter(name__startswith='v').last()  # TODO: implement SQL ordering for semver
Guilhem Saurel's avatar
Guilhem Saurel committed
307
308
        if tag is not None:
            self.version = tag.name[1:]
309
        robotpkg = self.robotpkg_set.order_by('-updated').first()
Guilhem Saurel's avatar
Guilhem Saurel committed
310
        branch = self.branch_set.order_by('-updated').first()
Guilhem Saurel's avatar
Guilhem Saurel committed
311
        if (branch is not None and branch.updated is not None) or robotpkg is not None:
Guilhem Saurel's avatar
Guilhem Saurel committed
312
313
            if robotpkg is None:
                self.updated = branch.updated
314
            elif branch is None or branch.updated is None:
Guilhem Saurel's avatar
Guilhem Saurel committed
315
316
317
                self.updated = robotpkg.updated
            else:
                self.updated = max(branch.updated, robotpkg.updated)
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
318
        self.checkout()
319
        self.cmake()
Guilhem Saurel's avatar
Guilhem Saurel committed
320
        self.ros()
321
322
        self.save()

Guilhem Saurel's avatar
Guilhem Saurel committed
323
    def commits_since(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
324
325
326
327
328
        try:
            commits = self.git().git.rev_list(f'v{self.version}..{self.main_branch()}')
            return len(commits.split('\n')) if commits else 0
        except git.exc.GitCommandError:
            pass
Guilhem Saurel's avatar
Guilhem Saurel committed
329

330
331
332
333
334
335
    def open_issues(self):
        return query_sum(self.repo_set, 'open_issues')

    def open_pr(self):
        return query_sum(self.repo_set, 'open_pr')

336
337
338
    def gitlabciyml(self):
        return get_template('rainboard/gitlab-ci.yml').render({'project': self})

Guilhem Saurel's avatar
Guilhem Saurel committed
339
340
341
342
343
344
345
346
347
    def contributors(self, update=False):
        if update:
            for guy in self.git().git.shortlog('-nse').split('\n'):
                name, mail = guy[7:-1].split(' <')
                contributor = get_contributor(name, mail)
                contributor.projects.add(self)
                contributor.save()
        return self.contributor_set.all()

348
349
350
    def registry(self):
        return settings.PUBLIC_REGISTRY if self.public else settings.PRIVATE_REGISTRY

Guilhem Saurel's avatar
Guilhem Saurel committed
351
352
353
354
    def doc_coverage_image(self):
        images = Image.objects.filter(robotpkg__project=self, py3=False, target__name='16.04')
        return images.order_by(Length('robotpkg__name').desc()).first()

355
    def print_deps(self):
356
        return mark_safe(', '.join(d.library.get_link() for d in self.dependencies.all()))
357
358

    def print_rdeps(self):
359
        return mark_safe(', '.join(d.project.get_link() for d in self.rdeps.all()))
360

Guilhem Saurel's avatar
Guilhem Saurel committed
361
362
363
    def ordered_robotpkg(self):
        return self.robotpkg_set.order_by('name')

Guilhem Saurel's avatar
Guilhem Saurel committed
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
    def url_travis(self):
        return f'https://travis-ci.org/{self.main_namespace.slug}/{self.slug}'

    def url_gitlab(self):
        return f'https://gepgitlab.laas.fr/{self.main_namespace.slug}/{self.slug}'

    def badge(self, link, img, alt):
        return mark_safe(f'<a href="{link}"><img src="{img}" alt="{alt}" /></a> ')

    def badge_travis(self):
        return self.badge(self.url_travis(), f'{self.url_travis()}.svg?branch=master', 'Building Status')

    def badge_gitlab(self):
        return self.badge(self.url_gitlab(), f'{self.url_gitlab()}/badges/master/pipeline.svg', 'Pipeline Status')

    def badge_coverage(self):
        return self.badge(f'{DOC_URL}/{self.main_namespace.slug}/{self.slug}/master/coverage',
Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
381
                          f'{self.url_gitlab()}/badges/master/coverage.svg?job=doc-coverage"', 'Coverage Report')
Guilhem Saurel's avatar
Guilhem Saurel committed
382
383
384
385

    def badges(self):
        return self.badge_travis() + self.badge_gitlab() + self.badge_coverage()

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
386

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
387
388
class Repo(TimeStampedModel):
    name = models.CharField(max_length=200)
Guilhem Saurel's avatar
Guilhem Saurel committed
389
    slug = AutoSlugField(populate_from='name', slugify=slugify_with_dots)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
390
391
392
393
394
395
396
397
398
399
    forge = models.ForeignKey(Forge, on_delete=models.CASCADE)
    namespace = models.ForeignKey(Namespace, on_delete=models.CASCADE)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    homepage = models.URLField(max_length=200, blank=True, null=True)
    url = models.URLField(max_length=200, blank=True, null=True)
    default_branch = models.CharField(max_length=50)
    open_issues = models.PositiveSmallIntegerField(blank=True, null=True)
    open_pr = models.PositiveSmallIntegerField(blank=True, null=True)
    repo_id = models.PositiveIntegerField()
Guilhem Saurel's avatar
Guilhem Saurel committed
400
    forked_from = models.PositiveIntegerField(blank=True, null=True)
401
    clone_url = models.URLField(max_length=200)
Guilhem Saurel's avatar
travis    
Guilhem Saurel committed
402
    travis_id = models.PositiveIntegerField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
403
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
404
405
406

    def __str__(self):
        return self.name
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
407
408

    def api_url(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
409
        api_url = self.forge.api_url()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
410
        return {
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
411
412
413
            SOURCES.github: f'{api_url}/repos/{self.namespace.slug}/{self.slug}',
            SOURCES.redmine: f'{api_url}/projects/{self.repo_id}.json',
            SOURCES.gitlab: f'{api_url}/projects/{self.repo_id}',
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
414
        }[self.forge.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
415

Guilhem Saurel's avatar
Guilhem Saurel committed
416
    def api_req(self, url='', name=None, page=1):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
417
        logger.debug(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}')
418
        try:
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
419
420
            return requests.get(
                self.api_url() + url, {'page': page}, verify=self.forge.verify, headers=self.forge.headers())
421
422
        except requests.exceptions.ConnectionError:
            logger.error(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page} - SECOND TRY')
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
423
424
            return requests.get(
                self.api_url() + url, {'page': page}, verify=self.forge.verify, headers=self.forge.headers())
Guilhem Saurel's avatar
Guilhem Saurel committed
425
426
427
428
429
430

    def api_data(self, url=''):
        req = self.api_req(url)
        return req.json() if req.status_code == 200 else []  # TODO

    def api_list(self, url='', name=None):
Guilhem Saurel's avatar
Guilhem Saurel committed
431
432
        page = 1
        while page:
Guilhem Saurel's avatar
Guilhem Saurel committed
433
            req = self.api_req(url, name, page)
Guilhem Saurel's avatar
Guilhem Saurel committed
434
435
436
437
            if req.status_code != 200:
                return []  # TODO
            data = req.json()
            if name is not None:
Guilhem Saurel's avatar
Guilhem Saurel committed
438
439
440
441
                if name in data:
                    data = data[name]
                else:
                    return []  # TODO
Guilhem Saurel's avatar
Guilhem Saurel committed
442
            yield from data
Guilhem Saurel's avatar
Guilhem Saurel committed
443
            page = api_next(self.forge.source, req)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
444

Guilhem Saurel's avatar
Guilhem Saurel committed
445
    def api_update(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
446
        data = self.api_data()
Guilhem Saurel's avatar
Guilhem Saurel committed
447
        if data:
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
448
            return getattr(self, f'api_update_{self.forge.get_source_display()}')(data)
Guilhem Saurel's avatar
Guilhem Saurel committed
449
450

    def api_update_gitlab(self, data):
451
        update_gitlab(self.forge, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
452

Guilhem Saurel's avatar
Guilhem Saurel committed
453
    def api_update_github(self, data):
454
        update_github(self.forge, self.namespace, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
455

Guilhem Saurel's avatar
git    
Guilhem Saurel committed
456
457
458
    def get_clone_url(self):
        if self.forge.source == SOURCES.gitlab:
            return self.clone_url.replace('://', f'://gitlab-ci-token:{self.forge.token}@')
459
460
        if self.forge.source == SOURCES.github:
            return self.clone_url.replace('://', f'://{settings.GITHUB_USER}:{self.forge.token}@')
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
461
462
        return self.clone_url

463
464
465
    def git_remote(self):
        return f'{self.forge.slug}/{self.namespace.slug}'

Guilhem Saurel's avatar
git    
Guilhem Saurel committed
466
    def git(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
467
        git_repo = self.project.git()
468
        remote = self.git_remote()
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
469
        try:
Guilhem Saurel's avatar
Guilhem Saurel committed
470
            return git_repo.remote(remote)
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
471
        except ValueError:
Guilhem Saurel's avatar
Guilhem Saurel committed
472
            logger.info(f'Creating remote {remote}')
Guilhem Saurel's avatar
Guilhem Saurel committed
473
            return git_repo.create_remote(remote, self.get_clone_url())
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
474

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
475
    def fetch(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
476
477
        git_repo = self.git()
        logger.debug(f'fetching {self.forge} / {self.namespace} / {self.project}')
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
478
        try:
Guilhem Saurel's avatar
Guilhem Saurel committed
479
            git_repo.fetch()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
480
        except git.exc.GitCommandError:
Guilhem Saurel's avatar
Guilhem Saurel committed
481
            logger.warning(f'fetching {self.forge} / {self.namespace} / {self.project} - SECOND TRY')
482
483
484
485
486
            try:
                git_repo.fetch()
            except git.exc.GitCommandError:
                return False
        return True
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
487

Guilhem Saurel's avatar
Guilhem Saurel committed
488
489
490
    def main_branch(self):
        return self.project.branch_set.get(name=f'{self.git_remote()}/{self.default_branch}')

Guilhem Saurel's avatar
Guilhem Saurel committed
491
    def ahead(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
492
493
        main_branch = self.main_branch()
        return main_branch.ahead if main_branch is not None else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
494
495

    def behind(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
496
497
        main_branch = self.main_branch()
        return main_branch.behind if main_branch is not None else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
498

Guilhem Saurel's avatar
Guilhem Saurel committed
499
500
501
502
    def get_builds(self):
        return getattr(self, f'get_builds_{self.forge.get_source_display()}')()

    def get_builds_gitlab(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
503
        for pipeline in self.api_list('/pipelines'):
Guilhem Saurel's avatar
Guilhem Saurel committed
504
505
506
            pid, ref = pipeline['id'], pipeline['ref']
            if self.project.tag_set.filter(name=ref).exists():
                continue
Guilhem Saurel's avatar
Guilhem Saurel committed
507
            data = self.api_data(f'/pipelines/{pid}')
Guilhem Saurel's avatar
Guilhem Saurel committed
508
509
510
511
            branch_name = f'{self.forge.slug}/{self.namespace.slug}/{ref}'
            branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
            if created:
                branch.update()
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
512
513
514
515
516
517
518
519
            CIBuild.objects.get_or_create(
                repo=self,
                build_id=pid,
                defaults={
                    'passed': GITLAB_STATUS[pipeline['status']],
                    'started': parse_datetime(data['created_at']),
                    'branch': branch,
                })
Guilhem Saurel's avatar
Guilhem Saurel committed
520
521
522
523

    def get_builds_github(self):
        if self.travis_id is not None:
            travis = Forge.objects.get(source=SOURCES.travis)
Guilhem Saurel's avatar
Guilhem Saurel committed
524
            for build in travis.api_list(f'/repo/{self.travis_id}/builds', name='builds'):
Guilhem Saurel's avatar
Guilhem Saurel committed
525
526
527
528
529
                if self.project.tag_set.filter(name=build['branch']['name']).exists():
                    continue
                branch_name = f'{self.forge.slug}/{self.namespace.slug}/{build["branch"]["name"]}'
                branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
                if created:
Guilhem Saurel's avatar
Guilhem Saurel committed
530
                    branch.update()
Guilhem Saurel's avatar
Guilhem Saurel committed
531
                started = build['started_at'] if build['started_at'] is not None else build['finished_at']
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
532
533
534
535
536
537
538
539
                CIBuild.objects.get_or_create(
                    repo=self,
                    build_id=build['id'],
                    defaults={
                        'passed': TRAVIS_STATE[build['state']],
                        'started': parse_datetime(started),
                        'branch': branch,
                    })
Guilhem Saurel's avatar
Guilhem Saurel committed
540
541

    def update(self, pull=True):
542
        ok = True
Guilhem Saurel's avatar
Guilhem Saurel committed
543
544
        self.project.update_tags()
        if pull:
545
546
547
548
549
550
551
            ok = self.fetch()
        if ok:
            self.api_update()
            self.get_builds()
        else:
            logger.error(f'fetching {self.forge} / {self.namespace} / {self.project} - NOT FOUND - DELETING')
            logger.error(str(self.delete()))
Guilhem Saurel's avatar
Guilhem Saurel committed
552

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
553
554
555
556

class Commit(NamedModel, TimeStampedModel):
    project = models.ForeignKey(Project, on_delete=models.CASCADE)

Guilhem Saurel's avatar
Guilhem Saurel committed
557
558
559
    class Meta:
        unique_together = ('project', 'name')

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
560

Guilhem Saurel's avatar
Guilhem Saurel committed
561
562
563
564
565
class Branch(TimeStampedModel):
    name = models.CharField(max_length=200)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    ahead = models.PositiveSmallIntegerField(blank=True, null=True)
    behind = models.PositiveSmallIntegerField(blank=True, null=True)
566
    updated = models.DateTimeField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
567
    repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
568
    deleted = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
569
    keep_doc = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
570
571

    def __str__(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
572
573
574
        return self.name

    class Meta:
Guilhem Saurel's avatar
Guilhem Saurel committed
575
        unique_together = ('project', 'name', 'repo')
Guilhem Saurel's avatar
Guilhem Saurel committed
576
577

    def get_ahead(self, branch='master'):
Guilhem Saurel's avatar
Guilhem Saurel committed
578
579
        commits = self.project.git().git.rev_list(f'{branch}..{self}')
        return len(commits.split('\n')) if commits else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
580
581

    def get_behind(self, branch='master'):
Guilhem Saurel's avatar
Guilhem Saurel committed
582
583
        commits = self.project.git().git.rev_list(f'{self}..{branch}')
        return len(commits.split('\n')) if commits else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
584

585
586
587
    def git(self):
        git_repo = self.project.git()
        if self.name not in git_repo.branches:
Guilhem Saurel's avatar
Guilhem Saurel committed
588
            remote = self.repo.git()
589
590
591
592
593
            _, _, branch = self.name.split('/', maxsplit=2)
            git_repo.create_head(self.name, remote.refs[branch]).set_tracking_branch(remote.refs[branch])
        return git_repo.branches[self.name]

    def update(self, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
594
595
596
597
598
599
600
        if self.deleted:
            return
        try:
            if pull:
                self.repo.fetch()
                if self.repo != self.project.main_repo():
                    self.project.main_repo().fetch()
Guilhem Saurel's avatar
details    
Guilhem Saurel committed
601
602
            try:
                main_branch = self.project.main_branch()
Guilhem Saurel's avatar
Guilhem Saurel committed
603
604
                self.ahead = self.get_ahead(main_branch)
                self.behind = self.get_behind(main_branch)
Guilhem Saurel's avatar
typo    
Guilhem Saurel committed
605
            except Branch.DoesNotExist:
Guilhem Saurel's avatar
details    
Guilhem Saurel committed
606
                pass
Guilhem Saurel's avatar
Guilhem Saurel committed
607
608
609
            self.updated = self.git().commit.authored_datetime
        except (git.exc.GitCommandError, IndexError):
            self.deleted = True
Guilhem Saurel's avatar
Guilhem Saurel committed
610
        self.save()
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
611

Guilhem Saurel's avatar
Guilhem Saurel committed
612
613
614
615
616
617
618
    def ci(self):
        build = self.cibuild_set.last()
        if build is None:
            return ''
        status = {True: '✓', False: '✗', None: '?'}[build.passed]
        return mark_safe(f'<a href="{build.url()}">{status}</a>')

619
    def forge(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
620
        return self.repo.forge
621
622

    def namespace(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
623
        return self.repo.namespace
624

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
625

Guilhem Saurel's avatar
Guilhem Saurel committed
626
627
628
629
630
class ActiveQuerySet(models.QuerySet):
    def active(self):
        return self.filter(active=True)


Guilhem Saurel's avatar
Guilhem Saurel committed
631
class Target(NamedModel):
Guilhem Saurel's avatar
Guilhem Saurel committed
632
633
634
    active = models.BooleanField(default=True)

    objects = ActiveQuerySet.as_manager()
Guilhem Saurel's avatar
Guilhem Saurel committed
635
636
637


# class Test(TimeStampedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
638
639
640
641
642
643
644
#     project = models.ForeignKey(Project, on_delete=models.CASCADE)
#     branch = models.ForeignKey(Branch, on_delete=models.CASCADE)
#     commit = models.ForeignKey(Commit, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
#     passed = models.BooleanField(default=False)
#     TODO: travis vs gitlab-ci ?
#     TODO: deploy binary, doc, coverage, lint
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
645

Guilhem Saurel's avatar
Guilhem Saurel committed
646
# class SystemDependency(NamedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
647
648
#     project = models.ForeignKey(Project, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
649
650


Guilhem Saurel's avatar
Guilhem Saurel committed
651
class Robotpkg(NamedModel):
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
652
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
653
    category = models.CharField(max_length=50)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
654

Guilhem Saurel's avatar
Guilhem Saurel committed
655
656
657
658
659
660
    pkgbase = models.CharField(max_length=50, default='')
    pkgversion = models.CharField(max_length=20, default='')
    master_sites = models.CharField(max_length=200, default='')
    master_repository = models.CharField(max_length=200, default='')
    maintainer = models.CharField(max_length=200, default='')
    comment = models.TextField()
661
    homepage = models.URLField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
662
663

    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
664
    public = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
665
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
666
667
668
669
670
671
672
673
674
675
    updated = models.DateTimeField(blank=True, null=True)

    def main_page(self):
        if self.category != 'wip':
            return f'{RPKG_URL}/robotpkg/{self.category}/{self.name}'

    def build_page(self):
        path = '-wip/wip' if self.category == 'wip' else f'/{self.category}'
        return f'{RPKG_URL}/rbulk/robotpkg{path}/{self.name}'

676
    def update_images(self):
677
678
        py3s = [False, True] if self.name.startswith('py-') else [False]
        debugs = [False, True] if self.project.debug else [False]
Guilhem Saurel's avatar
Guilhem Saurel committed
679
        for target in Target.objects.active():
680
681
682
            for py3 in py3s:
                for debug in debugs:
                    Image.objects.get_or_create(robotpkg=self, target=target, py3=py3, debug=debug)[0].update()
Guilhem Saurel's avatar
Guilhem Saurel committed
683

684
    def update(self, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
685
686
687
688
689
690
        path = settings.RAINBOARD_RPKG
        repo = git.Repo(str(path / 'wip' / '.git' if self.category == 'wip' else path / '.git'))
        if pull:
            repo.remotes.origin.pull()

        cwd = path / self.category / self.name
Guilhem Saurel's avatar
Guilhem Saurel committed
691
692
693
        if not cwd.is_dir():
            logger.warning(f'deleted {self}: {self.delete()}')
            return
Guilhem Saurel's avatar
Guilhem Saurel committed
694
695
696
697
698
699
700
701
702
703
704
705
706
        for field in RPKG_FIELDS:
            cmd = ['make', 'show-var', f'VARNAME={field}']
            self.__dict__[field.lower()] = check_output(cmd, cwd=cwd).decode().strip()

        repo_path = self.name if self.category == 'wip' else f'{self.category}/{self.name}'
        last_commit = next(repo.iter_commits(paths=repo_path, max_count=1))
        self.updated = last_commit.authored_datetime

        license = check_output(['make', 'show-var', f'VARNAME=LICENSE'], cwd=cwd).decode().strip()
        if license in RPKG_LICENSES:
            self.license = License.objects.get(spdx_id=RPKG_LICENSES[license])
        else:
            logger.warning(f'Unknown robotpkg license: {license}')
Guilhem Saurel's avatar
Guilhem Saurel committed
707
        self.public = not bool(check_output(['make', 'show-var', f'VARNAME=RESTRICTED'], cwd=cwd).decode().strip())
Guilhem Saurel's avatar
Guilhem Saurel committed
708
709
710
        with (cwd / 'DESCR').open() as f:
            self.description = f.read().strip()

711
        self.update_images()
Guilhem Saurel's avatar
Guilhem Saurel committed
712
        self.save()
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
713

714
    def valid_images(self):
715
        return self.image_set.filter(created__isnull=False, target__active=True).order_by('target__name')
716

717
718
719
720
    def without_py(self):
        if 'py-' in self.name:
            return Robotpkg.objects.filter(name=self.name.replace('py-', '')).first()

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
721

Guilhem Saurel's avatar
Guilhem Saurel committed
722
# class RobotpkgBuild(TimeStampedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
723
724
725
#     robotpkg = models.ForeignKey(Robotpkg, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
#     passed = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
726
727


728
729
class Image(models.Model):
    robotpkg = models.ForeignKey(Robotpkg, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
730
    target = models.ForeignKey(Target, on_delete=models.CASCADE)
731
732
    created = models.DateTimeField(blank=True, null=True)
    image = models.CharField(max_length=12, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
733
    py3 = models.BooleanField(default=False)
734
    debug = models.BooleanField(default=False)
735
736

    class Meta:
737
        unique_together = ('robotpkg', 'target', 'py3', 'debug')
738
739

    def __str__(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
740
741
        py = '-py3' if self.py3 else ''
        return f'{self.robotpkg}{py}:{self.target}'
742
743

    def get_build_args(self):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
744
        ret = {'TARGET': self.target, 'ROBOTPKG': self.robotpkg, 'REGISTRY': self.robotpkg.project.registry()}
Guilhem Saurel's avatar
Guilhem Saurel committed
745
        if not self.robotpkg.project.public:
Guilhem Saurel's avatar
Guilhem Saurel committed
746
747
748
            ret['IMAGE'] = 'robotpkg-jrl-py3' if self.py3 else 'robotpkg-jrl'
        elif self.py3:
            ret['IMAGE'] = 'robotpkg-py3'
Guilhem Saurel's avatar
Guilhem Saurel committed
749
        return ret
750
751
752

    def get_image_name(self):
        project = self.robotpkg.project
753
        return f'{project.registry()}/{project.main_namespace.slug}/{project.slug}/{self}'.lower()
754

755
756
757
    def get_image_url(self):
        project = self.robotpkg.project
        manifest = str(self).replace(':', '/manifests/')
758
        return f'https://{project.registry()}/v2/{project.main_namespace.slug}/{project.slug}/{manifest}'
759

Guilhem Saurel's avatar
Guilhem Saurel committed
760
    def get_job_name(self):
761
762
        mode = 'debug' if self.debug else 'release'
        return f'robotpkg-{self}-{mode}'.replace(':', '-')
Guilhem Saurel's avatar
Guilhem Saurel committed
763

764
765
766
    def build(self):
        args = self.get_build_args()
        build_args = sum((['--build-arg', f'{key}={value}'] for key, value in args.items()), list())
767
768
769
770
771
772
773
        return ['docker', 'build', '-t', self.get_image_name()] + build_args + ['.']

    def pull(self):
        return ['docker', 'pull', self.get_image_name()]

    def push(self):
        return ['docker', 'push', self.get_image_name()]
774

Guilhem Saurel's avatar
Guilhem Saurel committed
775
    def update(self, pull=False):
776
777
778
779
780
        r = requests.get(self.get_image_url())
        if r.status_code == 200:
            self.image = r.json()['fsLayers'][0]['blobSum'].split(':')[1][:12]
            self.created = parse_datetime(json.loads(r.json()['history'][0]['v1Compatibility'])['created'])
            self.save()
781
782


Guilhem Saurel's avatar
Guilhem Saurel committed
783
784
785
786
787
788
789
790
class CIBuild(models.Model):
    repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
    passed = models.NullBooleanField()
    build_id = models.PositiveIntegerField()
    started = models.DateTimeField()
    branch = models.ForeignKey(Branch, on_delete=models.CASCADE)

    class Meta:
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
791
        ordering = ('started', )
Guilhem Saurel's avatar
Guilhem Saurel committed
792
793
794
795
796
797
798
799
800
801
802
803
804
805

    def url(self):
        if self.repo.forge.source == SOURCES.github:
            return f'https://travis-ci.org/{self.repo.namespace.slug}/{self.repo.slug}/builds/{self.build_id}'
        if self.repo.forge.source == SOURCES.gitlab:
            return f'{self.repo.forge.url}/{self.repo.namespace.slug}/{self.repo.slug}/pipelines/{self.build_id}'


class Tag(models.Model):
    name = models.CharField(max_length=200)
    slug = AutoSlugField(populate_from='name', slugify=slugify_with_dots)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)

    class Meta:
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
806
        ordering = ('name', )
Guilhem Saurel's avatar
Guilhem Saurel committed
807
808
        unique_together = ('name', 'project')

Guilhem Saurel's avatar
Guilhem Saurel committed
809
810
811
    def __str__(self):
        return f'{self.project} {self.name}'

Guilhem Saurel's avatar
Guilhem Saurel committed
812

813
814
815
816
817
class GepettistQuerySet(models.QuerySet):
    def gepettist(self):
        return self.filter(projects__from_gepetto=True)


Guilhem Saurel's avatar
Guilhem Saurel committed
818
819
class Contributor(models.Model):
    projects = models.ManyToManyField(Project)
Guilhem Saurel's avatar
Guilhem Saurel committed
820
    agreement_signed = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
821

822
823
    objects = GepettistQuerySet.as_manager()

Guilhem Saurel's avatar
Guilhem Saurel committed
824
825
826
827
828
829
830
831
832
833
834
835
    def __str__(self):
        name = self.contributorname_set.first()
        mail = self.contributormail_set.first()
        return f'{name} <{mail}>'

    def names(self):
        return ', '.join(str(name) for name in self.contributorname_set.all())

    def mails(self):
        return ', '.join(str(mail) for mail in self.contributormail_set.filter(invalid=False))

    def contributed(self):
836
        return ', '.join(str(project) for project in self.projects.filter(from_gepetto=True))
Guilhem Saurel's avatar
Guilhem Saurel committed
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853


class ContributorName(models.Model):
    contributor = models.ForeignKey(Contributor, on_delete=models.CASCADE, blank=True, null=True)
    name = models.CharField(max_length=200, unique=True)

    def __str__(self):
        return self.name


class ContributorMail(models.Model):
    contributor = models.ForeignKey(Contributor, on_delete=models.CASCADE, blank=True, null=True)
    mail = models.EmailField(unique=True)
    invalid = models.BooleanField(default=False)

    def __str__(self):
        return self.mail
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
854
855


Guilhem Saurel's avatar
Guilhem Saurel committed
856
857
858
class Dependency(models.Model):
    project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name='dependencies')
    library = models.ForeignKey(Project, on_delete=models.CASCADE, related_name='rdeps')
859
    robotpkg = models.BooleanField(default=False)  # TODO NYI
Guilhem Saurel's avatar
Guilhem Saurel committed
860
    cmake = models.BooleanField(default=False)
861
    ros = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
862
863
864
865
866
867

    class Meta:
        verbose_name_plural = 'dependencies'
        unique_together = ('project', 'library')

    def __str__(self):
868
        return f'{self.project} depends on {self.library}: {self.robotpkg:d} {self.cmake:d}'
Guilhem Saurel's avatar
Guilhem Saurel committed
869
870


Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
871
872
873
874
875
876
877
878
879
880
881
882
def get_default_forge(project):
    for forge in Forge.objects.order_by('source'):
        if project.repo_set.filter(forge=forge).exists():
            logger.info(f'default forge for {project} set to {forge}')
            project.main_forge = forge
            project.save()
            return forge
    else:
        logger.error(f'NO DEFAULT FORGE for {project}')


def update_gitlab(forge, data):
883
884
    if data['archived']:
        return
885
    if 'default_branch' not in data or data['default_branch'] is None:
886
        return
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
887
    logger.info(f'update {data["name"]} from {forge}')
Guilhem Saurel's avatar
Guilhem Saurel committed
888
    public = data['visibility'] not in ['private', 'internal']
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
889
    project, created = Project.objects.get_or_create(
Guilhem Saurel's avatar
Guilhem Saurel committed
890
        name=valid_name(data['name']), defaults={
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
            'main_forge': forge,
            'public': public
        })
    namespace, _ = Namespace.objects.get_or_create(
        slug=data['namespace']['path'], defaults={'name': data['namespace']['name']})
    repo, _ = Repo.objects.get_or_create(
        forge=forge,
        namespace=namespace,
        project=project,
        defaults={
            'repo_id': data['id'],
            'name': data['name'],
            'url': data['web_url'],
            'default_branch': data['default_branch'],
            'clone_url': data['http_url_to_repo']
        })
907
908
909
910
911
912
913
    repo.name = data['name']
    repo.slug = data['path']
    repo.url = data['web_url']
    repo.repo_id = data['id']
    repo.clone_url = data['http_url_to_repo']
    repo.open_issues = data['open_issues_count']
    repo.default_branch = data['default_branch']
Guilhem Saurel's avatar
Guilhem Saurel committed
914
915
    repo.description = data['description']
    # TODO license (https://gitlab.com/gitlab-org/gitlab-ce/issues/28267), open_pr
916
    if 'forked_from_project' in data and data['forked_from_project'] is not None:
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
917
918
919
920
        repo.forked_from = data['forked_from_project']['id']
    elif created or project.main_namespace is None:
        project.main_namespace = namespace
        project.save()
921
    repo.save()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
922
923
924


def update_github(forge, namespace, data):
925
926
    if data['archived']:
        return
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
927
    logger.info(f'update {data["name"]} from {forge}')
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
928
    project, _ = Project.objects.get_or_create(
Guilhem Saurel's avatar
Guilhem Saurel committed
929
        name=valid_name(data['name']), defaults={
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
930
931
932
933
934
935
936
937
938
939
940
941
942
            'homepage': data['homepage'],
            'main_namespace': namespace,
            'main_forge': forge
        })
    repo, _ = Repo.objects.get_or_create(
        forge=forge,
        namespace=namespace,
        project=project,
        defaults={
            'repo_id': data['id'],
            'name': data['name'],
            'clone_url': data['clone_url']
        })
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
943
944
945
946
947
    repo.homepage = data['homepage']
    repo.url = data['html_url']
    repo.repo_id = data['id']
    repo.default_branch = data['default_branch']
    repo.open_issues = data['open_issues']
Guilhem Saurel's avatar
Guilhem Saurel committed
948
    repo.description = data['description']
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
949
950
951
952

    repo_data = repo.api_data()
    if repo_data and 'license' in repo_data and repo_data['license']:
        if 'spdx_id' in repo_data['license'] and repo_data['license']['spdx_id']:
953
954
955
956
957
958
959
960
            if repo_data['license']['spdx_id'] != 'NOASSERTION':
                try:
                    license = License.objects.get(spdx_id=repo_data['license']['spdx_id'])
                except License.DoesNotExist:
                    raise ValueError('No License with spdx_id=' + repo_data['license']['spdx_id'])
                repo.license = license
                if not project.license:
                    project.license = license
961
962
        if 'source' in repo_data:
            repo.forked_from = repo_data['source']['id']
963
964
    if repo_data:
        repo.open_issues = repo_data['open_issues_count']
965
    repo.clone_url = data['clone_url']
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
966
967
968
    repo.open_pr = len(list(repo.api_list('/pulls')))
    repo.save()
    project.save()
Guilhem Saurel's avatar
Guilhem Saurel committed