models.py 39.6 KB
Newer Older
1
import json
2
import logging
Guilhem Saurel's avatar
Guilhem Saurel committed
3
import re
Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
4
from subprocess import check_output
5

Guilhem Saurel's avatar
Guilhem Saurel committed
6
from django.conf import settings
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
7
from django.db import models
Guilhem Saurel's avatar
Guilhem Saurel committed
8
from django.db.models import Q
Guilhem Saurel's avatar
Guilhem Saurel committed
9
from django.db.models.functions import Length
10
from django.db.utils import DataError
11
from django.template.loader import get_template
Guilhem Saurel's avatar
Guilhem Saurel committed
12
13
from django.utils.dateparse import parse_datetime
from django.utils.safestring import mark_safe
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
14

Guilhem Saurel's avatar
Guilhem Saurel committed
15
import requests
Guilhem Saurel's avatar
Guilhem Saurel committed
16
17

import git
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
18
from autoslug import AutoSlugField
Guilhem Saurel's avatar
Guilhem Saurel committed
19
from ndh.models import Links, NamedModel, TimeStampedModel
Guilhem Saurel's avatar
Guilhem Saurel committed
20
from ndh.utils import enum_to_choices, query_sum
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
21

Guilhem Saurel's avatar
Guilhem Saurel committed
22
from .utils import SOURCES, api_next, invalid_mail, slugify_with_dots, valid_name
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
23

24
25
logger = logging.getLogger('rainboard.models')

Guilhem Saurel's avatar
Guilhem Saurel committed
26
MAIN_BRANCHES = ['master', 'devel']
Guilhem Saurel's avatar
Guilhem Saurel committed
27
RPKG_URL = 'http://robotpkg.openrobots.org'
Guilhem Saurel's avatar
Guilhem Saurel committed
28
DOC_URL = 'http://projects.laas.fr/gepetto/doc'
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
29
30
31
32
33
34
35
36
37
RPKG_LICENSES = {
    'gnu-lgpl-v3': 'LGPL-3.0',
    'gnu-lgpl-v2': 'LGPL-2.0',
    'mit': 'MIT',
    'gnu-gpl-v3': 'GPL-3.0',
    '2-clause-bsd': 'BSD-2-Clause',
    'eclipse': 'EPL-1.0',
    'modified-bsd': 'BSD-3-Clause'
}
Guilhem Saurel's avatar
Guilhem Saurel committed
38
RPKG_FIELDS = ['PKGBASE', 'PKGVERSION', 'MASTER_SITES', 'MASTER_REPOSITORY', 'MAINTAINER', 'COMMENT', 'HOMEPAGE']
Guilhem Saurel's avatar
Guilhem Saurel committed
39
CMAKE_FIELDS = {'NAME': 'cmake_name', 'DESCRIPTION': 'description', 'URL': 'homepage', 'VERSION': 'version'}
Guilhem Saurel's avatar
Guilhem Saurel committed
40
TRAVIS_STATE = {'created': None, 'passed': True, 'started': None, 'failed': False, 'errored': False, 'canceled': False}
Guilhem Saurel's avatar
Guilhem Saurel committed
41
GITLAB_STATUS = {'failed': False, 'success': True, 'pending': None, 'skipped': None, 'canceled': None, 'running': None}
Guilhem Saurel's avatar
Guilhem Saurel committed
42

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
43
44

class Namespace(NamedModel):
45
    group = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
46
47


Guilhem Saurel's avatar
Guilhem Saurel committed
48
49
50
class License(models.Model):
    name = models.CharField(max_length=200)
    spdx_id = models.CharField(max_length=50, unique=True)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
51
52
53
54
55
56
    url = models.URLField(max_length=200)

    def __str__(self):
        return self.spdx_id or self.name


Guilhem Saurel's avatar
Guilhem Saurel committed
57
class Forge(Links, NamedModel):
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
58
59
60
61
62
    source = models.PositiveSmallIntegerField(choices=enum_to_choices(SOURCES))
    url = models.URLField(max_length=200)
    token = models.CharField(max_length=50, blank=True, null=True)
    verify = models.BooleanField(default=True)

Guilhem Saurel's avatar
Guilhem Saurel committed
63
64
65
    def get_absolute_url(self):
        return self.url

Guilhem Saurel's avatar
Guilhem Saurel committed
66
    def api_req(self, url='', name=None, page=1):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
67
        logger.debug(f'requesting api {self} {url}, page {page}')
68
69
70
71
72
        try:
            return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
        except requests.exceptions.ConnectionError:
            logger.error(f'requesting api {self} {url}, page {page} - SECOND TRY')
            return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
Guilhem Saurel's avatar
Guilhem Saurel committed
73
74
75
76
77
78

    def api_data(self, url=''):
        req = self.api_req(url)
        return req.json() if req.status_code == 200 else []  # TODO

    def api_list(self, url='', name=None):
Guilhem Saurel's avatar
Guilhem Saurel committed
79
80
        page = 1
        while page:
Guilhem Saurel's avatar
Guilhem Saurel committed
81
            req = self.api_req(url, name, page)
Guilhem Saurel's avatar
Guilhem Saurel committed
82
83
84
85
86
            if req.status_code != 200:
                return []  # TODO
            data = req.json()
            if name is not None:
                data = data[name]
Guilhem Saurel's avatar
Guilhem Saurel committed
87
            yield from data
Guilhem Saurel's avatar
Guilhem Saurel committed
88
            page = api_next(self.source, req)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
89
90

    def headers(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
91
        return {
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
92
93
94
95
96
97
98
99
100
101
102
103
104
105
            SOURCES.github: {
                'Authorization': f'token {self.token}',
                'Accept': 'application/vnd.github.drax-preview+json'
            },
            SOURCES.gitlab: {
                'Private-Token': self.token
            },
            SOURCES.redmine: {
                'X-Redmine-API-Key': self.token
            },
            SOURCES.travis: {
                'Authorization': f'token {self.token}',
                'TRAVIS-API-Version': '3'
            },
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
106
        }[self.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
107
108

    def api_url(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
109
110
111
112
        return {
            SOURCES.github: 'https://api.github.com',
            SOURCES.gitlab: f'{self.url}/api/v4',
            SOURCES.redmine: self.url,
Guilhem Saurel's avatar
travis    
Guilhem Saurel committed
113
            SOURCES.travis: 'https://api.travis-ci.org',
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
114
        }[self.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
115

116
117
    def get_namespaces_github(self):
        for namespace in Namespace.objects.filter(group=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
118
            for data in self.api_list(f'/orgs/{namespace.slug}/members'):
Guilhem Saurel's avatar
Guilhem Saurel committed
119
120
                Namespace.objects.get_or_create(slug=data['login'].lower(),
                                                defaults={'name': data['login'], 'group': False})
121

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
122
123
    def get_namespaces_gitlab(self):
        for data in self.api_list('/namespaces'):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
124
125
126
127
128
            Namespace.objects.get_or_create(
                slug=data['path'], defaults={
                    'name': data['name'],
                    'group': data['kind'] == 'group'
                })
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
129
130
131
        for data in self.api_list('/users'):
            Namespace.objects.get_or_create(slug=data['username'], defaults={'name': data['name']})

Guilhem Saurel's avatar
Guilhem Saurel committed
132
133
134
    def get_namespaces_redmine(self):
        pass  # TODO

Guilhem Saurel's avatar
Guilhem Saurel committed
135
136
137
    def get_namespaces_travis(self):
        pass

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
138
139
140
    def get_projects(self):
        getattr(self, f'get_namespaces_{self.get_source_display()}')()
        return getattr(self, f'get_projects_{self.get_source_display()}')()
141

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
142
    def get_projects_github(self):
143
        for org in Namespace.objects.filter(group=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
144
            for data in self.api_list(f'/orgs/{org.slug}/repos'):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
145
                update_github(self, org, data)
146
        for user in Namespace.objects.filter(group=False):
Guilhem Saurel's avatar
Guilhem Saurel committed
147
            for data in self.api_list(f'/users/{user.slug}/repos'):
Guilhem Saurel's avatar
Guilhem Saurel committed
148
                if Project.objects.filter(name=valid_name(data['name'])).exists():
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
149
                    update_github(self, user, data)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
150
151

    def get_projects_gitlab(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
152
        for data in self.api_list('/projects'):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
153
            update_gitlab(self, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
154
155

        for orphan in Project.objects.filter(main_namespace=None):
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
156
            repo = orphan.repo_set.filter(forge__source=SOURCES.gitlab).first()
Guilhem Saurel's avatar
Guilhem Saurel committed
157
158
            if repo:
                update_gitlab(self, self.api_data(f'/projects/{repo.forked_from}'))
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
159
160
161
162

    def get_projects_redmine(self):
        pass  # TODO

Guilhem Saurel's avatar
Guilhem Saurel committed
163
164
165
166
167
168
    def get_projects_travis(self):
        for namespace in Namespace.objects.all():
            for repository in self.api_list(f'/owner/{namespace.slug}/repos', 'repositories'):
                if repository['active']:
                    update_travis(namespace, repository)

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
169

Guilhem Saurel's avatar
Guilhem Saurel committed
170
class Project(Links, NamedModel, TimeStampedModel):
Guilhem Saurel's avatar
Guilhem Saurel committed
171
    public = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
172
173
174
175
    main_namespace = models.ForeignKey(Namespace, on_delete=models.SET_NULL, null=True, blank=True)
    main_forge = models.ForeignKey(Forge, on_delete=models.SET_NULL, null=True, blank=True)
    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    homepage = models.URLField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
176
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
177
    version = models.CharField(max_length=20, blank=True, null=True)
178
    updated = models.DateTimeField(blank=True, null=True)
179
180
    tests = models.BooleanField(default=True)
    docs = models.BooleanField(default=True)
181
    debug = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
182
    from_gepetto = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
183
    cmake_name = models.CharField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
184

185
186
187
188
    def save(self, *args, **kwargs):
        self.name = valid_name(self.name)
        super().save(*args, **kwargs)

Guilhem Saurel's avatar
Guilhem Saurel committed
189
    def git_path(self):
190
        return settings.RAINBOARD_GITS / self.main_namespace.slug / self.slug.strip()  # workaround SafeText TypeError
Guilhem Saurel's avatar
Guilhem Saurel committed
191

Guilhem Saurel's avatar
Guilhem Saurel committed
192
    def git(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
193
        path = self.git_path()
Guilhem Saurel's avatar
Guilhem Saurel committed
194
195
196
197
198
        if not path.exists():
            logger.info(f'Creating repo for {self.main_namespace.slug}/{self.slug}')
            return git.Repo.init(path)
        return git.Repo(str(path / '.git'))

199
    def main_repo(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
200
        forge = self.main_forge if self.main_forge else get_default_forge(self)
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
201
202
203
204
205
206
207
208
209
        repo, created = Repo.objects.get_or_create(
            forge=forge,
            namespace=self.main_namespace,
            project=self,
            defaults={
                'name': self.name,
                'default_branch': 'master',
                'repo_id': 0
            })
Guilhem Saurel's avatar
Guilhem Saurel committed
210
        if created:
211
            repo.api_update()
Guilhem Saurel's avatar
Guilhem Saurel committed
212
213
        return repo

Guilhem Saurel's avatar
Guilhem Saurel committed
214
    def update_branches(self, main=True, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
215
216
217
        branches = [b[2:] for b in self.git().git.branch('-a', '--no-color').split('\n')]
        if main:
            branches = [b for b in branches if b.endswith('master') or b.endswith('devel')]
Guilhem Saurel's avatar
Guilhem Saurel committed
218
        for branch in branches:
Guilhem Saurel's avatar
Guilhem Saurel committed
219
            logger.info(f'update branch {branch}')
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
220
221
            if branch.startswith('remotes/'):
                branch = branch[8:]
222
223
224
            if branch.count('/') < 2:
                logger.error(f'wrong branch "{branch}" in {self.git_path()}')
                continue
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
            forge, namespace, name = branch.split('/', maxsplit=2)
            namespace, _ = Namespace.objects.get_or_create(slug=namespace)
            forge = Forge.objects.get(slug=forge)
            repo, created = Repo.objects.get_or_create(
                forge=forge,
                namespace=namespace,
                project=self,
                defaults={
                    'name': self.name,
                    'default_branch': 'master',
                    'repo_id': 0
                })
            if created:
                repo.api_update()
            instance, bcreated = Branch.objects.get_or_create(name=branch, project=self, repo=repo)
Guilhem Saurel's avatar
Guilhem Saurel committed
240
            if bcreated:
Guilhem Saurel's avatar
Guilhem Saurel committed
241
                instance.update(pull=pull)
242

Guilhem Saurel's avatar
update    
Guilhem Saurel committed
243
244
245
    def checkout(self):
        self.main_branch().git().checkout()

Guilhem Saurel's avatar
Guilhem Saurel committed
246
    def main_branch(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
247
        return self.main_repo().main_branch()
Guilhem Saurel's avatar
Guilhem Saurel committed
248

Guilhem Saurel's avatar
Guilhem Saurel committed
249
250
251
252
253
254
255
    def cmake(self):
        filename = self.git_path() / 'CMakeLists.txt'
        if not filename.exists():
            return
        with filename.open() as f:
            content = f.read()
        for key, value in CMAKE_FIELDS.items():
Guilhem Saurel's avatar
Guilhem Saurel committed
256
            search = re.search(fr'set\s*\(\s*project_{key}\s+([^)]+)*\)', content, re.I)
Guilhem Saurel's avatar
Guilhem Saurel committed
257
            if search:
258
                try:
259
                    old = getattr(self, value)
260
                    new = search.groups()[0].strip(''' \r\n\t'"''').replace('_', '-')
261
262
263
                    if old != new:
                        setattr(self, value, new)
                        self.save()
264
                except DataError:
265
                    setattr(self, value, old)
266
        for dependency in re.findall(r'ADD_[A-Z]+_DEPENDENCY\s*\(["\']?([^ "\')]+).*["\']?\)', content, re.I):
Guilhem Saurel's avatar
Guilhem Saurel committed
267
            project = Project.objects.filter(name=valid_name(dependency))
268
            if project.exists():
Guilhem Saurel's avatar
oops    
Guilhem Saurel committed
269
270
                dependency, _ = Dependency.objects.get_or_create(project=self, library=project.first())
                if not dependency.cmake:
271
272
                    dependency.cmake = True
                    dependency.save()
Guilhem Saurel's avatar
Guilhem Saurel committed
273

274
    def ros(self):
275
276
        try:
            filename = self.git_path() / 'package.xml'
277
        except TypeError:
278
            return
279
280
281
282
283
        if not filename.exists():
            return
        with filename.open() as f:
            content = f.read()
        for dependency in re.findall(r'<run_depend>(\w+).*</run_depend>', content, re.I):
Guilhem Saurel's avatar
Guilhem Saurel committed
284
            project = Project.objects.filter(name=valid_name(dependency))
285
286
287
288
289
290
            if project.exists():
                dependency, _ = Dependency.objects.get_or_create(project=self, library=project.first())
                if not dependency.ros:
                    dependency.ros = True
                    dependency.save()

291
292
293
294
295
296
    def repos(self):
        return self.repo_set.count()

    def rpkgs(self):
        return self.robotpkg_set.count()

Guilhem Saurel's avatar
Guilhem Saurel committed
297
298
299
300
    def update_tags(self):
        for tag in self.git().tags:
            Tag.objects.get_or_create(name=str(tag), project=self)

301
    def update(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
302
        self.update_branches()
Guilhem Saurel's avatar
Guilhem Saurel committed
303
        self.update_tags()
304
        tag = self.tag_set.filter(name__startswith='v').last()  # TODO: implement SQL ordering for semver
Guilhem Saurel's avatar
Guilhem Saurel committed
305
306
        if tag is not None:
            self.version = tag.name[1:]
307
        robotpkg = self.robotpkg_set.order_by('-updated').first()
Guilhem Saurel's avatar
Guilhem Saurel committed
308
        branch = self.branch_set.order_by('-updated').first()
Guilhem Saurel's avatar
Guilhem Saurel committed
309
        if (branch is not None and branch.updated is not None) or robotpkg is not None:
Guilhem Saurel's avatar
Guilhem Saurel committed
310
311
            if robotpkg is None:
                self.updated = branch.updated
312
            elif branch is None or branch.updated is None:
Guilhem Saurel's avatar
Guilhem Saurel committed
313
314
315
                self.updated = robotpkg.updated
            else:
                self.updated = max(branch.updated, robotpkg.updated)
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
316
        self.checkout()
317
        self.cmake()
Guilhem Saurel's avatar
Guilhem Saurel committed
318
        self.ros()
319
320
        self.save()

Guilhem Saurel's avatar
Guilhem Saurel committed
321
    def commits_since(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
322
323
324
325
326
        try:
            commits = self.git().git.rev_list(f'v{self.version}..{self.main_branch()}')
            return len(commits.split('\n')) if commits else 0
        except git.exc.GitCommandError:
            pass
Guilhem Saurel's avatar
Guilhem Saurel committed
327

328
329
330
331
332
333
    def open_issues(self):
        return query_sum(self.repo_set, 'open_issues')

    def open_pr(self):
        return query_sum(self.repo_set, 'open_pr')

334
335
336
    def gitlabciyml(self):
        return get_template('rainboard/gitlab-ci.yml').render({'project': self})

Guilhem Saurel's avatar
Guilhem Saurel committed
337
338
339
340
341
342
343
344
345
    def contributors(self, update=False):
        if update:
            for guy in self.git().git.shortlog('-nse').split('\n'):
                name, mail = guy[7:-1].split(' <')
                contributor = get_contributor(name, mail)
                contributor.projects.add(self)
                contributor.save()
        return self.contributor_set.all()

346
347
348
    def registry(self):
        return settings.PUBLIC_REGISTRY if self.public else settings.PRIVATE_REGISTRY

Guilhem Saurel's avatar
Guilhem Saurel committed
349
350
351
352
    def doc_coverage_image(self):
        images = Image.objects.filter(robotpkg__project=self, py3=False, target__name='16.04')
        return images.order_by(Length('robotpkg__name').desc()).first()

353
    def print_deps(self):
354
        return mark_safe(', '.join(d.library.get_link() for d in self.dependencies.all()))
355
356

    def print_rdeps(self):
357
        return mark_safe(', '.join(d.project.get_link() for d in self.rdeps.all()))
358

Guilhem Saurel's avatar
Guilhem Saurel committed
359
360
361
    def ordered_robotpkg(self):
        return self.robotpkg_set.order_by('name')

Guilhem Saurel's avatar
Guilhem Saurel committed
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
    def url_travis(self):
        return f'https://travis-ci.org/{self.main_namespace.slug}/{self.slug}'

    def url_gitlab(self):
        return f'https://gepgitlab.laas.fr/{self.main_namespace.slug}/{self.slug}'

    def badge(self, link, img, alt):
        return mark_safe(f'<a href="{link}"><img src="{img}" alt="{alt}" /></a> ')

    def badge_travis(self):
        return self.badge(self.url_travis(), f'{self.url_travis()}.svg?branch=master', 'Building Status')

    def badge_gitlab(self):
        return self.badge(self.url_gitlab(), f'{self.url_gitlab()}/badges/master/pipeline.svg', 'Pipeline Status')

    def badge_coverage(self):
        return self.badge(f'{DOC_URL}/{self.main_namespace.slug}/{self.slug}/master/coverage',
Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
379
                          f'{self.url_gitlab()}/badges/master/coverage.svg?job=doc-coverage"', 'Coverage Report')
Guilhem Saurel's avatar
Guilhem Saurel committed
380
381
382
383

    def badges(self):
        return self.badge_travis() + self.badge_gitlab() + self.badge_coverage()

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
384

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
385
386
class Repo(TimeStampedModel):
    name = models.CharField(max_length=200)
Guilhem Saurel's avatar
Guilhem Saurel committed
387
    slug = AutoSlugField(populate_from='name', slugify=slugify_with_dots)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
388
389
390
391
392
393
394
395
396
397
    forge = models.ForeignKey(Forge, on_delete=models.CASCADE)
    namespace = models.ForeignKey(Namespace, on_delete=models.CASCADE)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    homepage = models.URLField(max_length=200, blank=True, null=True)
    url = models.URLField(max_length=200, blank=True, null=True)
    default_branch = models.CharField(max_length=50)
    open_issues = models.PositiveSmallIntegerField(blank=True, null=True)
    open_pr = models.PositiveSmallIntegerField(blank=True, null=True)
    repo_id = models.PositiveIntegerField()
Guilhem Saurel's avatar
Guilhem Saurel committed
398
    forked_from = models.PositiveIntegerField(blank=True, null=True)
399
    clone_url = models.URLField(max_length=200)
Guilhem Saurel's avatar
travis    
Guilhem Saurel committed
400
    travis_id = models.PositiveIntegerField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
401
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
402
403
404

    def __str__(self):
        return self.name
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
405
406

    def api_url(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
407
        api_url = self.forge.api_url()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
408
        return {
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
409
410
411
            SOURCES.github: f'{api_url}/repos/{self.namespace.slug}/{self.slug}',
            SOURCES.redmine: f'{api_url}/projects/{self.repo_id}.json',
            SOURCES.gitlab: f'{api_url}/projects/{self.repo_id}',
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
412
        }[self.forge.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
413

Guilhem Saurel's avatar
Guilhem Saurel committed
414
    def api_req(self, url='', name=None, page=1):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
415
        logger.debug(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}')
416
        try:
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
417
418
            return requests.get(
                self.api_url() + url, {'page': page}, verify=self.forge.verify, headers=self.forge.headers())
419
420
        except requests.exceptions.ConnectionError:
            logger.error(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page} - SECOND TRY')
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
421
422
            return requests.get(
                self.api_url() + url, {'page': page}, verify=self.forge.verify, headers=self.forge.headers())
Guilhem Saurel's avatar
Guilhem Saurel committed
423
424
425
426
427
428

    def api_data(self, url=''):
        req = self.api_req(url)
        return req.json() if req.status_code == 200 else []  # TODO

    def api_list(self, url='', name=None):
Guilhem Saurel's avatar
Guilhem Saurel committed
429
430
        page = 1
        while page:
Guilhem Saurel's avatar
Guilhem Saurel committed
431
            req = self.api_req(url, name, page)
Guilhem Saurel's avatar
Guilhem Saurel committed
432
433
434
435
            if req.status_code != 200:
                return []  # TODO
            data = req.json()
            if name is not None:
Guilhem Saurel's avatar
Guilhem Saurel committed
436
437
438
439
                if name in data:
                    data = data[name]
                else:
                    return []  # TODO
Guilhem Saurel's avatar
Guilhem Saurel committed
440
            yield from data
Guilhem Saurel's avatar
Guilhem Saurel committed
441
            page = api_next(self.forge.source, req)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
442

Guilhem Saurel's avatar
Guilhem Saurel committed
443
    def api_update(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
444
        data = self.api_data()
Guilhem Saurel's avatar
Guilhem Saurel committed
445
        if data:
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
446
            return getattr(self, f'api_update_{self.forge.get_source_display()}')(data)
Guilhem Saurel's avatar
Guilhem Saurel committed
447
448

    def api_update_gitlab(self, data):
449
        update_gitlab(self.forge, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
450

Guilhem Saurel's avatar
Guilhem Saurel committed
451
    def api_update_github(self, data):
452
        update_github(self.forge, self.namespace, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
453

Guilhem Saurel's avatar
git    
Guilhem Saurel committed
454
455
456
    def get_clone_url(self):
        if self.forge.source == SOURCES.gitlab:
            return self.clone_url.replace('://', f'://gitlab-ci-token:{self.forge.token}@')
457
458
        if self.forge.source == SOURCES.github:
            return self.clone_url.replace('://', f'://{settings.GITHUB_USER}:{self.forge.token}@')
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
459
460
        return self.clone_url

461
462
463
    def git_remote(self):
        return f'{self.forge.slug}/{self.namespace.slug}'

Guilhem Saurel's avatar
git    
Guilhem Saurel committed
464
    def git(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
465
        git_repo = self.project.git()
466
        remote = self.git_remote()
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
467
        try:
Guilhem Saurel's avatar
Guilhem Saurel committed
468
            return git_repo.remote(remote)
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
469
        except ValueError:
Guilhem Saurel's avatar
Guilhem Saurel committed
470
            logger.info(f'Creating remote {remote}')
Guilhem Saurel's avatar
Guilhem Saurel committed
471
            return git_repo.create_remote(remote, self.get_clone_url())
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
472

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
473
    def fetch(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
474
475
        git_repo = self.git()
        logger.debug(f'fetching {self.forge} / {self.namespace} / {self.project}')
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
476
        try:
Guilhem Saurel's avatar
Guilhem Saurel committed
477
            git_repo.fetch()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
478
        except git.exc.GitCommandError:
Guilhem Saurel's avatar
Guilhem Saurel committed
479
            logger.warning(f'fetching {self.forge} / {self.namespace} / {self.project} - SECOND TRY')
480
481
482
483
484
            try:
                git_repo.fetch()
            except git.exc.GitCommandError:
                return False
        return True
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
485

Guilhem Saurel's avatar
Guilhem Saurel committed
486
487
488
    def main_branch(self):
        return self.project.branch_set.get(name=f'{self.git_remote()}/{self.default_branch}')

Guilhem Saurel's avatar
Guilhem Saurel committed
489
    def ahead(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
490
491
        main_branch = self.main_branch()
        return main_branch.ahead if main_branch is not None else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
492
493

    def behind(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
494
495
        main_branch = self.main_branch()
        return main_branch.behind if main_branch is not None else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
496

Guilhem Saurel's avatar
Guilhem Saurel committed
497
498
499
500
    def get_builds(self):
        return getattr(self, f'get_builds_{self.forge.get_source_display()}')()

    def get_builds_gitlab(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
501
        for pipeline in self.api_list('/pipelines'):
Guilhem Saurel's avatar
Guilhem Saurel committed
502
503
504
            pid, ref = pipeline['id'], pipeline['ref']
            if self.project.tag_set.filter(name=ref).exists():
                continue
Guilhem Saurel's avatar
Guilhem Saurel committed
505
            data = self.api_data(f'/pipelines/{pid}')
Guilhem Saurel's avatar
Guilhem Saurel committed
506
507
508
509
            branch_name = f'{self.forge.slug}/{self.namespace.slug}/{ref}'
            branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
            if created:
                branch.update()
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
510
511
512
513
514
515
516
517
            CIBuild.objects.get_or_create(
                repo=self,
                build_id=pid,
                defaults={
                    'passed': GITLAB_STATUS[pipeline['status']],
                    'started': parse_datetime(data['created_at']),
                    'branch': branch,
                })
Guilhem Saurel's avatar
Guilhem Saurel committed
518
519
520
521

    def get_builds_github(self):
        if self.travis_id is not None:
            travis = Forge.objects.get(source=SOURCES.travis)
Guilhem Saurel's avatar
Guilhem Saurel committed
522
            for build in travis.api_list(f'/repo/{self.travis_id}/builds', name='builds'):
Guilhem Saurel's avatar
Guilhem Saurel committed
523
524
525
526
527
                if self.project.tag_set.filter(name=build['branch']['name']).exists():
                    continue
                branch_name = f'{self.forge.slug}/{self.namespace.slug}/{build["branch"]["name"]}'
                branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
                if created:
Guilhem Saurel's avatar
Guilhem Saurel committed
528
                    branch.update()
Guilhem Saurel's avatar
Guilhem Saurel committed
529
                started = build['started_at'] if build['started_at'] is not None else build['finished_at']
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
530
531
532
533
534
535
536
537
                CIBuild.objects.get_or_create(
                    repo=self,
                    build_id=build['id'],
                    defaults={
                        'passed': TRAVIS_STATE[build['state']],
                        'started': parse_datetime(started),
                        'branch': branch,
                    })
Guilhem Saurel's avatar
Guilhem Saurel committed
538
539

    def update(self, pull=True):
540
        ok = True
Guilhem Saurel's avatar
Guilhem Saurel committed
541
542
        self.project.update_tags()
        if pull:
543
544
545
546
547
548
549
            ok = self.fetch()
        if ok:
            self.api_update()
            self.get_builds()
        else:
            logger.error(f'fetching {self.forge} / {self.namespace} / {self.project} - NOT FOUND - DELETING')
            logger.error(str(self.delete()))
Guilhem Saurel's avatar
Guilhem Saurel committed
550

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
551
552
553
554

class Commit(NamedModel, TimeStampedModel):
    project = models.ForeignKey(Project, on_delete=models.CASCADE)

Guilhem Saurel's avatar
Guilhem Saurel committed
555
556
557
    class Meta:
        unique_together = ('project', 'name')

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
558

Guilhem Saurel's avatar
Guilhem Saurel committed
559
560
561
562
563
class Branch(TimeStampedModel):
    name = models.CharField(max_length=200)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    ahead = models.PositiveSmallIntegerField(blank=True, null=True)
    behind = models.PositiveSmallIntegerField(blank=True, null=True)
564
    updated = models.DateTimeField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
565
    repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
566
    deleted = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
567
    keep_doc = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
568
569

    def __str__(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
570
571
572
        return self.name

    class Meta:
Guilhem Saurel's avatar
Guilhem Saurel committed
573
        unique_together = ('project', 'name', 'repo')
Guilhem Saurel's avatar
Guilhem Saurel committed
574
575

    def get_ahead(self, branch='master'):
Guilhem Saurel's avatar
Guilhem Saurel committed
576
577
        commits = self.project.git().git.rev_list(f'{branch}..{self}')
        return len(commits.split('\n')) if commits else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
578
579

    def get_behind(self, branch='master'):
Guilhem Saurel's avatar
Guilhem Saurel committed
580
581
        commits = self.project.git().git.rev_list(f'{self}..{branch}')
        return len(commits.split('\n')) if commits else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
582

583
584
585
    def git(self):
        git_repo = self.project.git()
        if self.name not in git_repo.branches:
Guilhem Saurel's avatar
Guilhem Saurel committed
586
            remote = self.repo.git()
587
588
589
590
591
            _, _, branch = self.name.split('/', maxsplit=2)
            git_repo.create_head(self.name, remote.refs[branch]).set_tracking_branch(remote.refs[branch])
        return git_repo.branches[self.name]

    def update(self, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
592
593
594
595
596
597
598
        if self.deleted:
            return
        try:
            if pull:
                self.repo.fetch()
                if self.repo != self.project.main_repo():
                    self.project.main_repo().fetch()
Guilhem Saurel's avatar
details    
Guilhem Saurel committed
599
600
            try:
                main_branch = self.project.main_branch()
Guilhem Saurel's avatar
Guilhem Saurel committed
601
602
                self.ahead = self.get_ahead(main_branch)
                self.behind = self.get_behind(main_branch)
Guilhem Saurel's avatar
typo    
Guilhem Saurel committed
603
            except Branch.DoesNotExist:
Guilhem Saurel's avatar
details    
Guilhem Saurel committed
604
                pass
Guilhem Saurel's avatar
Guilhem Saurel committed
605
606
607
            self.updated = self.git().commit.authored_datetime
        except (git.exc.GitCommandError, IndexError):
            self.deleted = True
Guilhem Saurel's avatar
Guilhem Saurel committed
608
        self.save()
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
609

Guilhem Saurel's avatar
Guilhem Saurel committed
610
611
612
613
614
615
616
    def ci(self):
        build = self.cibuild_set.last()
        if build is None:
            return ''
        status = {True: '✓', False: '✗', None: '?'}[build.passed]
        return mark_safe(f'<a href="{build.url()}">{status}</a>')

617
    def forge(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
618
        return self.repo.forge
619
620

    def namespace(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
621
        return self.repo.namespace
622

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
623

Guilhem Saurel's avatar
Guilhem Saurel committed
624
625
626
627
628
class ActiveQuerySet(models.QuerySet):
    def active(self):
        return self.filter(active=True)


Guilhem Saurel's avatar
Guilhem Saurel committed
629
class Target(NamedModel):
Guilhem Saurel's avatar
Guilhem Saurel committed
630
631
632
    active = models.BooleanField(default=True)

    objects = ActiveQuerySet.as_manager()
Guilhem Saurel's avatar
Guilhem Saurel committed
633
634
635


# class Test(TimeStampedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
636
637
638
639
640
641
642
#     project = models.ForeignKey(Project, on_delete=models.CASCADE)
#     branch = models.ForeignKey(Branch, on_delete=models.CASCADE)
#     commit = models.ForeignKey(Commit, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
#     passed = models.BooleanField(default=False)
#     TODO: travis vs gitlab-ci ?
#     TODO: deploy binary, doc, coverage, lint
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
643

Guilhem Saurel's avatar
Guilhem Saurel committed
644
# class SystemDependency(NamedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
645
646
#     project = models.ForeignKey(Project, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
647
648


Guilhem Saurel's avatar
Guilhem Saurel committed
649
class Robotpkg(NamedModel):
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
650
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
651
    category = models.CharField(max_length=50)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
652

Guilhem Saurel's avatar
Guilhem Saurel committed
653
654
655
656
657
658
    pkgbase = models.CharField(max_length=50, default='')
    pkgversion = models.CharField(max_length=20, default='')
    master_sites = models.CharField(max_length=200, default='')
    master_repository = models.CharField(max_length=200, default='')
    maintainer = models.CharField(max_length=200, default='')
    comment = models.TextField()
659
    homepage = models.URLField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
660
661

    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
662
    public = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
663
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
664
665
666
667
668
669
670
671
672
673
    updated = models.DateTimeField(blank=True, null=True)

    def main_page(self):
        if self.category != 'wip':
            return f'{RPKG_URL}/robotpkg/{self.category}/{self.name}'

    def build_page(self):
        path = '-wip/wip' if self.category == 'wip' else f'/{self.category}'
        return f'{RPKG_URL}/rbulk/robotpkg{path}/{self.name}'

674
    def update_images(self):
675
676
        py3s = [False, True] if self.name.startswith('py-') else [False]
        debugs = [False, True] if self.project.debug else [False]
Guilhem Saurel's avatar
Guilhem Saurel committed
677
        for target in Target.objects.active():
678
679
680
            for py3 in py3s:
                for debug in debugs:
                    Image.objects.get_or_create(robotpkg=self, target=target, py3=py3, debug=debug)[0].update()
Guilhem Saurel's avatar
Guilhem Saurel committed
681

682
    def update(self, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
683
684
685
686
687
688
        path = settings.RAINBOARD_RPKG
        repo = git.Repo(str(path / 'wip' / '.git' if self.category == 'wip' else path / '.git'))
        if pull:
            repo.remotes.origin.pull()

        cwd = path / self.category / self.name
Guilhem Saurel's avatar
Guilhem Saurel committed
689
690
691
        if not cwd.is_dir():
            logger.warning(f'deleted {self}: {self.delete()}')
            return
Guilhem Saurel's avatar
Guilhem Saurel committed
692
693
694
695
696
697
698
699
700
701
702
703
704
        for field in RPKG_FIELDS:
            cmd = ['make', 'show-var', f'VARNAME={field}']
            self.__dict__[field.lower()] = check_output(cmd, cwd=cwd).decode().strip()

        repo_path = self.name if self.category == 'wip' else f'{self.category}/{self.name}'
        last_commit = next(repo.iter_commits(paths=repo_path, max_count=1))
        self.updated = last_commit.authored_datetime

        license = check_output(['make', 'show-var', f'VARNAME=LICENSE'], cwd=cwd).decode().strip()
        if license in RPKG_LICENSES:
            self.license = License.objects.get(spdx_id=RPKG_LICENSES[license])
        else:
            logger.warning(f'Unknown robotpkg license: {license}')
Guilhem Saurel's avatar
Guilhem Saurel committed
705
        self.public = not bool(check_output(['make', 'show-var', f'VARNAME=RESTRICTED'], cwd=cwd).decode().strip())
Guilhem Saurel's avatar
Guilhem Saurel committed
706
707
708
        with (cwd / 'DESCR').open() as f:
            self.description = f.read().strip()

709
        self.update_images()
Guilhem Saurel's avatar
Guilhem Saurel committed
710
        self.save()
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
711

712
    def valid_images(self):
713
        return self.image_set.filter(created__isnull=False, target__active=True).order_by('target__name')
714

715
716
717
718
    def without_py(self):
        if 'py-' in self.name:
            return Robotpkg.objects.filter(name=self.name.replace('py-', '')).first()

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
719

Guilhem Saurel's avatar
Guilhem Saurel committed
720
# class RobotpkgBuild(TimeStampedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
721
722
723
#     robotpkg = models.ForeignKey(Robotpkg, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
#     passed = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
724
725


726
727
class Image(models.Model):
    robotpkg = models.ForeignKey(Robotpkg, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
728
    target = models.ForeignKey(Target, on_delete=models.CASCADE)
729
730
    created = models.DateTimeField(blank=True, null=True)
    image = models.CharField(max_length=12, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
731
    py3 = models.BooleanField(default=False)
732
    debug = models.BooleanField(default=False)
733
734

    class Meta:
735
        unique_together = ('robotpkg', 'target', 'py3', 'debug')
736
737

    def __str__(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
738
739
        py = '-py3' if self.py3 else ''
        return f'{self.robotpkg}{py}:{self.target}'
740
741

    def get_build_args(self):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
742
        ret = {'TARGET': self.target, 'ROBOTPKG': self.robotpkg, 'REGISTRY': self.robotpkg.project.registry()}
Guilhem Saurel's avatar
Guilhem Saurel committed
743
        if not self.robotpkg.project.public:
Guilhem Saurel's avatar
Guilhem Saurel committed
744
745
746
            ret['IMAGE'] = 'robotpkg-jrl-py3' if self.py3 else 'robotpkg-jrl'
        elif self.py3:
            ret['IMAGE'] = 'robotpkg-py3'
Guilhem Saurel's avatar
Guilhem Saurel committed
747
        return ret
748
749
750

    def get_image_name(self):
        project = self.robotpkg.project
751
        return f'{project.registry()}/{project.main_namespace.slug}/{project.slug}/{self}'.lower()
752

753
754
755
    def get_image_url(self):
        project = self.robotpkg.project
        manifest = str(self).replace(':', '/manifests/')
756
        return f'https://{project.registry()}/v2/{project.main_namespace.slug}/{project.slug}/{manifest}'
757

Guilhem Saurel's avatar
Guilhem Saurel committed
758
    def get_job_name(self):
759
760
        mode = 'debug' if self.debug else 'release'
        return f'robotpkg-{self}-{mode}'.replace(':', '-')
Guilhem Saurel's avatar
Guilhem Saurel committed
761

762
763
764
    def build(self):
        args = self.get_build_args()
        build_args = sum((['--build-arg', f'{key}={value}'] for key, value in args.items()), list())
765
766
767
768
769
770
771
        return ['docker', 'build', '-t', self.get_image_name()] + build_args + ['.']

    def pull(self):
        return ['docker', 'pull', self.get_image_name()]

    def push(self):
        return ['docker', 'push', self.get_image_name()]
772

Guilhem Saurel's avatar
Guilhem Saurel committed
773
    def update(self, pull=False):
774
775
776
777
778
        r = requests.get(self.get_image_url())
        if r.status_code == 200:
            self.image = r.json()['fsLayers'][0]['blobSum'].split(':')[1][:12]
            self.created = parse_datetime(json.loads(r.json()['history'][0]['v1Compatibility'])['created'])
            self.save()
779
780


Guilhem Saurel's avatar
Guilhem Saurel committed
781
782
783
784
785
786
787
788
class CIBuild(models.Model):
    repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
    passed = models.NullBooleanField()
    build_id = models.PositiveIntegerField()
    started = models.DateTimeField()
    branch = models.ForeignKey(Branch, on_delete=models.CASCADE)

    class Meta:
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
789
        ordering = ('started', )
Guilhem Saurel's avatar
Guilhem Saurel committed
790
791
792
793
794
795
796
797
798
799
800
801
802
803

    def url(self):
        if self.repo.forge.source == SOURCES.github:
            return f'https://travis-ci.org/{self.repo.namespace.slug}/{self.repo.slug}/builds/{self.build_id}'
        if self.repo.forge.source == SOURCES.gitlab:
            return f'{self.repo.forge.url}/{self.repo.namespace.slug}/{self.repo.slug}/pipelines/{self.build_id}'


class Tag(models.Model):
    name = models.CharField(max_length=200)
    slug = AutoSlugField(populate_from='name', slugify=slugify_with_dots)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)

    class Meta:
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
804
        ordering = ('name', )
Guilhem Saurel's avatar
Guilhem Saurel committed
805
806
        unique_together = ('name', 'project')

Guilhem Saurel's avatar
Guilhem Saurel committed
807
808
809
    def __str__(self):
        return f'{self.project} {self.name}'

Guilhem Saurel's avatar
Guilhem Saurel committed
810

811
812
813
814
815
class GepettistQuerySet(models.QuerySet):
    def gepettist(self):
        return self.filter(projects__from_gepetto=True)


Guilhem Saurel's avatar
Guilhem Saurel committed
816
817
class Contributor(models.Model):
    projects = models.ManyToManyField(Project)
Guilhem Saurel's avatar
Guilhem Saurel committed
818
    agreement_signed = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
819

820
821
    objects = GepettistQuerySet.as_manager()

Guilhem Saurel's avatar
Guilhem Saurel committed
822
823
824
825
826
827
828
829
830
831
832
833
    def __str__(self):
        name = self.contributorname_set.first()
        mail = self.contributormail_set.first()
        return f'{name} <{mail}>'

    def names(self):
        return ', '.join(str(name) for name in self.contributorname_set.all())

    def mails(self):
        return ', '.join(str(mail) for mail in self.contributormail_set.filter(invalid=False))

    def contributed(self):
834
        return ', '.join(str(project) for project in self.projects.filter(from_gepetto=True))
Guilhem Saurel's avatar
Guilhem Saurel committed
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851


class ContributorName(models.Model):
    contributor = models.ForeignKey(Contributor, on_delete=models.CASCADE, blank=True, null=True)
    name = models.CharField(max_length=200, unique=True)

    def __str__(self):
        return self.name


class ContributorMail(models.Model):
    contributor = models.ForeignKey(Contributor, on_delete=models.CASCADE, blank=True, null=True)
    mail = models.EmailField(unique=True)
    invalid = models.BooleanField(default=False)

    def __str__(self):
        return self.mail
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
852
853


Guilhem Saurel's avatar
Guilhem Saurel committed
854
855
856
class Dependency(models.Model):
    project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name='dependencies')
    library = models.ForeignKey(Project, on_delete=models.CASCADE, related_name='rdeps')
857
    robotpkg = models.BooleanField(default=False)  # TODO NYI
Guilhem Saurel's avatar
Guilhem Saurel committed
858
    cmake = models.BooleanField(default=False)
859
    ros = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
860
861
862
863
864
865

    class Meta:
        verbose_name_plural = 'dependencies'
        unique_together = ('project', 'library')

    def __str__(self):
866
        return f'{self.project} depends on {self.library}: {self.robotpkg:d} {self.cmake:d}'
Guilhem Saurel's avatar
Guilhem Saurel committed
867
868


Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
869
870
871
872
873
874
875
876
877
878
879
880
def get_default_forge(project):
    for forge in Forge.objects.order_by('source'):
        if project.repo_set.filter(forge=forge).exists():
            logger.info(f'default forge for {project} set to {forge}')
            project.main_forge = forge
            project.save()
            return forge
    else:
        logger.error(f'NO DEFAULT FORGE for {project}')


def update_gitlab(forge, data):
881
882
    if data['archived']:
        return
883
    if 'default_branch' not in data or data['default_branch'] is None:
884
        return
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
885
    logger.info(f'update {data["name"]} from {forge}')
Guilhem Saurel's avatar
Guilhem Saurel committed
886
    public = data['visibility'] not in ['private', 'internal']
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
887
    project, created = Project.objects.get_or_create(
Guilhem Saurel's avatar
Guilhem Saurel committed
888
        name=valid_name(data['name']), defaults={
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
            'main_forge': forge,
            'public': public
        })
    namespace, _ = Namespace.objects.get_or_create(
        slug=data['namespace']['path'], defaults={'name': data['namespace']['name']})
    repo, _ = Repo.objects.get_or_create(
        forge=forge,
        namespace=namespace,
        project=project,
        defaults={
            'repo_id': data['id'],
            'name': data['name'],
            'url': data['web_url'],
            'default_branch': data['default_branch'],
            'clone_url': data['http_url_to_repo']
        })
905
906
907
908
909
910
911
    repo.name = data['name']
    repo.slug = data['path']
    repo.url = data['web_url']
    repo.repo_id = data['id']
    repo.clone_url = data['http_url_to_repo']
    repo.open_issues = data['open_issues_count']
    repo.default_branch = data['default_branch']
Guilhem Saurel's avatar
Guilhem Saurel committed
912
913
    repo.description = data['description']
    # TODO license (https://gitlab.com/gitlab-org/gitlab-ce/issues/28267), open_pr
914
    if 'forked_from_project' in data and data['forked_from_project'] is not None:
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
915
916
917
918
        repo.forked_from = data['forked_from_project']['id']
    elif created or project.main_namespace is None:
        project.main_namespace = namespace
        project.save()
919
    repo.save()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
920
921
922


def update_github(forge, namespace, data):
923
924
    if data['archived']:
        return
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
925
    logger.info(f'update {data["name"]} from {forge}')
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
926
    project, _ = Project.objects.get_or_create(
Guilhem Saurel's avatar
Guilhem Saurel committed
927
        name=valid_name(data['name']), defaults={
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
928
929
930
931
932
933
934
935
936
937
938
939
940
            'homepage': data['homepage'],
            'main_namespace': namespace,
            'main_forge': forge
        })
    repo, _ = Repo.objects.get_or_create(
        forge=forge,
        namespace=namespace,
        project=project,
        defaults={
            'repo_id': data['id'],
            'name': data['name'],
            'clone_url': data['clone_url']
        })
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
941
942
943
944
945
    repo.homepage = data['homepage']
    repo.url = data['html_url']
    repo.repo_id = data['id']
    repo.default_branch = data['default_branch']
    repo.open_issues = data['open_issues']
Guilhem Saurel's avatar
Guilhem Saurel committed
946
    repo.description = data['description']
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
947
948
949
950

    repo_data = repo.api_data()
    if repo_data and 'license' in repo_data and repo_data['license']:
        if 'spdx_id' in repo_data['license'] and repo_data['license']['spdx_id']:
951
952
953
954
955
956
957
958
            if repo_data['license']['spdx_id'] != 'NOASSERTION':
                try:
                    license = License.objects.get(spdx_id=repo_data['license']['spdx_id'])
                except License.DoesNotExist:
                    raise ValueError('No License with spdx_id=' + repo_data['license']['spdx_id'])
                repo.license = license
                if not project.license:
                    project.license = license
959
960
        if 'source' in repo_data:
            repo.forked_from = repo_data['source']['id']
961
962
    if repo_data:
        repo.open_issues = repo_data['open_issues_count']
963
    repo.clone_url = data['clone_url']
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
964
965
966
    repo.open_pr = len(list(repo.api_list('/pulls')))
    repo.save()
    project.save()
Guilhem Saurel's avatar
Guilhem Saurel committed
967
968
969


def update_travis(namespace, data):
Guilhem Saurel's avatar
Guilhem Saurel committed