models.py 39.8 KB
Newer Older
1
import json
2
import logging
Guilhem Saurel's avatar
Guilhem Saurel committed
3
import re
Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
4
from subprocess import check_output
5

Guilhem Saurel's avatar
Guilhem Saurel committed
6
from django.conf import settings
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
7
from django.db import models
Guilhem Saurel's avatar
Guilhem Saurel committed
8
from django.db.models import Q
Guilhem Saurel's avatar
Guilhem Saurel committed
9
from django.db.models.functions import Length
10
from django.db.utils import DataError
11
from django.template.loader import get_template
Guilhem Saurel's avatar
Guilhem Saurel committed
12
13
from django.utils.dateparse import parse_datetime
from django.utils.safestring import mark_safe
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
14

Guilhem Saurel's avatar
Guilhem Saurel committed
15
import requests
Guilhem Saurel's avatar
Guilhem Saurel committed
16
17

import git
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
18
from autoslug import AutoSlugField
Guilhem Saurel's avatar
Guilhem Saurel committed
19
from ndh.models import Links, NamedModel, TimeStampedModel
Guilhem Saurel's avatar
Guilhem Saurel committed
20
from ndh.utils import enum_to_choices, query_sum
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
21

Guilhem Saurel's avatar
Guilhem Saurel committed
22
from .utils import SOURCES, api_next, invalid_mail, slugify_with_dots, valid_name
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
23

24
25
logger = logging.getLogger('rainboard.models')

Guilhem Saurel's avatar
Guilhem Saurel committed
26
MAIN_BRANCHES = ['master', 'devel']
Guilhem Saurel's avatar
Guilhem Saurel committed
27
RPKG_URL = 'http://robotpkg.openrobots.org'
Guilhem Saurel's avatar
Guilhem Saurel committed
28
DOC_URL = 'http://projects.laas.fr/gepetto/doc'
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
29
30
31
32
33
34
35
36
37
RPKG_LICENSES = {
    'gnu-lgpl-v3': 'LGPL-3.0',
    'gnu-lgpl-v2': 'LGPL-2.0',
    'mit': 'MIT',
    'gnu-gpl-v3': 'GPL-3.0',
    '2-clause-bsd': 'BSD-2-Clause',
    'eclipse': 'EPL-1.0',
    'modified-bsd': 'BSD-3-Clause'
}
Guilhem Saurel's avatar
Guilhem Saurel committed
38
RPKG_FIELDS = ['PKGBASE', 'PKGVERSION', 'MASTER_SITES', 'MASTER_REPOSITORY', 'MAINTAINER', 'COMMENT', 'HOMEPAGE']
Guilhem Saurel's avatar
Guilhem Saurel committed
39
CMAKE_FIELDS = {'NAME': 'cmake_name', 'DESCRIPTION': 'description', 'URL': 'homepage', 'VERSION': 'version'}
Guilhem Saurel's avatar
Guilhem Saurel committed
40
TRAVIS_STATE = {'created': None, 'passed': True, 'started': None, 'failed': False, 'errored': False, 'canceled': False}
Guilhem Saurel's avatar
Guilhem Saurel committed
41
GITLAB_STATUS = {'failed': False, 'success': True, 'pending': None, 'skipped': None, 'canceled': None, 'running': None}
Guilhem Saurel's avatar
Guilhem Saurel committed
42

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
43
44

class Namespace(NamedModel):
45
    group = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
46
47


Guilhem Saurel's avatar
Guilhem Saurel committed
48
49
50
class License(models.Model):
    name = models.CharField(max_length=200)
    spdx_id = models.CharField(max_length=50, unique=True)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
51
52
53
54
55
56
    url = models.URLField(max_length=200)

    def __str__(self):
        return self.spdx_id or self.name


Guilhem Saurel's avatar
Guilhem Saurel committed
57
class Forge(Links, NamedModel):
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
58
59
60
61
62
    source = models.PositiveSmallIntegerField(choices=enum_to_choices(SOURCES))
    url = models.URLField(max_length=200)
    token = models.CharField(max_length=50, blank=True, null=True)
    verify = models.BooleanField(default=True)

Guilhem Saurel's avatar
Guilhem Saurel committed
63
64
65
    def get_absolute_url(self):
        return self.url

Guilhem Saurel's avatar
Guilhem Saurel committed
66
    def api_req(self, url='', name=None, page=1):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
67
        logger.debug(f'requesting api {self} {url}, page {page}')
68
69
70
71
72
        try:
            return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
        except requests.exceptions.ConnectionError:
            logger.error(f'requesting api {self} {url}, page {page} - SECOND TRY')
            return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
Guilhem Saurel's avatar
Guilhem Saurel committed
73
74
75
76
77
78

    def api_data(self, url=''):
        req = self.api_req(url)
        return req.json() if req.status_code == 200 else []  # TODO

    def api_list(self, url='', name=None):
Guilhem Saurel's avatar
Guilhem Saurel committed
79
80
        page = 1
        while page:
Guilhem Saurel's avatar
Guilhem Saurel committed
81
            req = self.api_req(url, name, page)
Guilhem Saurel's avatar
Guilhem Saurel committed
82
83
84
85
86
            if req.status_code != 200:
                return []  # TODO
            data = req.json()
            if name is not None:
                data = data[name]
Guilhem Saurel's avatar
Guilhem Saurel committed
87
            yield from data
Guilhem Saurel's avatar
Guilhem Saurel committed
88
            page = api_next(self.source, req)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
89
90

    def headers(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
91
        return {
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
92
93
94
95
96
97
98
99
100
101
102
103
104
105
            SOURCES.github: {
                'Authorization': f'token {self.token}',
                'Accept': 'application/vnd.github.drax-preview+json'
            },
            SOURCES.gitlab: {
                'Private-Token': self.token
            },
            SOURCES.redmine: {
                'X-Redmine-API-Key': self.token
            },
            SOURCES.travis: {
                'Authorization': f'token {self.token}',
                'TRAVIS-API-Version': '3'
            },
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
106
        }[self.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
107
108

    def api_url(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
109
110
111
112
        return {
            SOURCES.github: 'https://api.github.com',
            SOURCES.gitlab: f'{self.url}/api/v4',
            SOURCES.redmine: self.url,
Guilhem Saurel's avatar
travis    
Guilhem Saurel committed
113
            SOURCES.travis: 'https://api.travis-ci.org',
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
114
        }[self.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
115

116
117
    def get_namespaces_github(self):
        for namespace in Namespace.objects.filter(group=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
118
            for data in self.api_list(f'/orgs/{namespace.slug}/members'):
Guilhem Saurel's avatar
Guilhem Saurel committed
119
120
121
122
123
                Namespace.objects.get_or_create(
                    slug=data['login'].lower(), defaults={
                        'name': data['login'],
                        'group': False
                    })
124

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
125
126
    def get_namespaces_gitlab(self):
        for data in self.api_list('/namespaces'):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
127
128
129
130
131
            Namespace.objects.get_or_create(
                slug=data['path'], defaults={
                    'name': data['name'],
                    'group': data['kind'] == 'group'
                })
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
132
133
134
        for data in self.api_list('/users'):
            Namespace.objects.get_or_create(slug=data['username'], defaults={'name': data['name']})

Guilhem Saurel's avatar
Guilhem Saurel committed
135
136
137
    def get_namespaces_redmine(self):
        pass  # TODO

Guilhem Saurel's avatar
Guilhem Saurel committed
138
139
140
    def get_namespaces_travis(self):
        pass

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
141
142
143
    def get_projects(self):
        getattr(self, f'get_namespaces_{self.get_source_display()}')()
        return getattr(self, f'get_projects_{self.get_source_display()}')()
144

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
145
    def get_projects_github(self):
146
        for org in Namespace.objects.filter(group=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
147
            for data in self.api_list(f'/orgs/{org.slug}/repos'):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
148
                update_github(self, org, data)
149
        for user in Namespace.objects.filter(group=False):
Guilhem Saurel's avatar
Guilhem Saurel committed
150
            for data in self.api_list(f'/users/{user.slug}/repos'):
Guilhem Saurel's avatar
Guilhem Saurel committed
151
                if Project.objects.filter(name=valid_name(data['name'])).exists():
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
152
                    update_github(self, user, data)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
153
154

    def get_projects_gitlab(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
155
        for data in self.api_list('/projects'):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
156
            update_gitlab(self, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
157
158

        for orphan in Project.objects.filter(main_namespace=None):
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
159
            repo = orphan.repo_set.filter(forge__source=SOURCES.gitlab).first()
Guilhem Saurel's avatar
Guilhem Saurel committed
160
161
            if repo:
                update_gitlab(self, self.api_data(f'/projects/{repo.forked_from}'))
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
162
163
164
165

    def get_projects_redmine(self):
        pass  # TODO

Guilhem Saurel's avatar
Guilhem Saurel committed
166
167
168
169
170
171
    def get_projects_travis(self):
        for namespace in Namespace.objects.all():
            for repository in self.api_list(f'/owner/{namespace.slug}/repos', 'repositories'):
                if repository['active']:
                    update_travis(namespace, repository)

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
172

Guilhem Saurel's avatar
Guilhem Saurel committed
173
class Project(Links, NamedModel, TimeStampedModel):
Guilhem Saurel's avatar
Guilhem Saurel committed
174
    public = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
175
176
177
178
    main_namespace = models.ForeignKey(Namespace, on_delete=models.SET_NULL, null=True, blank=True)
    main_forge = models.ForeignKey(Forge, on_delete=models.SET_NULL, null=True, blank=True)
    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    homepage = models.URLField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
179
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
180
    version = models.CharField(max_length=20, blank=True, null=True)
181
    updated = models.DateTimeField(blank=True, null=True)
182
183
    tests = models.BooleanField(default=True)
    docs = models.BooleanField(default=True)
184
    debug = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
185
    from_gepetto = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
186
    cmake_name = models.CharField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
187

188
189
190
191
    def save(self, *args, **kwargs):
        self.name = valid_name(self.name)
        super().save(*args, **kwargs)

Guilhem Saurel's avatar
Guilhem Saurel committed
192
    def git_path(self):
193
        return settings.RAINBOARD_GITS / self.main_namespace.slug / self.slug.strip()  # workaround SafeText TypeError
Guilhem Saurel's avatar
Guilhem Saurel committed
194

Guilhem Saurel's avatar
Guilhem Saurel committed
195
    def git(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
196
        path = self.git_path()
Guilhem Saurel's avatar
Guilhem Saurel committed
197
198
199
200
201
        if not path.exists():
            logger.info(f'Creating repo for {self.main_namespace.slug}/{self.slug}')
            return git.Repo.init(path)
        return git.Repo(str(path / '.git'))

202
    def main_repo(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
203
        forge = self.main_forge if self.main_forge else get_default_forge(self)
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
204
205
206
207
208
209
210
211
212
        repo, created = Repo.objects.get_or_create(
            forge=forge,
            namespace=self.main_namespace,
            project=self,
            defaults={
                'name': self.name,
                'default_branch': 'master',
                'repo_id': 0
            })
Guilhem Saurel's avatar
Guilhem Saurel committed
213
        if created:
214
            repo.api_update()
Guilhem Saurel's avatar
Guilhem Saurel committed
215
216
        return repo

Guilhem Saurel's avatar
Guilhem Saurel committed
217
    def update_branches(self, main=True, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
218
219
220
        branches = [b[2:] for b in self.git().git.branch('-a', '--no-color').split('\n')]
        if main:
            branches = [b for b in branches if b.endswith('master') or b.endswith('devel')]
Guilhem Saurel's avatar
Guilhem Saurel committed
221
        for branch in branches:
Guilhem Saurel's avatar
Guilhem Saurel committed
222
            logger.info(f'update branch {branch}')
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
223
224
            if branch.startswith('remotes/'):
                branch = branch[8:]
225
226
227
            if branch.count('/') < 2:
                logger.error(f'wrong branch "{branch}" in {self.git_path()}')
                continue
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
            forge, namespace, name = branch.split('/', maxsplit=2)
            namespace, _ = Namespace.objects.get_or_create(slug=namespace)
            forge = Forge.objects.get(slug=forge)
            repo, created = Repo.objects.get_or_create(
                forge=forge,
                namespace=namespace,
                project=self,
                defaults={
                    'name': self.name,
                    'default_branch': 'master',
                    'repo_id': 0
                })
            if created:
                repo.api_update()
            instance, bcreated = Branch.objects.get_or_create(name=branch, project=self, repo=repo)
Guilhem Saurel's avatar
Guilhem Saurel committed
243
            if bcreated:
Guilhem Saurel's avatar
Guilhem Saurel committed
244
                instance.update(pull=pull)
245

Guilhem Saurel's avatar
update    
Guilhem Saurel committed
246
247
248
    def checkout(self):
        self.main_branch().git().checkout()

Guilhem Saurel's avatar
Guilhem Saurel committed
249
    def main_branch(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
250
        return self.main_repo().main_branch()
Guilhem Saurel's avatar
Guilhem Saurel committed
251

Guilhem Saurel's avatar
Guilhem Saurel committed
252
253
254
255
256
257
258
    def cmake(self):
        filename = self.git_path() / 'CMakeLists.txt'
        if not filename.exists():
            return
        with filename.open() as f:
            content = f.read()
        for key, value in CMAKE_FIELDS.items():
Guilhem Saurel's avatar
Guilhem Saurel committed
259
            search = re.search(fr'set\s*\(\s*project_{key}\s+([^)]+)*\)', content, re.I)
Guilhem Saurel's avatar
Guilhem Saurel committed
260
            if search:
261
                try:
262
                    old = getattr(self, value)
263
                    new = search.groups()[0].strip(''' \r\n\t'"''').replace('_', '-')
264
265
266
                    if old != new:
                        setattr(self, value, new)
                        self.save()
267
                except DataError:
268
                    setattr(self, value, old)
269
        for dependency in re.findall(r'ADD_[A-Z]+_DEPENDENCY\s*\(["\']?([^ "\')]+).*["\']?\)', content, re.I):
Guilhem Saurel's avatar
Guilhem Saurel committed
270
            project = Project.objects.filter(name=valid_name(dependency))
271
            if project.exists():
Guilhem Saurel's avatar
oops    
Guilhem Saurel committed
272
273
                dependency, _ = Dependency.objects.get_or_create(project=self, library=project.first())
                if not dependency.cmake:
274
275
                    dependency.cmake = True
                    dependency.save()
Guilhem Saurel's avatar
Guilhem Saurel committed
276

277
    def ros(self):
278
279
        try:
            filename = self.git_path() / 'package.xml'
280
        except TypeError:
281
            return
282
283
284
285
286
        if not filename.exists():
            return
        with filename.open() as f:
            content = f.read()
        for dependency in re.findall(r'<run_depend>(\w+).*</run_depend>', content, re.I):
Guilhem Saurel's avatar
Guilhem Saurel committed
287
            project = Project.objects.filter(name=valid_name(dependency))
288
289
290
291
292
293
            if project.exists():
                dependency, _ = Dependency.objects.get_or_create(project=self, library=project.first())
                if not dependency.ros:
                    dependency.ros = True
                    dependency.save()

294
295
296
297
298
299
    def repos(self):
        return self.repo_set.count()

    def rpkgs(self):
        return self.robotpkg_set.count()

Guilhem Saurel's avatar
Guilhem Saurel committed
300
301
302
303
    def update_tags(self):
        for tag in self.git().tags:
            Tag.objects.get_or_create(name=str(tag), project=self)

304
    def update(self, only_main_branches=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
305
306
        if self.main_namespace is None:
            return
307
        self.update_branches(main=only_main_branches)
Guilhem Saurel's avatar
Guilhem Saurel committed
308
        self.update_tags()
309
        tag = self.tag_set.filter(name__startswith='v').last()  # TODO: implement SQL ordering for semver
Guilhem Saurel's avatar
Guilhem Saurel committed
310
311
        if tag is not None:
            self.version = tag.name[1:]
312
        robotpkg = self.robotpkg_set.order_by('-updated').first()
Guilhem Saurel's avatar
Guilhem Saurel committed
313
        branch = self.branch_set.order_by('-updated').first()
Guilhem Saurel's avatar
Guilhem Saurel committed
314
315
316
        branch_updated = branch is not None and branch.updated is not None
        robotpkg_updated = robotpkg is not None and robotpkg.updated is not None
        if branch_updated or robotpkg_updated:
Guilhem Saurel's avatar
Guilhem Saurel committed
317
            if not robotpkg_updated:
Guilhem Saurel's avatar
Guilhem Saurel committed
318
                self.updated = branch.updated
Guilhem Saurel's avatar
Guilhem Saurel committed
319
            elif not branch_updated:
Guilhem Saurel's avatar
Guilhem Saurel committed
320
321
322
                self.updated = robotpkg.updated
            else:
                self.updated = max(branch.updated, robotpkg.updated)
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
323
        self.checkout()
324
        self.cmake()
Guilhem Saurel's avatar
Guilhem Saurel committed
325
        self.ros()
326
327
        self.save()

Guilhem Saurel's avatar
Guilhem Saurel committed
328
    def commits_since(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
329
330
331
332
333
        try:
            commits = self.git().git.rev_list(f'v{self.version}..{self.main_branch()}')
            return len(commits.split('\n')) if commits else 0
        except git.exc.GitCommandError:
            pass
Guilhem Saurel's avatar
Guilhem Saurel committed
334

335
336
337
338
339
340
    def open_issues(self):
        return query_sum(self.repo_set, 'open_issues')

    def open_pr(self):
        return query_sum(self.repo_set, 'open_pr')

341
342
343
    def gitlabciyml(self):
        return get_template('rainboard/gitlab-ci.yml').render({'project': self})

Guilhem Saurel's avatar
Guilhem Saurel committed
344
345
346
347
348
349
350
351
352
    def contributors(self, update=False):
        if update:
            for guy in self.git().git.shortlog('-nse').split('\n'):
                name, mail = guy[7:-1].split(' <')
                contributor = get_contributor(name, mail)
                contributor.projects.add(self)
                contributor.save()
        return self.contributor_set.all()

353
354
355
    def registry(self):
        return settings.PUBLIC_REGISTRY if self.public else settings.PRIVATE_REGISTRY

Guilhem Saurel's avatar
Guilhem Saurel committed
356
357
358
359
    def doc_coverage_image(self):
        images = Image.objects.filter(robotpkg__project=self, py3=False, target__name='16.04')
        return images.order_by(Length('robotpkg__name').desc()).first()

360
    def print_deps(self):
361
        return mark_safe(', '.join(d.library.get_link() for d in self.dependencies.all()))
362
363

    def print_rdeps(self):
364
        return mark_safe(', '.join(d.project.get_link() for d in self.rdeps.all()))
365

Guilhem Saurel's avatar
Guilhem Saurel committed
366
367
368
    def ordered_robotpkg(self):
        return self.robotpkg_set.order_by('name')

Guilhem Saurel's avatar
Guilhem Saurel committed
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
    def url_travis(self):
        return f'https://travis-ci.org/{self.main_namespace.slug}/{self.slug}'

    def url_gitlab(self):
        return f'https://gepgitlab.laas.fr/{self.main_namespace.slug}/{self.slug}'

    def badge(self, link, img, alt):
        return mark_safe(f'<a href="{link}"><img src="{img}" alt="{alt}" /></a> ')

    def badge_travis(self):
        return self.badge(self.url_travis(), f'{self.url_travis()}.svg?branch=master', 'Building Status')

    def badge_gitlab(self):
        return self.badge(self.url_gitlab(), f'{self.url_gitlab()}/badges/master/pipeline.svg', 'Pipeline Status')

    def badge_coverage(self):
        return self.badge(f'{DOC_URL}/{self.main_namespace.slug}/{self.slug}/master/coverage',
Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
386
                          f'{self.url_gitlab()}/badges/master/coverage.svg?job=doc-coverage"', 'Coverage Report')
Guilhem Saurel's avatar
Guilhem Saurel committed
387
388
389
390

    def badges(self):
        return self.badge_travis() + self.badge_gitlab() + self.badge_coverage()

Guilhem Saurel's avatar
flake8    
Guilhem Saurel committed
391

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
392
393
class Repo(TimeStampedModel):
    name = models.CharField(max_length=200)
Guilhem Saurel's avatar
Guilhem Saurel committed
394
    slug = AutoSlugField(populate_from='name', slugify=slugify_with_dots)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
395
396
397
398
399
400
401
402
403
404
    forge = models.ForeignKey(Forge, on_delete=models.CASCADE)
    namespace = models.ForeignKey(Namespace, on_delete=models.CASCADE)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    homepage = models.URLField(max_length=200, blank=True, null=True)
    url = models.URLField(max_length=200, blank=True, null=True)
    default_branch = models.CharField(max_length=50)
    open_issues = models.PositiveSmallIntegerField(blank=True, null=True)
    open_pr = models.PositiveSmallIntegerField(blank=True, null=True)
    repo_id = models.PositiveIntegerField()
Guilhem Saurel's avatar
Guilhem Saurel committed
405
    forked_from = models.PositiveIntegerField(blank=True, null=True)
406
    clone_url = models.URLField(max_length=200)
Guilhem Saurel's avatar
travis    
Guilhem Saurel committed
407
    travis_id = models.PositiveIntegerField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
408
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
409
410
411

    def __str__(self):
        return self.name
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
412
413

    def api_url(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
414
        api_url = self.forge.api_url()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
415
        return {
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
416
417
418
            SOURCES.github: f'{api_url}/repos/{self.namespace.slug}/{self.slug}',
            SOURCES.redmine: f'{api_url}/projects/{self.repo_id}.json',
            SOURCES.gitlab: f'{api_url}/projects/{self.repo_id}',
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
419
        }[self.forge.source]
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
420

Guilhem Saurel's avatar
Guilhem Saurel committed
421
    def api_req(self, url='', name=None, page=1):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
422
        logger.debug(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}')
423
        try:
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
424
425
            return requests.get(
                self.api_url() + url, {'page': page}, verify=self.forge.verify, headers=self.forge.headers())
426
427
        except requests.exceptions.ConnectionError:
            logger.error(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page} - SECOND TRY')
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
428
429
            return requests.get(
                self.api_url() + url, {'page': page}, verify=self.forge.verify, headers=self.forge.headers())
Guilhem Saurel's avatar
Guilhem Saurel committed
430
431
432
433
434
435

    def api_data(self, url=''):
        req = self.api_req(url)
        return req.json() if req.status_code == 200 else []  # TODO

    def api_list(self, url='', name=None):
Guilhem Saurel's avatar
Guilhem Saurel committed
436
437
        page = 1
        while page:
Guilhem Saurel's avatar
Guilhem Saurel committed
438
            req = self.api_req(url, name, page)
Guilhem Saurel's avatar
Guilhem Saurel committed
439
440
441
442
            if req.status_code != 200:
                return []  # TODO
            data = req.json()
            if name is not None:
Guilhem Saurel's avatar
Guilhem Saurel committed
443
444
445
446
                if name in data:
                    data = data[name]
                else:
                    return []  # TODO
Guilhem Saurel's avatar
Guilhem Saurel committed
447
            yield from data
Guilhem Saurel's avatar
Guilhem Saurel committed
448
            page = api_next(self.forge.source, req)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
449

Guilhem Saurel's avatar
Guilhem Saurel committed
450
    def api_update(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
451
        data = self.api_data()
Guilhem Saurel's avatar
Guilhem Saurel committed
452
        if data:
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
453
            return getattr(self, f'api_update_{self.forge.get_source_display()}')(data)
Guilhem Saurel's avatar
Guilhem Saurel committed
454
455

    def api_update_gitlab(self, data):
456
        update_gitlab(self.forge, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
457

Guilhem Saurel's avatar
Guilhem Saurel committed
458
    def api_update_github(self, data):
459
        update_github(self.forge, self.namespace, data)
Guilhem Saurel's avatar
Guilhem Saurel committed
460

Guilhem Saurel's avatar
git    
Guilhem Saurel committed
461
462
463
    def get_clone_url(self):
        if self.forge.source == SOURCES.gitlab:
            return self.clone_url.replace('://', f'://gitlab-ci-token:{self.forge.token}@')
464
465
        if self.forge.source == SOURCES.github:
            return self.clone_url.replace('://', f'://{settings.GITHUB_USER}:{self.forge.token}@')
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
466
467
        return self.clone_url

468
469
470
    def git_remote(self):
        return f'{self.forge.slug}/{self.namespace.slug}'

Guilhem Saurel's avatar
git    
Guilhem Saurel committed
471
    def git(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
472
        git_repo = self.project.git()
473
        remote = self.git_remote()
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
474
        try:
Guilhem Saurel's avatar
Guilhem Saurel committed
475
            return git_repo.remote(remote)
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
476
        except ValueError:
Guilhem Saurel's avatar
Guilhem Saurel committed
477
            logger.info(f'Creating remote {remote}')
Guilhem Saurel's avatar
Guilhem Saurel committed
478
            return git_repo.create_remote(remote, self.get_clone_url())
Guilhem Saurel's avatar
git    
Guilhem Saurel committed
479

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
480
    def fetch(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
481
482
        git_repo = self.git()
        logger.debug(f'fetching {self.forge} / {self.namespace} / {self.project}')
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
483
        try:
Guilhem Saurel's avatar
Guilhem Saurel committed
484
            git_repo.fetch()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
485
        except git.exc.GitCommandError:
Guilhem Saurel's avatar
Guilhem Saurel committed
486
            logger.warning(f'fetching {self.forge} / {self.namespace} / {self.project} - SECOND TRY')
487
488
489
490
491
            try:
                git_repo.fetch()
            except git.exc.GitCommandError:
                return False
        return True
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
492

Guilhem Saurel's avatar
Guilhem Saurel committed
493
494
495
    def main_branch(self):
        return self.project.branch_set.get(name=f'{self.git_remote()}/{self.default_branch}')

Guilhem Saurel's avatar
Guilhem Saurel committed
496
    def ahead(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
497
498
        main_branch = self.main_branch()
        return main_branch.ahead if main_branch is not None else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
499
500

    def behind(self):
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
501
502
        main_branch = self.main_branch()
        return main_branch.behind if main_branch is not None else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
503

Guilhem Saurel's avatar
Guilhem Saurel committed
504
505
506
507
    def get_builds(self):
        return getattr(self, f'get_builds_{self.forge.get_source_display()}')()

    def get_builds_gitlab(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
508
        for pipeline in self.api_list('/pipelines'):
Guilhem Saurel's avatar
Guilhem Saurel committed
509
510
511
            pid, ref = pipeline['id'], pipeline['ref']
            if self.project.tag_set.filter(name=ref).exists():
                continue
Guilhem Saurel's avatar
Guilhem Saurel committed
512
            data = self.api_data(f'/pipelines/{pid}')
Guilhem Saurel's avatar
Guilhem Saurel committed
513
514
515
516
            branch_name = f'{self.forge.slug}/{self.namespace.slug}/{ref}'
            branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
            if created:
                branch.update()
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
517
518
519
520
521
522
523
524
            CIBuild.objects.get_or_create(
                repo=self,
                build_id=pid,
                defaults={
                    'passed': GITLAB_STATUS[pipeline['status']],
                    'started': parse_datetime(data['created_at']),
                    'branch': branch,
                })
Guilhem Saurel's avatar
Guilhem Saurel committed
525
526
527
528

    def get_builds_github(self):
        if self.travis_id is not None:
            travis = Forge.objects.get(source=SOURCES.travis)
Guilhem Saurel's avatar
Guilhem Saurel committed
529
            for build in travis.api_list(f'/repo/{self.travis_id}/builds', name='builds'):
Guilhem Saurel's avatar
Guilhem Saurel committed
530
531
532
533
534
                if self.project.tag_set.filter(name=build['branch']['name']).exists():
                    continue
                branch_name = f'{self.forge.slug}/{self.namespace.slug}/{build["branch"]["name"]}'
                branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
                if created:
Guilhem Saurel's avatar
Guilhem Saurel committed
535
                    branch.update()
Guilhem Saurel's avatar
Guilhem Saurel committed
536
                started = build['started_at'] if build['started_at'] is not None else build['finished_at']
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
537
538
539
540
541
542
543
544
                CIBuild.objects.get_or_create(
                    repo=self,
                    build_id=build['id'],
                    defaults={
                        'passed': TRAVIS_STATE[build['state']],
                        'started': parse_datetime(started),
                        'branch': branch,
                    })
Guilhem Saurel's avatar
Guilhem Saurel committed
545
546

    def update(self, pull=True):
547
        ok = True
Guilhem Saurel's avatar
Guilhem Saurel committed
548
549
        self.project.update_tags()
        if pull:
550
551
552
553
554
555
556
            ok = self.fetch()
        if ok:
            self.api_update()
            self.get_builds()
        else:
            logger.error(f'fetching {self.forge} / {self.namespace} / {self.project} - NOT FOUND - DELETING')
            logger.error(str(self.delete()))
Guilhem Saurel's avatar
Guilhem Saurel committed
557

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
558
559
560
561

class Commit(NamedModel, TimeStampedModel):
    project = models.ForeignKey(Project, on_delete=models.CASCADE)

Guilhem Saurel's avatar
Guilhem Saurel committed
562
563
564
    class Meta:
        unique_together = ('project', 'name')

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
565

Guilhem Saurel's avatar
Guilhem Saurel committed
566
567
568
569
570
class Branch(TimeStampedModel):
    name = models.CharField(max_length=200)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    ahead = models.PositiveSmallIntegerField(blank=True, null=True)
    behind = models.PositiveSmallIntegerField(blank=True, null=True)
571
    updated = models.DateTimeField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
572
    repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
573
    deleted = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
574
    keep_doc = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
575
576

    def __str__(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
577
578
579
        return self.name

    class Meta:
Guilhem Saurel's avatar
Guilhem Saurel committed
580
        unique_together = ('project', 'name', 'repo')
Guilhem Saurel's avatar
Guilhem Saurel committed
581
582

    def get_ahead(self, branch='master'):
Guilhem Saurel's avatar
Guilhem Saurel committed
583
584
        commits = self.project.git().git.rev_list(f'{branch}..{self}')
        return len(commits.split('\n')) if commits else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
585
586

    def get_behind(self, branch='master'):
Guilhem Saurel's avatar
Guilhem Saurel committed
587
588
        commits = self.project.git().git.rev_list(f'{self}..{branch}')
        return len(commits.split('\n')) if commits else 0
Guilhem Saurel's avatar
Guilhem Saurel committed
589

590
591
592
    def git(self):
        git_repo = self.project.git()
        if self.name not in git_repo.branches:
Guilhem Saurel's avatar
Guilhem Saurel committed
593
            remote = self.repo.git()
594
595
596
597
598
            _, _, branch = self.name.split('/', maxsplit=2)
            git_repo.create_head(self.name, remote.refs[branch]).set_tracking_branch(remote.refs[branch])
        return git_repo.branches[self.name]

    def update(self, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
599
600
601
602
603
604
605
        if self.deleted:
            return
        try:
            if pull:
                self.repo.fetch()
                if self.repo != self.project.main_repo():
                    self.project.main_repo().fetch()
Guilhem Saurel's avatar
details    
Guilhem Saurel committed
606
607
            try:
                main_branch = self.project.main_branch()
Guilhem Saurel's avatar
Guilhem Saurel committed
608
609
                self.ahead = self.get_ahead(main_branch)
                self.behind = self.get_behind(main_branch)
Guilhem Saurel's avatar
typo    
Guilhem Saurel committed
610
            except Branch.DoesNotExist:
Guilhem Saurel's avatar
details    
Guilhem Saurel committed
611
                pass
Guilhem Saurel's avatar
Guilhem Saurel committed
612
613
614
            self.updated = self.git().commit.authored_datetime
        except (git.exc.GitCommandError, IndexError):
            self.deleted = True
Guilhem Saurel's avatar
Guilhem Saurel committed
615
        self.save()
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
616

Guilhem Saurel's avatar
Guilhem Saurel committed
617
618
619
620
621
622
623
    def ci(self):
        build = self.cibuild_set.last()
        if build is None:
            return ''
        status = {True: '✓', False: '✗', None: '?'}[build.passed]
        return mark_safe(f'<a href="{build.url()}">{status}</a>')

624
    def forge(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
625
        return self.repo.forge
626
627

    def namespace(self):
Guilhem Saurel's avatar
update    
Guilhem Saurel committed
628
        return self.repo.namespace
629

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
630

Guilhem Saurel's avatar
Guilhem Saurel committed
631
632
633
634
635
class ActiveQuerySet(models.QuerySet):
    def active(self):
        return self.filter(active=True)


Guilhem Saurel's avatar
Guilhem Saurel committed
636
class Target(NamedModel):
Guilhem Saurel's avatar
Guilhem Saurel committed
637
638
639
    active = models.BooleanField(default=True)

    objects = ActiveQuerySet.as_manager()
Guilhem Saurel's avatar
Guilhem Saurel committed
640
641
642


# class Test(TimeStampedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
643
644
645
646
647
648
649
#     project = models.ForeignKey(Project, on_delete=models.CASCADE)
#     branch = models.ForeignKey(Branch, on_delete=models.CASCADE)
#     commit = models.ForeignKey(Commit, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
#     passed = models.BooleanField(default=False)
#     TODO: travis vs gitlab-ci ?
#     TODO: deploy binary, doc, coverage, lint
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
650

Guilhem Saurel's avatar
Guilhem Saurel committed
651
# class SystemDependency(NamedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
652
653
#     project = models.ForeignKey(Project, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
654
655


Guilhem Saurel's avatar
Guilhem Saurel committed
656
class Robotpkg(NamedModel):
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
657
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
658
    category = models.CharField(max_length=50)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
659

Guilhem Saurel's avatar
Guilhem Saurel committed
660
    pkgbase = models.CharField(max_length=50, default='')
661
    pkgversion = models.CharField(max_length=50, default='')
Guilhem Saurel's avatar
Guilhem Saurel committed
662
663
664
665
    master_sites = models.CharField(max_length=200, default='')
    master_repository = models.CharField(max_length=200, default='')
    maintainer = models.CharField(max_length=200, default='')
    comment = models.TextField()
666
    homepage = models.URLField(max_length=200, blank=True, null=True)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
667
668

    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
669
    public = models.BooleanField(default=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
670
    description = models.TextField(blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
671
672
673
674
675
676
677
678
679
680
    updated = models.DateTimeField(blank=True, null=True)

    def main_page(self):
        if self.category != 'wip':
            return f'{RPKG_URL}/robotpkg/{self.category}/{self.name}'

    def build_page(self):
        path = '-wip/wip' if self.category == 'wip' else f'/{self.category}'
        return f'{RPKG_URL}/rbulk/robotpkg{path}/{self.name}'

681
    def update_images(self):
682
683
        py3s = [False, True] if self.name.startswith('py-') else [False]
        debugs = [False, True] if self.project.debug else [False]
Guilhem Saurel's avatar
Guilhem Saurel committed
684
        for target in Target.objects.active():
685
686
687
            for py3 in py3s:
                for debug in debugs:
                    Image.objects.get_or_create(robotpkg=self, target=target, py3=py3, debug=debug)[0].update()
Guilhem Saurel's avatar
Guilhem Saurel committed
688

689
    def update(self, pull=True):
Guilhem Saurel's avatar
Guilhem Saurel committed
690
691
692
693
694
695
        path = settings.RAINBOARD_RPKG
        repo = git.Repo(str(path / 'wip' / '.git' if self.category == 'wip' else path / '.git'))
        if pull:
            repo.remotes.origin.pull()

        cwd = path / self.category / self.name
Guilhem Saurel's avatar
Guilhem Saurel committed
696
697
698
        if not cwd.is_dir():
            logger.warning(f'deleted {self}: {self.delete()}')
            return
Guilhem Saurel's avatar
Guilhem Saurel committed
699
700
701
702
703
704
705
706
707
708
709
710
711
        for field in RPKG_FIELDS:
            cmd = ['make', 'show-var', f'VARNAME={field}']
            self.__dict__[field.lower()] = check_output(cmd, cwd=cwd).decode().strip()

        repo_path = self.name if self.category == 'wip' else f'{self.category}/{self.name}'
        last_commit = next(repo.iter_commits(paths=repo_path, max_count=1))
        self.updated = last_commit.authored_datetime

        license = check_output(['make', 'show-var', f'VARNAME=LICENSE'], cwd=cwd).decode().strip()
        if license in RPKG_LICENSES:
            self.license = License.objects.get(spdx_id=RPKG_LICENSES[license])
        else:
            logger.warning(f'Unknown robotpkg license: {license}')
Guilhem Saurel's avatar
Guilhem Saurel committed
712
        self.public = not bool(check_output(['make', 'show-var', f'VARNAME=RESTRICTED'], cwd=cwd).decode().strip())
Guilhem Saurel's avatar
Guilhem Saurel committed
713
714
715
        with (cwd / 'DESCR').open() as f:
            self.description = f.read().strip()

716
        self.update_images()
Guilhem Saurel's avatar
Guilhem Saurel committed
717
        self.save()
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
718

719
    def valid_images(self):
720
        return self.image_set.filter(created__isnull=False, target__active=True).order_by('target__name')
721

722
723
724
725
    def without_py(self):
        if 'py-' in self.name:
            return Robotpkg.objects.filter(name=self.name.replace('py-', '')).first()

Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
726

Guilhem Saurel's avatar
Guilhem Saurel committed
727
# class RobotpkgBuild(TimeStampedModel):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
728
729
730
#     robotpkg = models.ForeignKey(Robotpkg, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
#     passed = models.BooleanField(default=False)
Guilhem Saurel's avatar
initial  
Guilhem Saurel committed
731
732


733
734
class Image(models.Model):
    robotpkg = models.ForeignKey(Robotpkg, on_delete=models.CASCADE)
Guilhem Saurel's avatar
Guilhem Saurel committed
735
    target = models.ForeignKey(Target, on_delete=models.CASCADE)
736
737
    created = models.DateTimeField(blank=True, null=True)
    image = models.CharField(max_length=12, blank=True, null=True)
Guilhem Saurel's avatar
Guilhem Saurel committed
738
    py3 = models.BooleanField(default=False)
739
    debug = models.BooleanField(default=False)
740
741

    class Meta:
742
        unique_together = ('robotpkg', 'target', 'py3', 'debug')
743
744

    def __str__(self):
Guilhem Saurel's avatar
Guilhem Saurel committed
745
746
        py = '-py3' if self.py3 else ''
        return f'{self.robotpkg}{py}:{self.target}'
747
748

    def get_build_args(self):
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
749
        ret = {'TARGET': self.target, 'ROBOTPKG': self.robotpkg, 'REGISTRY': self.robotpkg.project.registry()}
Guilhem Saurel's avatar
Guilhem Saurel committed
750
        if not self.robotpkg.project.public:
Guilhem Saurel's avatar
Guilhem Saurel committed
751
752
753
            ret['IMAGE'] = 'robotpkg-jrl-py3' if self.py3 else 'robotpkg-jrl'
        elif self.py3:
            ret['IMAGE'] = 'robotpkg-py3'
Guilhem Saurel's avatar
Guilhem Saurel committed
754
        return ret
755
756
757

    def get_image_name(self):
        project = self.robotpkg.project
758
        return f'{project.registry()}/{project.main_namespace.slug}/{project.slug}/{self}'.lower()
759

760
761
762
    def get_image_url(self):
        project = self.robotpkg.project
        manifest = str(self).replace(':', '/manifests/')
763
        return f'https://{project.registry()}/v2/{project.main_namespace.slug}/{project.slug}/{manifest}'
764

Guilhem Saurel's avatar
Guilhem Saurel committed
765
    def get_job_name(self):
766
767
        mode = 'debug' if self.debug else 'release'
        return f'robotpkg-{self}-{mode}'.replace(':', '-')
Guilhem Saurel's avatar
Guilhem Saurel committed
768

769
770
771
    def build(self):
        args = self.get_build_args()
        build_args = sum((['--build-arg', f'{key}={value}'] for key, value in args.items()), list())
772
773
774
775
776
777
778
        return ['docker', 'build', '-t', self.get_image_name()] + build_args + ['.']

    def pull(self):
        return ['docker', 'pull', self.get_image_name()]

    def push(self):
        return ['docker', 'push', self.get_image_name()]
779

Guilhem Saurel's avatar
Guilhem Saurel committed
780
    def update(self, pull=False):
781
782
783
784
785
        r = requests.get(self.get_image_url())
        if r.status_code == 200:
            self.image = r.json()['fsLayers'][0]['blobSum'].split(':')[1][:12]
            self.created = parse_datetime(json.loads(r.json()['history'][0]['v1Compatibility'])['created'])
            self.save()
786
787


Guilhem Saurel's avatar
Guilhem Saurel committed
788
789
790
791
792
793
794
795
class CIBuild(models.Model):
    repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
    passed = models.NullBooleanField()
    build_id = models.PositiveIntegerField()
    started = models.DateTimeField()
    branch = models.ForeignKey(Branch, on_delete=models.CASCADE)

    class Meta:
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
796
        ordering = ('started', )
Guilhem Saurel's avatar
Guilhem Saurel committed
797
798
799
800
801
802
803
804
805
806
807
808
809
810

    def url(self):
        if self.repo.forge.source == SOURCES.github:
            return f'https://travis-ci.org/{self.repo.namespace.slug}/{self.repo.slug}/builds/{self.build_id}'
        if self.repo.forge.source == SOURCES.gitlab:
            return f'{self.repo.forge.url}/{self.repo.namespace.slug}/{self.repo.slug}/pipelines/{self.build_id}'


class Tag(models.Model):
    name = models.CharField(max_length=200)
    slug = AutoSlugField(populate_from='name', slugify=slugify_with_dots)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)

    class Meta:
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
811
        ordering = ('name', )
Guilhem Saurel's avatar
Guilhem Saurel committed
812
813
        unique_together = ('name', 'project')

Guilhem Saurel's avatar
Guilhem Saurel committed
814
815
816
    def __str__(self):
        return f'{self.project} {self.name}'

Guilhem Saurel's avatar
Guilhem Saurel committed
817

818
819
820
821
822
class GepettistQuerySet(models.QuerySet):
    def gepettist(self):
        return self.filter(projects__from_gepetto=True)


Guilhem Saurel's avatar
Guilhem Saurel committed
823
824
class Contributor(models.Model):
    projects = models.ManyToManyField(Project)
Guilhem Saurel's avatar
Guilhem Saurel committed
825
    agreement_signed = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
826

827
828
    objects = GepettistQuerySet.as_manager()

Guilhem Saurel's avatar
Guilhem Saurel committed
829
830
831
832
833
834
835
836
837
838
839
840
    def __str__(self):
        name = self.contributorname_set.first()
        mail = self.contributormail_set.first()
        return f'{name} <{mail}>'

    def names(self):
        return ', '.join(str(name) for name in self.contributorname_set.all())

    def mails(self):
        return ', '.join(str(mail) for mail in self.contributormail_set.filter(invalid=False))

    def contributed(self):
841
        return ', '.join(str(project) for project in self.projects.filter(from_gepetto=True))
Guilhem Saurel's avatar
Guilhem Saurel committed
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858


class ContributorName(models.Model):
    contributor = models.ForeignKey(Contributor, on_delete=models.CASCADE, blank=True, null=True)
    name = models.CharField(max_length=200, unique=True)

    def __str__(self):
        return self.name


class ContributorMail(models.Model):
    contributor = models.ForeignKey(Contributor, on_delete=models.CASCADE, blank=True, null=True)
    mail = models.EmailField(unique=True)
    invalid = models.BooleanField(default=False)

    def __str__(self):
        return self.mail
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
859
860


Guilhem Saurel's avatar
Guilhem Saurel committed
861
862
863
class Dependency(models.Model):
    project = models.ForeignKey(Project, on_delete=models.CASCADE, related_name='dependencies')
    library = models.ForeignKey(Project, on_delete=models.CASCADE, related_name='rdeps')
864
    robotpkg = models.BooleanField(default=False)  # TODO NYI
Guilhem Saurel's avatar
Guilhem Saurel committed
865
    cmake = models.BooleanField(default=False)
866
    ros = models.BooleanField(default=False)
Guilhem Saurel's avatar
Guilhem Saurel committed
867
868
869
870
871
872

    class Meta:
        verbose_name_plural = 'dependencies'
        unique_together = ('project', 'library')

    def __str__(self):
873
        return f'{self.project} depends on {self.library}: {self.robotpkg:d} {self.cmake:d}'
Guilhem Saurel's avatar
Guilhem Saurel committed
874
875


Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
876
877
878
879
880
881
882
883
884
885
886
887
def get_default_forge(project):
    for forge in Forge.objects.order_by('source'):
        if project.repo_set.filter(forge=forge).exists():
            logger.info(f'default forge for {project} set to {forge}')
            project.main_forge = forge
            project.save()
            return forge
    else:
        logger.error(f'NO DEFAULT FORGE for {project}')


def update_gitlab(forge, data):
888
889
    if data['archived']:
        return
890
    if 'default_branch' not in data or data['default_branch'] is None:
891
        return
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
892
    logger.info(f'update {data["name"]} from {forge}')
Guilhem Saurel's avatar
Guilhem Saurel committed
893
    public = data['visibility'] not in ['private', 'internal']
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
894
    project, created = Project.objects.get_or_create(
Guilhem Saurel's avatar
Guilhem Saurel committed
895
        name=valid_name(data['name']), defaults={
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
            'main_forge': forge,
            'public': public
        })
    namespace, _ = Namespace.objects.get_or_create(
        slug=data['namespace']['path'], defaults={'name': data['namespace']['name']})
    repo, _ = Repo.objects.get_or_create(
        forge=forge,
        namespace=namespace,
        project=project,
        defaults={
            'repo_id': data['id'],
            'name': data['name'],
            'url': data['web_url'],
            'default_branch': data['default_branch'],
            'clone_url': data['http_url_to_repo']
        })
912
913
914
915
916
917
918
    repo.name = data['name']
    repo.slug = data['path']
    repo.url = data['web_url']
    repo.repo_id = data['id']
    repo.clone_url = data['http_url_to_repo']
    repo.open_issues = data['open_issues_count']
    repo.default_branch = data['default_branch']
Guilhem Saurel's avatar
Guilhem Saurel committed
919
920
    repo.description = data['description']
    # TODO license (https://gitlab.com/gitlab-org/gitlab-ce/issues/28267), open_pr
921
    if 'forked_from_project' in data and data['forked_from_project'] is not None:
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
922
923
924
925
        repo.forked_from = data['forked_from_project']['id']
    elif created or project.main_namespace is None:
        project.main_namespace = namespace
        project.save()
926
    repo.save()
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
927
928
929


def update_github(forge, namespace, data):
930
931
    if data['archived']:
        return
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
932
    logger.info(f'update {data["name"]} from {forge}')
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
933
    project, _ = Project.objects.get_or_create(
Guilhem Saurel's avatar
Guilhem Saurel committed
934
935
        name=valid_name(data['name']),
        defaults={
Guilhem Saurel's avatar
yapf    
Guilhem Saurel committed
936
937
938
939
940
941
942
943
944
945
946
947
948
            'homepage': data['homepage'],
            'main_namespace': namespace,
            'main_forge': forge
        })
    repo, _ = Repo.objects.get_or_create(
        forge=forge,
        namespace=namespace,
        project=project,
        defaults={
            'repo_id': data['id'],
            'name': data['name'],
            'clone_url': data['clone_url']
        })
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
949
950
951
952
953
    repo.homepage = data['homepage']
    repo.url = data['html_url']
    repo.repo_id = data['id']
    repo.default_branch = data['default_branch']
    repo.open_issues = data['open_issues']
Guilhem Saurel's avatar
Guilhem Saurel committed
954
    repo.description = data['description']
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
955
956
957
958

    repo_data = repo.api_data()
    if repo_data and 'license' in repo_data and repo_data['license']:
        if 'spdx_id' in repo_data['license'] and repo_data['license']['spdx_id']:
959
960
961
962
963
964
965
966
            if repo_data['license']['spdx_id'] != 'NOASSERTION':
                try:
                    license = License.objects.get(spdx_id=repo_data['license']['spdx_id'])
                except License.DoesNotExist:
                    raise ValueError('No License with spdx_id=' + repo_data['license']['spdx_id'])
                repo.license = license
                if not project.license:
                    project.license = license
967
968
        if 'source' in repo_data:
            repo.forked_from = repo_data['source']['id']
969