models.py

import json
import logging
import re
from subprocess import check_output

from django.conf import settings
from django.db import models
from django.db.models import Q
from django.db.utils import DataError, IntegrityError
from django.template.loader import get_template
from django.utils import timezone
from django.utils.dateparse import parse_datetime
from django.utils.safestring import mark_safe

import git
import httpx
from autoslug import AutoSlugField
from autoslug.utils import slugify
from github import Github
from gitlab import Gitlab
from ndh.models import Links, NamedModel, TimeStampedModel
from ndh.utils import enum_to_choices, query_sum

from .utils import SOURCES, api_next, invalid_mail, slugify_with_dots, valid_name

logger = logging.getLogger('rainboard.models')

MAIN_BRANCHES = ['master', 'devel']
RPKG_URL = 'http://robotpkg.openrobots.org'
DOC_URL = 'https://gepettoweb.laas.fr/doc'
RPKG_LICENSES = {
    'gnu-lgpl-v3': 'LGPL-3.0',
    'gnu-lgpl-v2': 'LGPL-2.0',
    'gnu-lgpl-v2.1': 'LGPL-2.1',
    'mit': 'MIT',
    'gnu-gpl-v3': 'GPL-3.0',
    '2-clause-bsd': 'BSD-2-Clause',
    'eclipse': 'EPL-1.0',
    'modified-bsd': 'BSD-3-Clause'
}
RPKG_FIELDS = ['PKGBASE', 'PKGVERSION', 'MASTER_SITES', 'MASTER_REPOSITORY', 'MAINTAINER', 'COMMENT', 'HOMEPAGE']
CMAKE_FIELDS = {
    'NAME': 'cmake_name',
    'DESCRIPTION': 'description',
    'URL': 'homepage',
    'VERSION': 'version',
    'SUFFIX': 'suffix'
}
TRAVIS_STATE = {'created': None, 'passed': True, 'started': None, 'failed': False, 'errored': False, 'canceled': False}
GITLAB_STATUS = {'failed': False, 'success': True, 'pending': None, 'skipped': None, 'canceled': None, 'running': None}
GEPETTO_SLUGS = ['gepetto', 'stack-of-tasks', 'humanoid-path-planner', 'loco-3d']

BAD_ONES = Q(main_namespace__from_gepetto=False) | Q(robotpkg__isnull=True) | Q(archived=True)


class Namespace(NamedModel):
    group = models.BooleanField(default=False)
    from_gepetto = models.BooleanField(default=False)
    slug_gitlab = models.CharField(max_length=200, default='')
    slug_github = models.CharField(max_length=200, default='')

    def save(self, *args, **kwargs):
        if self.slug_gitlab == '':
            self.slug_gitlab = self.slug
        if self.slug_github == '':
            self.slug_github = self.slug
        super(Namespace, self).save(*args, **kwargs)


class License(models.Model):
    name = models.CharField(max_length=200)
    spdx_id = models.CharField(max_length=50, unique=True)
    url = models.URLField(max_length=200)

    def __str__(self):
        return self.spdx_id or self.name


class Forge(Links, NamedModel):
    source = models.PositiveSmallIntegerField(choices=enum_to_choices(SOURCES))
    url = models.URLField(max_length=200)
    token = models.CharField(max_length=50, blank=True, null=True)
    verify = models.BooleanField(default=True)

    def get_absolute_url(self):
        return self.url

    def api_req(self, url='', name=None, page=1):
        logger.debug(f'requesting api {self} {url}, page {page}')
        try:
            return httpx.get(self.api_url() + url, params={'page': page}, verify=self.verify, headers=self.headers())
        except httpx.HTTPError:
            logger.error(f'requesting api {self} {url}, page {page} - SECOND TRY')
            return httpx.get(self.api_url() + url, params={'page': page}, verify=self.verify, headers=self.headers())

    def api_data(self, url=''):
        req = self.api_req(url)
        return req.json() if req.status_code == 200 else []  # TODO

    def api_list(self, url='', name=None, limit=None):
        page = 1
        while page:
            req = self.api_req(url, name, page)
            if req.status_code != 200:
                return []  # TODO
            data = req.json()
            if name is not None:
                data = data[name]
            yield from data
            page = api_next(self.source, req)
            if limit is not None and page is not None and page > limit:
                break

    def headers(self):
        return {
            SOURCES.github: {
                'Authorization': f'token {self.token}',
                'Accept': 'application/vnd.github.drax-preview+json'
            },
            SOURCES.gitlab: {
                'Private-Token': self.token
            },
            SOURCES.redmine: {
                'X-Redmine-API-Key': self.token
            },
            SOURCES.travis: {
                'Authorization': f'token {self.token}',
                'TRAVIS-API-Version': '3'
            },
        }[self.source]

    def api_url(self):
        return {
            SOURCES.github: 'https://api.github.com',
            SOURCES.gitlab: f'{self.url}/api/v4',
            SOURCES.redmine: self.url,
            SOURCES.travis: 'https://api.travis-ci.org',
        }[self.source]

    def get_namespaces_github(self):
        for namespace in Namespace.objects.filter(group=True):
            for data in self.api_list(f'/orgs/{namespace.slug}/members'):
                Namespace.objects.get_or_create(slug=data['login'].lower(),
                                                defaults={
                                                    'name': data['login'],
                                                    'group': False
                                                })

    def get_namespaces_gitlab(self):
        for data in self.api_list('/namespaces'):
            if data['name'] == 'dockering':
                continue
            try:
                Namespace.objects.get_or_create(slug=slugify(data['path']),
                                                defaults={
                                                    'name': data['name'],
                                                    'group': data['kind'] == 'group'
                                                })
            except IntegrityError:
                Namespace.objects.get_or_create(slug=slugify(data['path']),
                                                defaults={
                                                    'name': data['name'] + ' 2',
                                                    'group': data['kind'] == 'group'
                                                })
        for data in self.api_list('/users'):
            if data['name'] == 'dockering':
                continue
            Namespace.objects.get_or_create(slug=slugify(data['username']), defaults={'name': data['name']})

    def get_namespaces_redmine(self):
        pass  # TODO

    def get_namespaces_travis(self):
        pass

    def get_projects(self):
        getattr(self, f'get_namespaces_{self.get_source_display()}')()
        return getattr(self, f'get_projects_{self.get_source_display()}')()

    def get_projects_github(self):
        for org in Namespace.objects.filter(group=True):
            for data in self.api_list(f'/orgs/{org.slug}/repos'):
                update_github(self, org, data)
        for user in Namespace.objects.filter(group=False):
            for data in self.api_list(f'/users/{user.slug}/repos'):
                if Project.objects.filter(name=valid_name(data['name'])).exists():
                    update_github(self, user, data)

    def get_projects_gitlab(self):
        for data in self.api_list('/projects'):
            update_gitlab(self, data)

        for orphan in Project.objects.filter(main_namespace=None).exclude(name__endswith='release'):
            repo = orphan.repo_set.filter(forge__source=SOURCES.gitlab).first()
            if repo:
                update_gitlab(self, self.api_data(f'/projects/{repo.forked_from}'))

    def get_projects_redmine(self):
        pass  # TODO

    def get_projects_travis(self):
        for namespace in Namespace.objects.all():
            for repository in self.api_list(f'/owner/{namespace.slug}/repos', 'repositories'):
                if repository['active']:
                    update_travis(namespace, repository)


class Project(Links, NamedModel, TimeStampedModel):
    public = models.BooleanField(default=True)
    main_namespace = models.ForeignKey(Namespace, on_delete=models.SET_NULL, null=True, blank=True)
    main_forge = models.ForeignKey(Forge, on_delete=models.SET_NULL, null=True, blank=True)
    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    homepage = models.URLField(max_length=200, blank=True, null=True)
    description = models.TextField(blank=True, null=True)
    version = models.CharField(max_length=20, blank=True, null=True)
    updated = models.DateTimeField(blank=True, null=True)
    tests = models.BooleanField(default=True)
    docs = models.BooleanField(default=True)
    cmake_name = models.CharField(max_length=200, blank=True, null=True)
    archived = models.BooleanField(default=False)
    suffix = models.CharField(max_length=50, default='', blank=True)
    allow_format_failure = models.BooleanField(default=True)
    has_python = models.BooleanField(default=True)
    accept_pr_to_master = models.BooleanField(default=False)
    ccache = models.BooleanField(default=True)

    def save(self, *args, **kwargs):
        self.name = valid_name(self.name)
        super().save(*args, **kwargs)

    def git_path(self):
        return settings.RAINBOARD_GITS / self.main_namespace.slug / self.slug.strip()  # workaround SafeText TypeError

    def git(self):
        path = self.git_path()
        if not path.exists():
            logger.info(f'Creating repo for {self.main_namespace.slug}/{self.slug}')
            return git.Repo.init(path)
        return git.Repo(str(path / '.git'))

    def github(self):
        github_forge = Forge.objects.get(slug='github')
        gh = Github(github_forge.token)
        return gh.get_repo(f'{self.main_namespace.slug_github}/{self.slug}')

    def gitlab(self):
        gitlab_forge = Forge.objects.get(slug='gitlab')
        gl = Gitlab(gitlab_forge.url, private_token=gitlab_forge.token)
        gl_repo = gl.projects.get(f'{self.main_namespace.slug_gitlab}/{self.slug}')
        return gl_repo

    def main_repo(self):
        forge = self.main_forge if self.main_forge else get_default_forge(self)
        repo, created = Repo.objects.get_or_create(forge=forge,
                                                   namespace=self.main_namespace,
                                                   project=self,
                                                   defaults={
                                                       'name': self.name,
                                                       'default_branch': 'master',
                                                       'repo_id': 0
                                                   })
        if created:
            repo.api_update()
        return repo

    def update_branches(self, main=True, pull=True):
        branches = [b[2:] for b in self.git().git.branch('-a', '--no-color').split('\n')]
        if main:
            branches = [b for b in branches if b.endswith('master') or b.endswith('devel')]
        for branch in branches:
            logger.info(f'update branch {branch}')
            if branch.startswith('remotes/'):
                branch = branch[8:]
            if branch.count('/') < 2:
                if branch != 'master':
                    logger.error(f'wrong branch "{branch}" in {self.git_path()}')
                continue
            forge, namespace, name = branch.split('/', maxsplit=2)
            namespace, _ = Namespace.objects.get_or_create(slug=slugify(namespace), defaults={'name': namespace})
            forge = Forge.objects.get(slug=forge)
            repo, created = Repo.objects.get_or_create(forge=forge,
                                                       namespace=namespace,
                                                       project=self,
                                                       defaults={
                                                           'name': self.name,
                                                           'default_branch': 'master',
                                                           'repo_id': 0
                                                       })
            if created:
                repo.api_update()
            instance, bcreated = Branch.objects.get_or_create(name=branch, project=self, repo=repo)
            if bcreated:
                instance.update(pull=pull)

    def checkout(self):
        self.main_branch().git().checkout()

    def main_branch(self):
        return self.main_repo().main_branch()

    def cmake(self):
        filename = self.git_path() / 'CMakeLists.txt'
        if not filename.exists():
            return
        with filename.open() as f:
            content = f.read()
        for key, value in CMAKE_FIELDS.items():
            search = re.search(r'set\s*\(\s*project_%s\s+([^)]+)*\)' % key, content, re.I)
            if search:
                try:
                    old = getattr(self, value)
                    new = search.groups()[0].strip(''' \r\n\t'"''').replace('_', '-')
                    if old != new:
                        setattr(self, value, new)
                        self.save()
                except DataError:
                    setattr(self, value, old)
        for dependency in re.findall(r'ADD_[A-Z]+_DEPENDENCY\s*\(["\']?([^ "\')]+).*["\']?\)', content, re.I):
            project = Project.objects.filter(name=valid_name(dependency))
            if project.exists():
                dependency, _ = Dependency.objects.get_or_create(project=self, library=project.first())
                if not dependency.cmake:
                    dependency.cmake = True
                    dependency.save()

    def ros(self):
        try:
            filename = self.git_path() / 'package.xml'
        except TypeError:
            return
        if not filename.exists():
            return
        with filename.open() as f:
            content = f.read()
        for dependency in re.findall(r'<run_depend>(\w+).*</run_depend>', content, re.I):
            project = Project.objects.filter(name=valid_name(dependency))
            if project.exists():
                dependency, _ = Dependency.objects.get_or_create(project=self, library=project.first())
                if not dependency.ros:
                    dependency.ros = True
                    dependency.save()

    def repos(self):
        return self.repo_set.count()

    def rpkgs(self):
        return self.robotpkg_set.count()

    def update_tags(self):
        for tag in self.git().tags:
            Tag.objects.get_or_create(name=str(tag), project=self)

    def update_repo(self):
        branch = str(self.main_branch()).split('/', maxsplit=2)[2]
        self.git().head.commit = self.git().remotes[self.main_repo().git_remote()].refs[branch].commit

    def main_gitlab_repo(self):
        return self.repo_set.get(forge__source=SOURCES.gitlab, namespace=self.main_namespace)

    def ci_jobs(self):
        r = self.main_gitlab_repo()
        r.get_jobs_gitlab()
        r.get_builds_gitlab()

    def update(self, only_main_branches=True):
        if self.main_namespace is None:
            return
        self.update_branches(main=only_main_branches)
        self.update_tags()
        self.update_repo()
        tag = self.tag_set.filter(name__startswith='v').last()  # TODO: implement SQL ordering for semver
        if tag is not None:
            self.version = tag.name[1:]
        robotpkg = self.robotpkg_set.order_by('-updated').first()
        branch = self.branch_set.order_by('-updated').first()
        branch_updated = branch is not None and branch.updated is not None
        robotpkg_updated = robotpkg is not None and robotpkg.updated is not None
        if branch_updated or robotpkg_updated:
            if not robotpkg_updated:
                self.updated = branch.updated
            elif not branch_updated:
                self.updated = robotpkg.updated
            else:
                self.updated = max(branch.updated, robotpkg.updated)
        self.ci_jobs()
        self.checkout()
        self.cmake()
        self.ros()
        self.save()

    def commits_since(self):
        try:
            commits = self.git().git.rev_list(f'v{self.version}..{self.main_branch()}')
            return len(commits.split('\n')) if commits else 0
        except git.exc.GitCommandError:
            pass

    def open_issues(self):
        return query_sum(self.repo_set, 'open_issues')

    def open_pr(self):
        return query_sum(self.repo_set, 'open_pr')

    def gitlabciyml(self):
        return get_template('rainboard/gitlab-ci.yml').render({'project': self})

    def contributors(self, update=False):
        if update:
            for guy in self.git().git.shortlog('-nse').split('\n'):
                name, mail = guy[7:-1].split(' <')
                contributor = get_contributor(name, mail)
                contributor.projects.add(self)
                contributor.save()
        return self.contributor_set.all()

    def registry(self):
        return settings.PUBLIC_REGISTRY if self.public else settings.PRIVATE_REGISTRY

    def doc_coverage_image(self):
        images = Image.objects.filter(robotpkg__project=self, target__main=True)
        return images.order_by('-py3', '-debug').first()  # 18.04 / Python 3 / Debug is the preferred image

    def print_deps(self):
        return mark_safe(', '.join(d.library.get_link() for d in self.dependencies.all()))

    def print_rdeps(self):
        return mark_safe(', '.join(d.project.get_link() for d in self.rdeps.all()))

    def ordered_robotpkg(self):
        return self.robotpkg_set.order_by('name')

    def url_travis(self):
        return f'https://travis-ci.org/{self.main_namespace.slug}/{self.slug}'

    def url_gitlab(self):
        return f'https://gitlab.laas.fr/{self.main_namespace.slug_gitlab}/{self.slug}'

    def remote_url_gitlab(self):
        gitlab_forge = Forge.objects.get(source=SOURCES.gitlab)
        return self.url_gitlab().replace('://', f'://gitlab-ci-token:{gitlab_forge.token}@')

    def url_github(self):
        return f'https://github.com/{self.main_namespace.slug_github}/{self.slug}'

    def remote_url_github(self):
        github_forge = Forge.objects.get(source=SOURCES.github)
        return self.url_github().replace('://', f'://{settings.GITHUB_USER}:{github_forge.token}@')

    def badge(self, link, img, alt):
        return mark_safe(f'<a href="{link}"><img src="{img}" alt="{alt}" /></a> ')

    def badge_travis(self):
        return self.badge(self.url_travis(), f'{self.url_travis()}.svg?branch=master', 'Building Status')

    def badge_gitlab(self):
        return self.badge(self.url_gitlab(), f'{self.url_gitlab()}/badges/master/pipeline.svg', 'Pipeline Status')

    def badge_coverage(self):
        return self.badge(f'{DOC_URL}/{self.main_namespace.slug}/{self.slug}/master/coverage',
                          f'{self.url_gitlab()}/badges/master/coverage.svg?job=doc-coverage"', 'Coverage Report')

    def badges(self):
        # travis = self.badge_travis() if self.public else mark_safe('')
        return self.badge_gitlab() + self.badge_coverage()

    def cron(self):
        """ generate a cron-style interval description to run CI monthly on master """
        hour, day = (self.pk // 30) % 24, self.pk % 30 + 1
        return f'0 {hour} {day} * *'

    def pipeline_schedules(self):
        """ provides a link to gitlab's CI schedules page showing then cron rule to use with this project """
        repo = self.repo_set.filter(forge__source=SOURCES.gitlab, namespace__group=True)
        if repo.exists():
            link = repo.first().url + '/pipeline_schedules'
            return mark_safe(f'<a href="{link}">{self.cron()}</a>')

    def pipeline_results(self):
        """ Show state and link to latest master & devel gitlab pipelines """
        repo = self.main_gitlab_repo()
        ret = []
        for branch in ['master', 'main', 'devel']:
            build = repo.cibuild_set.filter(branch__name__endswith=branch).first()
            if build is not None:
                link = repo.url + f'/-/pipelines/{build.build_id}'
                sym = '✓' if build.passed else '✘'
                ret.append(f'<a href="{link}">{branch}: {sym}</a>')

        return mark_safe('<br>'.join(ret))


class Repo(TimeStampedModel):
    name = models.CharField(max_length=200)
    slug = AutoSlugField(populate_from='name', slugify=slugify_with_dots)
    forge = models.ForeignKey(Forge, on_delete=models.CASCADE)
    namespace = models.ForeignKey(Namespace, on_delete=models.CASCADE)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    homepage = models.URLField(max_length=200, blank=True, null=True)
    url = models.URLField(max_length=200, blank=True, null=True)
    default_branch = models.CharField(max_length=50)
    open_issues = models.PositiveSmallIntegerField(blank=True, null=True)
    open_pr = models.PositiveSmallIntegerField(blank=True, null=True)
    repo_id = models.PositiveIntegerField()
    forked_from = models.PositiveIntegerField(blank=True, null=True)
    clone_url = models.URLField(max_length=200)
    travis_id = models.PositiveIntegerField(blank=True, null=True)
    description = models.TextField(blank=True, null=True)
    archived = models.BooleanField(default=False)

    def __str__(self):
        return self.name

    def api_url(self):
        api_url = self.forge.api_url()
        return {
            SOURCES.github: f'{api_url}/repos/{self.namespace.slug}/{self.slug}',
            SOURCES.redmine: f'{api_url}/projects/{self.repo_id}.json',
            SOURCES.gitlab: f'{api_url}/projects/{self.repo_id}',
        }[self.forge.source]

    def api_req(self, url='', name=None, page=1):
        logger.debug(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}')
        try:
            return httpx.get(self.api_url() + url,
                             params={'page': page},
                             verify=self.forge.verify,
                             headers=self.forge.headers())
        except httpx.HTTPError:
            logger.error(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page} - SECOND TRY')
            return httpx.get(self.api_url() + url,
                             params={'page': page},
                             verify=self.forge.verify,
                             headers=self.forge.headers())

    def api_data(self, url=''):
        req = self.api_req(url)
        return req.json() if req.status_code == 200 else []  # TODO

    def api_list(self, url='', name=None, limit=None):
        page = 1
        while page:
            req = self.api_req(url, name, page)
            if req.status_code != 200:
                return []  # TODO
            data = req.json()
            if name is not None:
                if name in data:
                    data = data[name]
                else:
                    return []  # TODO
            yield from data
            page = api_next(self.forge.source, req)
            if limit is not None and page is not None and page > limit:
                break

    def api_update(self):
        data = self.api_data()
        if data:
            if data['archived']:
                if self.project.main_repo() == self:
                    self.project.archived = True
                    self.project.save()
                self.archived = True
                self.save()
            else:
                return getattr(self, f'api_update_{self.forge.get_source_display()}')(data)

    def api_update_gitlab(self, data):
        update_gitlab(self.forge, data)

    def api_update_github(self, data):
        update_github(self.forge, self.namespace, data)

    def get_clone_url(self):
        if self.forge.source == SOURCES.gitlab:
            return self.clone_url.replace('://', f'://gitlab-ci-token:{self.forge.token}@')
        if self.forge.source == SOURCES.github:
            return self.clone_url.replace('://', f'://{settings.GITHUB_USER}:{self.forge.token}@')
        return self.clone_url

    def git_remote(self):
        return f'{self.forge.slug}/{self.namespace.slug}'

    def git(self):
        git_repo = self.project.git()
        remote = self.git_remote()
        try:
            return git_repo.remote(remote)
        except ValueError:
            logger.info(f'Creating remote {remote}')
            return git_repo.create_remote(remote, self.get_clone_url())

    def fetch(self):
        git_repo = self.git()
        logger.debug(f'fetching {self.forge} / {self.namespace} / {self.project}')
        try:
            git_repo.fetch()
        except git.exc.GitCommandError:
            logger.warning(f'fetching {self.forge} / {self.namespace} / {self.project} - SECOND TRY')
            try:
                git_repo.fetch()
            except git.exc.GitCommandError:
                return False
        return True

    def main_branch(self):
        return self.project.branch_set.get(name=f'{self.git_remote()}/{self.default_branch}')

    def ahead(self):
        main_branch = self.main_branch()
        return main_branch.ahead if main_branch is not None else 0

    def behind(self):
        main_branch = self.main_branch()
        return main_branch.behind if main_branch is not None else 0

    def get_builds(self):
        return getattr(self, f'get_builds_{self.forge.get_source_display()}')()

    def get_builds_gitlab(self):
        for pipeline in self.api_list('/pipelines', limit=2):
            pid, ref = pipeline['id'], pipeline['ref']
            if self.project.tag_set.filter(name=ref).exists():
                continue
            data = self.api_data(f'/pipelines/{pid}')
            branch_name = f'{self.forge.slug}/{self.namespace.slug}/{ref}'
            branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
            if created:
                branch.update()
            ci_build, created = CIBuild.objects.get_or_create(repo=self,
                                                              build_id=pid,
                                                              defaults={
                                                                  'passed': GITLAB_STATUS[pipeline['status']],
                                                                  'started': parse_datetime(data['created_at']),
                                                                  'branch': branch,
                                                              })
            if not created and ci_build.passed != GITLAB_STATUS[pipeline['status']]:
                ci_build.passed = GITLAB_STATUS[pipeline['status']]
                ci_build.save()

    def get_jobs_gitlab(self):
        for data in self.api_list('/jobs', limit=4):
            branch_name = f'{self.forge.slug}/{self.namespace.slug}/{data["ref"]}'
            branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
            if created:
                branch.update()
            ci_job, created = CIJob.objects.get_or_create(repo=self,
                                                          job_id=data['id'],
                                                          defaults={
                                                              'passed': GITLAB_STATUS[data['status']],
                                                              'started': parse_datetime(data['created_at']),
                                                              'branch': branch,
                                                          })
            if not created and ci_job.passed != GITLAB_STATUS[data['status']]:
                ci_job.passed = GITLAB_STATUS[data['status']]
                ci_job.save()
            if self == self.project.main_repo():
                if data['name'] == 'format':
                    if self.project.allow_format_failure and GITLAB_STATUS[data['status']]:
                        self.project.allow_format_failure = False
                        self.project.save()
                        print(' format success', data['web_url'])
                elif data['name'].startswith('robotpkg-'):
                    py3 = '-py3' in data['name']
                    debug = '-debug' in data['name']
                    target = next(target for target in Target.objects.all() if target.name in data['name']).name
                    robotpkg = data['name'][9:-(3 + len(target) + (5 if debug else 7) + (3 if py3 else 0))]  # shame.
                    images = Image.objects.filter(robotpkg__name=robotpkg, target__name=target, debug=debug, py3=py3)
                    if not images.exists():
                        continue
                    image = images.first()
                    if image.allow_failure and GITLAB_STATUS[data['status']]:
                        image.allow_failure = False
                        image.save()
                        print('  success', data['web_url'])

    def get_builds_github(self):
        if self.travis_id is not None:
            travis = Forge.objects.get(source=SOURCES.travis)
            for build in travis.api_list(f'/repo/{self.travis_id}/builds', name='builds'):
                if build['branch'] is None or self.project.tag_set.filter(name=build['branch']['name']).exists():
                    continue
                branch_name = f'{self.forge.slug}/{self.namespace.slug}/{build["branch"]["name"]}'
                branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
                if created:
                    branch.update()
                started = build['started_at'] if build['started_at'] is not None else build['finished_at']
                CIBuild.objects.get_or_create(repo=self,
                                              build_id=build['id'],
                                              defaults={
                                                  'passed': TRAVIS_STATE[build['state']],
                                                  'started': parse_datetime(started),
                                                  'branch': branch,
                                              })

    def update(self, pull=True):
        ok = True
        if self.project.main_namespace is None:
            return
        self.project.update_tags()
        if pull:
            ok = self.fetch()
        if ok:
            self.api_update()
            self.get_builds()
        else:
            logger.error(f'fetching {self.forge} / {self.namespace} / {self.project} - NOT FOUND - DELETING')
            logger.error(str(self.delete()))


class IssuePr(models.Model):
    title = models.CharField(max_length=200)
    repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
    number = models.PositiveSmallIntegerField()
    days_since_updated = models.PositiveSmallIntegerField(blank=True, null=True)
    url = models.URLField(max_length=200)
    is_issue = models.BooleanField(default=True)

    class Meta:
        unique_together = ('repo', 'number', 'is_issue')

    def update(self, skip_label):
        gh = self.repo.project.github()
        issue_pr = gh.get_issue(number=self.number) if self.is_issue else gh.get_pull(number=self.number)
        self.days_since_updated = (timezone.now() - timezone.make_aware(issue_pr.updated_at)).days
        if issue_pr.state == 'closed' or skip_label in [label.name for label in issue_pr.get_labels()]:
            self.delete()
        else:
            self.save()


class Commit(NamedModel, TimeStampedModel):
    project = models.ForeignKey(Project, on_delete=models.CASCADE)

    class Meta:
        unique_together = ('project', 'name')


class Branch(TimeStampedModel):
    name = models.CharField(max_length=200)
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    ahead = models.PositiveSmallIntegerField(blank=True, null=True)
    behind = models.PositiveSmallIntegerField(blank=True, null=True)
    updated = models.DateTimeField(blank=True, null=True)
    repo = models.ForeignKey(Repo, on_delete=models.CASCADE)
    deleted = models.BooleanField(default=False)
    keep_doc = models.BooleanField(default=False)

    def __str__(self):
        return self.name

    class Meta:
        unique_together = ('project', 'name', 'repo')

    def get_ahead(self, branch='master'):
        commits = self.project.git().git.rev_list(f'{branch}..{self}')
        return len(commits.split('\n')) if commits else 0

    def get_behind(self, branch='master'):
        commits = self.project.git().git.rev_list(f'{self}..{branch}')
        return len(commits.split('\n')) if commits else 0

    def git(self):
        git_repo = self.project.git()
        if self.name not in git_repo.branches:
            remote = self.repo.git()
            _, _, branch = self.name.split('/', maxsplit=2)
            git_repo.create_head(self.name, remote.refs[branch]).set_tracking_branch(remote.refs[branch])
        return git_repo.branches[self.name]

    def update(self, pull=True):
        if self.deleted:
            return
        try:
            if pull:
                self.repo.fetch()
                if self.repo != self.project.main_repo():
                    self.project.main_repo().fetch()
            try:
                main_branch = self.project.main_branch()
                self.ahead = self.get_ahead(main_branch)
                self.behind = self.get_behind(main_branch)
            except Branch.DoesNotExist:
                pass
            self.updated = self.git().commit.authored_datetime
        except (git.exc.GitCommandError, IndexError):
            self.deleted = True
        self.save()

    def ci(self):
        build = self.cibuild_set.last()
        if build is None:
            return ''
        status = {True: '✓', False: '✗', None: '?'}[build.passed]
        return mark_safe(f'<a href="{build.url()}">{status}</a>')

    def forge(self):
        return self.repo.forge

    def namespace(self):
        return self.repo.namespace


class TargetQuerySet(models.QuerySet):
    def active(self):
        return self.filter(active=True)

    def main(self):
        return self.get(main=True)


class Target(NamedModel):
    active = models.BooleanField(default=True)
    main = models.BooleanField(default=False)
    py2_available = models.BooleanField(default=True)
    public = models.BooleanField(default=True)

    objects = TargetQuerySet.as_manager()


# class Test(TimeStampedModel):
#     project = models.ForeignKey(Project, on_delete=models.CASCADE)
#     branch = models.ForeignKey(Branch, on_delete=models.CASCADE)
#     commit = models.ForeignKey(Commit, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)
#     passed = models.BooleanField(default=False)
#     TODO: travis vs gitlab-ci ?
#     TODO: deploy binary, doc, coverage, lint

# class SystemDependency(NamedModel):
#     project = models.ForeignKey(Project, on_delete=models.CASCADE)
#     target = models.ForeignKey(Target, on_delete=models.CASCADE)


class Robotpkg(NamedModel):
    project = models.ForeignKey(Project, on_delete=models.CASCADE)
    category = models.CharField(max_length=50)

    pkgbase = models.CharField(max_length=50, default='')
    pkgversion = models.CharField(max_length=50, default='')
    master_sites = models.CharField(max_length=200, default='')
    master_repository = models.CharField(max_length=200, default='')
    maintainer = models.CharField(max_length=200, default='')
    comment = models.TextField()
    homepage = models.URLField(max_length=200, blank=True, null=True)

    license = models.ForeignKey(License, on_delete=models.SET_NULL, blank=True, null=True)
    public = models.BooleanField(default=True)
    description = models.TextField(blank=True, null=True)
    updated = models.DateTimeField(blank=True, null=True)

    same_py = models.BooleanField(default=True)

    extended_target = models.ManyToManyField(Target)

    def main_page(self):
        if self.category != 'wip':
            return f'{RPKG_URL}/robotpkg/{self.category}/{self.name}'

    def build_page(self):
        path = '-wip/wip' if self.category == 'wip' else f'/{self.category}'
        return f'{RPKG_URL}/rbulk/robotpkg{path}/{self.name}'

    def update_images(self):
        py3s = [False, True] if self.name.startswith('py-') else [False]
        debugs = [False, True]
        for target in list(Target.objects.active()) + list(self.extended_target.all()):
            for py3 in py3s:
                for debug in debugs:
                    if target.py2_available or py3 or not self.name.startswith('py-'):
                        Image.objects.get_or_create(robotpkg=self, target=target, py3=py3, debug=debug)[0].update()

    def update(self, pull=True):
        path = settings.RAINBOARD_RPKG
        repo = git.Repo(str(path / 'wip' / '.git' if self.category == 'wip' else path / '.git'))
        if pull:
            repo.remotes.origin.pull()

        cwd = path / self.category / self.name
        if not cwd.is_dir():
            logger.warning(f'deleted {self}: {self.delete()}')
            return
        for field in RPKG_FIELDS:
            cmd = ['make', 'show-var', f'VARNAME={field}']
            self.__dict__[field.lower()] = check_output(cmd, cwd=cwd).decode().strip()

        repo_path = self.name if self.category == 'wip' else f'{self.category}/{self.name}'
        last_commit = next(repo.iter_commits(paths=repo_path, max_count=1))
        self.updated = last_commit.authored_datetime

        license = check_output(['make', 'show-var', 'VARNAME=LICENSE'], cwd=cwd).decode().strip()
        if license in RPKG_LICENSES:
            self.license = License.objects.get(spdx_id=RPKG_LICENSES[license])
        else:
            logger.warning(f'Unknown robotpkg license: {license}')
        self.public = not bool(check_output(['make', 'show-var', 'VARNAME=RESTRICTED'], cwd=cwd).decode().strip())
        with (cwd / 'DESCR').open() as f:
            self.description = f.read().strip()

        self.update_images()
        self.save()

    def valid_images(self):
        def is_valid(_base_image, _image):
            """The image is valid if it has less than one different parameter from the base image."""
            if _image == self.project.doc_coverage_image():
                #  There is no need to keep this image because it is already tested by doc-coverage
                return False

            base_param = (_base_image.target.name, _base_image.py3, _base_image.debug)
            param = (_image.target.name, _image.py3, _image.debug)

            diff = 0  # Number of different parameters between the base image and the current image
            for i in range(len(base_param)):
                if base_param[i] != param[i]:
                    diff += 1
            return diff <= 1

        # 18.04 / Python 3 / Release is the preferred base image
        base_images = Image.objects.filter(robotpkg__project=self.project, target__main=True)
        base_image = base_images.order_by('-py3', 'debug').first()

        images = self.image_set.filter(Q(target__active=True) | Q(target__robotpkg=self),
                                       created__isnull=False).order_by('target__name')
        return (image for image in images if is_valid(base_image, image))

    def without_py(self):