Commit 4159650a authored by Guilhem Saurel's avatar Guilhem Saurel
Browse files

quick & dirty retries on network errors

parent 90bd990c
......@@ -2,8 +2,6 @@ import logging
from django.core.management.base import BaseCommand
import requests
from rainboard.models import Project
LICENSES = 'https://raw.githubusercontent.com/spdx/license-list-data/master/json/licenses.json'
......
......@@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand
import requests
from rainboard.models import Forge, License, Repo, Project
from rainboard.models import Forge, License, Repo
LICENSES = 'https://raw.githubusercontent.com/spdx/license-list-data/master/json/licenses.json'
logger = logging.getLogger('rainboard.management.populate')
......
......@@ -3,9 +3,8 @@ import logging
from django.conf import settings
from django.core.management.base import BaseCommand
import git
from rainboard.models import Project, Robotpkg
from rainboard.utils import update_robotpkg
logger = logging.getLogger('rainboard.robotpkg')
......@@ -17,8 +16,7 @@ class Command(BaseCommand):
path = settings.RAINBOARD_RPKG
logger.info('Pulling Robotpkg repositories')
git.Repo(str(path / '.git')).remotes.origin.pull()
git.Repo(str(path / 'wip' / '.git')).remotes.origin.pull()
update_robotpkg(path)
for project in Project.objects.all():
for pkg in path.glob(f'*/{project.slug}'):
......
......@@ -3,9 +3,8 @@ import logging
from django.conf import settings
from django.core.management.base import BaseCommand
import git
from rainboard.models import Branch, Project, Repo, Robotpkg
from rainboard.utils import update_robotpkg
logger = logging.getLogger('rainboard.management.update')
......@@ -25,8 +24,7 @@ class Command(BaseCommand):
branch.update(pull=False)
logger.info(f'\nPulling Robotpkg\n')
git.Repo(str(settings.RAINBOARD_RPKG / '.git')).remotes.origin.pull()
git.Repo(str(settings.RAINBOARD_RPKG / 'wip' / '.git')).remotes.origin.pull()
update_robotpkg(settings.RAINBOARD_RPKG)
logger.info(f'\nUpdating Robotpkg\n')
for robotpkg in Robotpkg.objects.all():
......
# Generated by Django 2.0.1 on 2018-02-06 18:47
import autoslug.fields
import django.db.models.deletion
from django.conf import settings
from django.db import migrations, models
import django.db.models.deletion
import autoslug.fields
import ndh.models
import rainboard.utils
......
......@@ -4,7 +4,6 @@ from subprocess import check_output
from django.conf import settings
from django.db import models
from django.urls import reverse
from django.utils.dateparse import parse_datetime
from django.utils.safestring import mark_safe
......@@ -14,7 +13,7 @@ from autoslug import AutoSlugField
from ndh.models import Links, NamedModel, TimeStampedModel
from ndh.utils import enum_to_choices, query_sum
from .utils import SOURCES, TARGETS, slugify_with_dots, api_next
from .utils import SOURCES, TARGETS, api_next, slugify_with_dots
logger = logging.getLogger('rainboard.models')
......@@ -59,7 +58,11 @@ class Forge(Links, NamedModel):
def api_req(self, url='', name=None, page=1):
logger.debug(f'requesting api {self} {url}, page {page}')
return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
try:
return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
except requests.exceptions.ConnectionError:
logger.error(f'requesting api {self} {url}, page {page} - SECOND TRY')
return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
def api_data(self, url=''):
req = self.api_req(url)
......@@ -247,7 +250,6 @@ class Project(Links, NamedModel, TimeStampedModel):
return query_sum(self.repo_set, 'open_pr')
class Repo(TimeStampedModel):
name = models.CharField(max_length=200)
slug = AutoSlugField(populate_from='name', slugify=slugify_with_dots)
......@@ -277,8 +279,13 @@ class Repo(TimeStampedModel):
def api_req(self, url='', name=None, page=1):
logger.debug(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}')
return requests.get(self.api_url() + url, {'page': page}, verify=self.forge.verify,
headers=self.forge.headers())
try:
return requests.get(self.api_url() + url, {'page': page}, verify=self.forge.verify,
headers=self.forge.headers())
except requests.exceptions.ConnectionError:
logger.error(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page} - SECOND TRY')
return requests.get(self.api_url() + url, {'page': page}, verify=self.forge.verify,
headers=self.forge.headers())
def api_data(self, url=''):
req = self.api_req(url)
......
import logging
import re
import unicodedata
from enum import IntEnum
from django.utils.safestring import mark_safe
import git
logger = logging.getLogger('rainboard.utils')
SOURCES = IntEnum('Sources', 'github gitlab redmine robotpkg travis')
TARGETS = IntEnum('Targets', '14.04 16.04 17.10 18.04 dubnium')
......@@ -36,6 +41,20 @@ def domain(url):
url = url.split('//')[1]
return url.split('/')[0]
def domain_link(url):
dn = domain(url)
return mark_safe(f'<a href="{url}">{dn}</a>')
def update_robotpkg(path):
try:
git.Repo(str(path / '.git')).remotes.origin.pull()
except git.exc.GitCommandError:
logger.error('Network error, retrying…')
git.Repo(str(path / '.git')).remotes.origin.pull()
try:
git.Repo(str(path / 'wip' / '.git')).remotes.origin.pull()
except git.exc.GitCommandError:
logger.error('Network error, retrying…')
git.Repo(str(path / 'wip' / '.git')).remotes.origin.pull()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment