quick & dirty retries on network errors

rainboard/management/commands/delete_perso.py

@@ -2,8 +2,6 @@ import logging
 
 from django.core.management.base import BaseCommand
 
-import requests
-
 from rainboard.models import Project
 
 LICENSES = 'https://raw.githubusercontent.com/spdx/license-list-data/master/json/licenses.json'

rainboard/management/commands/populate.py

@@ -4,7 +4,7 @@ from django.core.management.base import BaseCommand
 
 import requests
 
-from rainboard.models import Forge, License, Repo, Project
+from rainboard.models import Forge, License, Repo
 
 LICENSES = 'https://raw.githubusercontent.com/spdx/license-list-data/master/json/licenses.json'
 logger = logging.getLogger('rainboard.management.populate')

rainboard/management/commands/robotpkg.py

@@ -3,9 +3,8 @@ import logging
 from django.conf import settings
 from django.core.management.base import BaseCommand
 
-import git
-
 from rainboard.models import Project, Robotpkg
+from rainboard.utils import update_robotpkg
 
 logger = logging.getLogger('rainboard.robotpkg')
 
@@ -17,8 +16,7 @@ class Command(BaseCommand):
         path = settings.RAINBOARD_RPKG
 
         logger.info('Pulling Robotpkg repositories')
-        git.Repo(str(path / '.git')).remotes.origin.pull()
-        git.Repo(str(path / 'wip' / '.git')).remotes.origin.pull()
+        update_robotpkg(path)
 
         for project in Project.objects.all():
             for pkg in path.glob(f'*/{project.slug}'):

rainboard/management/commands/update.py

@@ -3,9 +3,8 @@ import logging
 from django.conf import settings
 from django.core.management.base import BaseCommand
 
-import git
-
 from rainboard.models import Branch, Project, Repo, Robotpkg
+from rainboard.utils import update_robotpkg
 
 logger = logging.getLogger('rainboard.management.update')
 
@@ -25,8 +24,7 @@ class Command(BaseCommand):
             branch.update(pull=False)
 
         logger.info(f'\nPulling Robotpkg\n')
-        git.Repo(str(settings.RAINBOARD_RPKG / '.git')).remotes.origin.pull()
-        git.Repo(str(settings.RAINBOARD_RPKG / 'wip' / '.git')).remotes.origin.pull()
+        update_robotpkg(settings.RAINBOARD_RPKG)
 
         logger.info(f'\nUpdating Robotpkg\n')
         for robotpkg in Robotpkg.objects.all():

rainboard/migrations/0001_initial.py

 # Generated by Django 2.0.1 on 2018-02-06 18:47
 
-import autoslug.fields
+import django.db.models.deletion
 from django.conf import settings
 from django.db import migrations, models
-import django.db.models.deletion
+
+import autoslug.fields
 import ndh.models
+
 import rainboard.utils
 
 

rainboard/models.py

@@ -4,7 +4,6 @@ from subprocess import check_output
 
 from django.conf import settings
 from django.db import models
-from django.urls import reverse
 from django.utils.dateparse import parse_datetime
 from django.utils.safestring import mark_safe
 
@@ -14,7 +13,7 @@ from autoslug import AutoSlugField
 from ndh.models import Links, NamedModel, TimeStampedModel
 from ndh.utils import enum_to_choices, query_sum
 
-from .utils import SOURCES, TARGETS, slugify_with_dots, api_next
+from .utils import SOURCES, TARGETS, api_next, slugify_with_dots
 
 logger = logging.getLogger('rainboard.models')
 
@@ -59,7 +58,11 @@ class Forge(Links, NamedModel):
 
     def api_req(self, url='', name=None, page=1):
         logger.debug(f'requesting api {self} {url}, page {page}')
-        return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
+        try:
+            return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
+        except requests.exceptions.ConnectionError:
+            logger.error(f'requesting api {self} {url}, page {page} - SECOND TRY')
+            return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
 
     def api_data(self, url=''):
         req = self.api_req(url)
@@ -247,7 +250,6 @@ class Project(Links, NamedModel, TimeStampedModel):
         return query_sum(self.repo_set, 'open_pr')
 
 
-
 class Repo(TimeStampedModel):
     name = models.CharField(max_length=200)
     slug = AutoSlugField(populate_from='name', slugify=slugify_with_dots)
@@ -277,8 +279,13 @@ class Repo(TimeStampedModel):
 
     def api_req(self, url='', name=None, page=1):
         logger.debug(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}')
-        return requests.get(self.api_url() + url, {'page': page}, verify=self.forge.verify,
-                            headers=self.forge.headers())
+        try:
+            return requests.get(self.api_url() + url, {'page': page}, verify=self.forge.verify,
+                                headers=self.forge.headers())
+        except requests.exceptions.ConnectionError:
+            logger.error(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page} - SECOND TRY')
+            return requests.get(self.api_url() + url, {'page': page}, verify=self.forge.verify,
+                                headers=self.forge.headers())
 
     def api_data(self, url=''):
         req = self.api_req(url)

rainboard/utils.py

+import logging
 import re
 import unicodedata
 from enum import IntEnum
 
 from django.utils.safestring import mark_safe
 
+import git
+
+logger = logging.getLogger('rainboard.utils')
+
 SOURCES = IntEnum('Sources', 'github gitlab redmine robotpkg travis')
 TARGETS = IntEnum('Targets', '14.04 16.04 17.10 18.04 dubnium')
 
@@ -36,6 +41,20 @@ def domain(url):
         url = url.split('//')[1]
     return url.split('/')[0]
 
+
 def domain_link(url):
     dn = domain(url)
     return mark_safe(f'<a href="{url}">{dn}</a>')
+
+
+def update_robotpkg(path):
+    try:
+        git.Repo(str(path / '.git')).remotes.origin.pull()
+    except git.exc.GitCommandError:
+        logger.error('Network error, retrying…')
+        git.Repo(str(path / '.git')).remotes.origin.pull()
+    try:
+        git.Repo(str(path / 'wip' / '.git')).remotes.origin.pull()
+    except git.exc.GitCommandError:
+        logger.error('Network error, retrying…')
+        git.Repo(str(path / 'wip' / '.git')).remotes.origin.pull()