git

dashboard/settings.py

@@ -11,6 +11,7 @@ https://docs.djangoproject.com/en/2.0/ref/settings/
 """
 
 import os
+from pathlib import Path
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -153,3 +154,7 @@ LOGGING = {
         },
     },
 }
+
+RAINBOARD_DATA = Path('/srv/dashboard')
+RAINBOARD_GITS = RAINBOARD_DATA / 'repositories'
+RAINBOARD_RPKG = RAINBOARD_DATA / 'robotpkg'

rainboard/management/commands/fetch.py

+import logging
+
+from django.core.management.base import BaseCommand
+
+from rainboard.models import Repo
+
+logger = logging.getLogger('rainboard.management.fetch')
+
+class Command(BaseCommand):
+    help = 'Fetch all remotes'
+
+    def handle(self, *args, **options):
+        logger.info(f'Fetching all repos')
+        for repo in Repo.objects.all():
+            logger.info(f' fetching {repo}')
+            repo.git().fetch()

rainboard/models.py

@@ -8,6 +8,7 @@ import requests
 from autoslug import AutoSlugField
 from ndh.models import Links, NamedModel, TimeStampedModel
 from ndh.utils import enum_to_choices
+import git
 
 from .utils import SOURCES, TARGETS
 
@@ -44,6 +45,13 @@ class Project(Links, NamedModel, TimeStampedModel):
     def get_absolute_url(self):
         return reverse('rainboard:project', kwargs={'slug': self.slug})
 
+    def git(self):
+        path = settings.RAINBOARD_GITS / self.main_namespace.slug / self.slug
+        if not path.exists():
+            logger.info(f'Creating repo for {self.main_namespace.slug}/{self.slug}')
+            return git.Repo.init(path)
+        return git.Repo(str(path / '.git'))
+
 
 class Forge(Links, NamedModel):
     source = models.PositiveSmallIntegerField(choices=enum_to_choices(SOURCES))
@@ -147,7 +155,7 @@ class Forge(Links, NamedModel):
             update_gitlab(data)
 
         for orphan in Project.objects.filter(main_namespace=None):
-            repo = orphan.repo_set.get(forge__source=SOURCES.gitlab)
+            repo = orphan.repo_set.filter(forge__source=SOURCES.gitlab).first()
             update_gitlab(self.api_data(f'/projects/{repo.forked_from}'))
 
     def get_projects_redmine(self):
@@ -225,6 +233,20 @@ class Repo(TimeStampedModel):
         self.clone_url = data['clone_url']
         self.save()
 
+    def get_clone_url(self):
+        if self.forge.source == SOURCES.gitlab:
+            return self.clone_url.replace('://', f'://gitlab-ci-token:{self.forge.token}@')
+        return self.clone_url
+
+    def git(self):
+        git = self.project.git()
+        remote = f'{self.forge.slug}/{self.namespace.slug}'
+        try:
+            return git.remote(remote)
+        except ValueError:
+            logger.info(f'Creating remote {self.forge.slug}/{self.namespace.slug}/{self.project.slug}')
+            return git.create_remote(remote, self.get_clone_url())
+
 
 class Commit(NamedModel, TimeStampedModel):
     project = models.ForeignKey(Project, on_delete=models.CASCADE)

requirements.in

@@ -2,6 +2,7 @@ Django
 django-bootstrap3
 django-filter
 django-tables2
+gitpython
 psycopg2
 requests
 https://github.com/nim65s/ndh/archive/master.zip#egg=ndh

requirements.txt

@@ -12,8 +12,11 @@ django-bootstrap3==9.1.0
 django-filter==1.1.0
 django-tables2==1.17.1
 django==2.0.1
+gitdb2==2.0.3             # via gitpython
+gitpython==2.1.8
 idna==2.6                 # via requests
 psycopg2==2.7.3.2
 pytz==2017.3              # via django
 requests==2.18.4
+smmap2==2.0.3             # via gitdb2
 urllib3==1.22             # via requests