Commit 40de2793 authored by Guilhem Saurel's avatar Guilhem Saurel
Browse files

clean

parent 29d46582
......@@ -15,8 +15,7 @@ class Command(BaseCommand):
for project in Project.objects.all():
logger.info(f' fetching repos for {project}')
for repo in project.repo_set.all():
logger.info(f' fetching {repo.forge} - {repo.namespace}')
repo.git().fetch()
repo.fetch()
logger.info(f' fetching devel & master for {project}')
git = project.git()
remote = git.remote(project.main_repo().git_remote())
......@@ -24,8 +23,11 @@ class Command(BaseCommand):
try:
git.create_head('devel', remote.refs.devel).set_tracking_branch(remote.refs.devel).checkout()
except AttributeError:
pass
logger.warning(f'Project {project} has no devel branch')
if 'master' not in git.heads:
git.create_head('master', remote.refs.master).set_tracking_branch(remote.refs.master).checkout()
try:
git.create_head('master', remote.refs.master).set_tracking_branch(remote.refs.master).checkout()
except AttributeError:
logger.warning(f'Project {project} has no master branch')
logger.info(f' updating branches for {project}')
project.update_branches(main=False)
......@@ -22,8 +22,8 @@ def forges(apps, schema_editor):
def groups(apps, schema_editor):
Namespace = apps.get_model('rainboard', 'Namespace')
Namespace.objects.create(name='Humanoid Path Planner', group=True)
Namespace.objects.create(name='Stack Of Tasks', group=True)
Namespace.objects.create(name='Humanoid Path Planner', group=True)
Namespace.objects.create(name='Gepetto', group=True)
Namespace.objects.create(name='Pyrène Dev', group=True)
......
......@@ -58,7 +58,7 @@ class Forge(Links, NamedModel):
return self.url
def api_req(self, url='', name=None, page=1):
logger.info(f'requesting api {self} {url}, page {page}')
logger.debug(f'requesting api {self} {url}, page {page}')
return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
def api_data(self, url=''):
......@@ -95,94 +95,44 @@ class Forge(Links, NamedModel):
SOURCES.travis: 'https://api.travis-ci.org',
}[self.source]
def get_projects(self):
return getattr(self, f'get_projects_{self.get_source_display()}')()
def get_namespaces_github(self):
for namespace in Namespace.objects.filter(group=True):
for data in self.api_list(f'/orgs/{namespace.slug}/members'):
Namespace.objects.get_or_create(slug=data['login'],
defaults={'name': data['login'], 'group': False})
def get_projects_github(self):
def update_github(namespace, data):
logger.info(f'update {data["name"]}')
project, _ = Project.objects.get_or_create(name=data['name'], defaults={
'homepage': data['homepage'], 'main_namespace': namespace, 'main_forge': self})
repo, _ = Repo.objects.get_or_create(forge=self, namespace=namespace, project=project,
defaults={'repo_id': data['id'], 'name': data['name']})
repo.homepage = data['homepage']
repo.url = data['html_url']
repo.repo_id = data['id']
repo.default_branch = data['default_branch']
repo.open_issues = data['open_issues']
repo_data = repo.api_data()
if repo_data and 'license' in repo_data and repo_data['license']:
if 'spdx_id' in repo_data['license'] and repo_data['license']['spdx_id']:
license = License.objects.get(spdx_id=repo_data['license']['spdx_id'])
repo.license = license
if not project.license:
project.license = license
repo.open_pr = len(list(repo.api_list('/pulls')))
repo.save()
project.save()
def get_namespaces_gitlab(self):
for data in self.api_list('/namespaces'):
Namespace.objects.get_or_create(slug=data['path'],
defaults={'name': data['name'], 'group': data['kind'] == 'group'})
for data in self.api_list('/users'):
Namespace.objects.get_or_create(slug=data['username'], defaults={'name': data['name']})
def get_projects(self):
getattr(self, f'get_namespaces_{self.get_source_display()}')()
return getattr(self, f'get_projects_{self.get_source_display()}')()
self.get_namespaces_github()
def get_projects_github(self):
for org in Namespace.objects.filter(group=True):
for data in self.api_list(f'/orgs/{org.slug}/repos'):
update_github(org, data)
update_github(self, org, data)
for user in Namespace.objects.filter(group=False):
for data in self.api_list(f'/users/{user.slug}/repos'):
if not Project.objects.filter(name=data['name']).exists():
continue
update_github(user, data)
def get_namespaces_gitlab(self):
for data in self.api_list('/namespaces'):
Namespace.objects.get_or_create(slug=data['path'],
defaults={'name': data['name'], 'group': data['kind'] == 'group'})
if Project.objects.filter(name=data['name']).exists():
update_github(self, user, data)
def get_projects_gitlab(self):
def update_gitlab(data):
logger.info(f'update {data["name"]}')
project, created = Project.objects.get_or_create(name=data['name'], defaults={'main_forge': self})
namespace, _ = Namespace.objects.get_or_create(name=data['namespace']['name'])
repo, _ = Repo.objects.get_or_create(forge=self, namespace=namespace, project=project,
defaults={'repo_id': data['id'], 'name': data['name'],
'url': data['web_url']})
# TODO license, open_pr
if 'forked_from_project' in data:
repo.forked_from = data['forked_from_project']['id']
repo.save()
elif created or project.main_namespace is None:
project.main_namespace = namespace
project.save()
self.get_namespaces_gitlab()
for data in self.api_list('/projects'):
update_gitlab(data)
update_gitlab(self, data)
for orphan in Project.objects.filter(main_namespace=None):
repo = orphan.repo_set.filter(forge__source=SOURCES.gitlab).first()
update_gitlab(self.api_data(f'/projects/{repo.forked_from}'))
update_gitlab(self, self.api_data(f'/projects/{repo.forked_from}'))
def get_projects_redmine(self):
pass # TODO
def get_default_forge(project):
for forge in Forge.objects.order_by('source'):
if project.repo_set.filter(forge=forge).exists():
logger.info(f'default forge for {project} set to {forge}')
project.main_forge = forge
project.save()
return forge
else:
logger.error(f'NO DEFAULT FORGE for {project}')
class Project(Links, NamedModel, TimeStampedModel):
private = models.BooleanField(default=False)
main_namespace = models.ForeignKey(Namespace, on_delete=models.SET_NULL, null=True, blank=True)
......@@ -240,7 +190,12 @@ class Project(Links, NamedModel, TimeStampedModel):
instance.update()
def main_branch(self):
return 'devel' if 'devel' in self.git().heads else 'master'
heads = self.git().heads
if heads:
for branch in MAIN_BRANCHES:
if branch in heads:
return branch
return heads[0]
def cmake(self):
filename = self.git_path() / 'CMakeLists.txt'
......@@ -311,7 +266,7 @@ class Repo(TimeStampedModel):
}[self.forge.source]
def api_req(self, url='', name=None, page=1):
logger.info(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}')
logger.debug(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}')
return requests.get(self.api_url() + url, {'page': page}, verify=self.forge.verify,
headers=self.forge.headers())
......@@ -380,14 +335,25 @@ class Repo(TimeStampedModel):
logger.info(f'Creating remote {remote}')
return git_repo.create_remote(remote, self.get_clone_url())
def fetch(self):
git = self.git()
logger.info(f'fetching {self.forge} - {self.namespace}')
try:
git.fetch()
except git.exc.GitCommandError:
logger.warning(f'fetching {self.forge} - {self.namespace} - SECOND TRY')
git.fetch()
def main_branch(self):
return self.project.branch_set.get(name=f'{self.git_remote()}/{self.default_branch}')
def ahead(self):
return self.main_branch().ahead
main_branch = self.main_branch()
return main_branch.ahead if main_branch is not None else 0
def behind(self):
return self.main_branch().behind
main_branch = self.main_branch()
return main_branch.behind if main_branch is not None else 0
def get_builds(self):
return getattr(self, f'get_builds_{self.forge.get_source_display()}')()
......@@ -435,7 +401,7 @@ class Repo(TimeStampedModel):
def update(self, pull=True):
self.project.update_tags()
if pull:
self.git().fetch()
self.fetch()
self.api_update()
self.get_builds()
......@@ -479,12 +445,13 @@ class Branch(TimeStampedModel):
def update(self, pull=True):
if pull:
self.project.main_repo().git().fetch()
self.project.main_repo().fetch()
if self.name not in MAIN_BRANCHES:
self.repo.git().fetch()
self.repo.fetch()
main_branch = self.project.main_branch()
self.ahead = self.get_ahead(main_branch)
self.behind = self.get_behind(main_branch)
if main_branch is not None:
self.ahead = self.get_ahead(main_branch)
self.behind = self.get_behind(main_branch)
self.updated = self.git().commit.authored_datetime
self.save()
......@@ -599,3 +566,54 @@ class Tag(models.Model):
# class Dockerfile(NamedModel, TimeStampedModel):
# project = models.ForeignKey(Project, on_delete=models.CASCADE)
# target = models.PositiveSmallIntegerField(choices=enum_to_choices(TARGETS))
def get_default_forge(project):
for forge in Forge.objects.order_by('source'):
if project.repo_set.filter(forge=forge).exists():
logger.info(f'default forge for {project} set to {forge}')
project.main_forge = forge
project.save()
return forge
else:
logger.error(f'NO DEFAULT FORGE for {project}')
def update_gitlab(forge, data):
logger.info(f'update {data["name"]} from {forge}')
project, created = Project.objects.get_or_create(name=data['name'], defaults={'main_forge': forge})
namespace, _ = Namespace.objects.get_or_create(name=data['namespace']['name'])
repo, _ = Repo.objects.get_or_create(forge=forge, namespace=namespace, project=project,
defaults={'repo_id': data['id'], 'name': data['name'],
'url': data['web_url']})
# TODO license, open_pr
if 'forked_from_project' in data:
repo.forked_from = data['forked_from_project']['id']
repo.save()
elif created or project.main_namespace is None:
project.main_namespace = namespace
project.save()
def update_github(forge, namespace, data):
logger.info(f'update {data["name"]} from {forge}')
project, _ = Project.objects.get_or_create(name=data['name'], defaults={
'homepage': data['homepage'], 'main_namespace': namespace, 'main_forge': forge})
repo, _ = Repo.objects.get_or_create(forge=forge, namespace=namespace, project=project,
defaults={'repo_id': data['id'], 'name': data['name']})
repo.homepage = data['homepage']
repo.url = data['html_url']
repo.repo_id = data['id']
repo.default_branch = data['default_branch']
repo.open_issues = data['open_issues']
repo_data = repo.api_data()
if repo_data and 'license' in repo_data and repo_data['license']:
if 'spdx_id' in repo_data['license'] and repo_data['license']['spdx_id']:
license = License.objects.get(spdx_id=repo_data['license']['spdx_id'])
repo.license = license
if not project.license:
project.license = license
repo.open_pr = len(list(repo.api_list('/pulls')))
repo.save()
project.save()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment