Commit 29d46582 authored by Guilhem Saurel's avatar Guilhem Saurel
Browse files

api_list

parent b9f5c379
...@@ -57,20 +57,24 @@ class Forge(Links, NamedModel): ...@@ -57,20 +57,24 @@ class Forge(Links, NamedModel):
def get_absolute_url(self): def get_absolute_url(self):
return self.url return self.url
def api_data(self, url='', name=None, paginate=True): def api_req(self, url='', name=None, page=1):
logger.info(f'requesting api {self} {url}, page {page}')
return requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
def api_data(self, url=''):
req = self.api_req(url)
return req.json() if req.status_code == 200 else [] # TODO
def api_list(self, url='', name=None):
page = 1 page = 1
while page: while page:
logger.info(f'requesting api {self} {url}, page {page}') req = self.api_req(url, name, page)
req = requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
if req.status_code != 200: if req.status_code != 200:
return [] # TODO return [] # TODO
data = req.json() data = req.json()
if name is not None: if name is not None:
data = data[name] data = data[name]
if not paginate: yield from data
return data
for item in data:
yield item
page = api_next(self.source, req) page = api_next(self.source, req)
def headers(self): def headers(self):
...@@ -96,12 +100,13 @@ class Forge(Links, NamedModel): ...@@ -96,12 +100,13 @@ class Forge(Links, NamedModel):
def get_namespaces_github(self): def get_namespaces_github(self):
for namespace in Namespace.objects.filter(group=True): for namespace in Namespace.objects.filter(group=True):
for data in self.api_data(f'/orgs/{namespace.slug}/members'): for data in self.api_list(f'/orgs/{namespace.slug}/members'):
Namespace.objects.get_or_create(slug=data['login'], Namespace.objects.get_or_create(slug=data['login'],
defaults={'name': data['login'], 'group': False}) defaults={'name': data['login'], 'group': False})
def get_projects_github(self): def get_projects_github(self):
def update_github(namespace, data): def update_github(namespace, data):
logger.info(f'update {data["name"]}')
project, _ = Project.objects.get_or_create(name=data['name'], defaults={ project, _ = Project.objects.get_or_create(name=data['name'], defaults={
'homepage': data['homepage'], 'main_namespace': namespace, 'main_forge': self}) 'homepage': data['homepage'], 'main_namespace': namespace, 'main_forge': self})
repo, _ = Repo.objects.get_or_create(forge=self, namespace=namespace, project=project, repo, _ = Repo.objects.get_or_create(forge=self, namespace=namespace, project=project,
...@@ -112,39 +117,41 @@ class Forge(Links, NamedModel): ...@@ -112,39 +117,41 @@ class Forge(Links, NamedModel):
repo.default_branch = data['default_branch'] repo.default_branch = data['default_branch']
repo.open_issues = data['open_issues'] repo.open_issues = data['open_issues']
repo_data = repo.api_data(paginate=False) repo_data = repo.api_data()
if repo_data and 'license' in repo_data and repo_data['license']: if repo_data and 'license' in repo_data and repo_data['license']:
if 'spdx_id' in repo_data['license'] and repo_data['license']['spdx_id']: if 'spdx_id' in repo_data['license'] and repo_data['license']['spdx_id']:
license = License.objects.get(spdx_id=repo_data['license']['spdx_id']) license = License.objects.get(spdx_id=repo_data['license']['spdx_id'])
repo.license = license repo.license = license
if not project.license: if not project.license:
project.license = license project.license = license
repo.open_pr = len(list(repo.api_data('/pulls'))) repo.open_pr = len(list(repo.api_list('/pulls')))
repo.save() repo.save()
project.save() project.save()
self.get_namespaces_github() self.get_namespaces_github()
for org in Namespace.objects.filter(group=True): for org in Namespace.objects.filter(group=True):
for data in self.api_data(f'/orgs/{org.slug}/repos'): for data in self.api_list(f'/orgs/{org.slug}/repos'):
update_github(org, data) update_github(org, data)
for user in Namespace.objects.filter(group=False): for user in Namespace.objects.filter(group=False):
for data in self.api_data(f'/users/{user.slug}/repos'): for data in self.api_list(f'/users/{user.slug}/repos'):
if not Project.objects.filter(name=data['name']).exists(): if not Project.objects.filter(name=data['name']).exists():
continue continue
update_github(user, data) update_github(user, data)
def get_namespaces_gitlab(self): def get_namespaces_gitlab(self):
for data in self.api_data('/namespaces'): for data in self.api_list('/namespaces'):
Namespace.objects.get_or_create(slug=data['path'], Namespace.objects.get_or_create(slug=data['path'],
defaults={'name': data['name'], 'group': data['kind'] == 'group'}) defaults={'name': data['name'], 'group': data['kind'] == 'group'})
def get_projects_gitlab(self): def get_projects_gitlab(self):
def update_gitlab(data): def update_gitlab(data):
logger.info(f'update {data["name"]}')
project, created = Project.objects.get_or_create(name=data['name'], defaults={'main_forge': self}) project, created = Project.objects.get_or_create(name=data['name'], defaults={'main_forge': self})
namespace, _ = Namespace.objects.get_or_create(name=data['namespace']['name']) namespace, _ = Namespace.objects.get_or_create(name=data['namespace']['name'])
repo, _ = Repo.objects.get_or_create(forge=self, namespace=namespace, project=project, repo, _ = Repo.objects.get_or_create(forge=self, namespace=namespace, project=project,
defaults={'repo_id': data['id'], 'name': data['name'], defaults={'repo_id': data['id'], 'name': data['name'],
'url': data['web_url']}) 'url': data['web_url']})
# TODO license, open_pr
if 'forked_from_project' in data: if 'forked_from_project' in data:
repo.forked_from = data['forked_from_project']['id'] repo.forked_from = data['forked_from_project']['id']
repo.save() repo.save()
...@@ -154,12 +161,12 @@ class Forge(Links, NamedModel): ...@@ -154,12 +161,12 @@ class Forge(Links, NamedModel):
self.get_namespaces_gitlab() self.get_namespaces_gitlab()
for data in self.api_data('/projects'): for data in self.api_list('/projects'):
update_gitlab(data) update_gitlab(data)
for orphan in Project.objects.filter(main_namespace=None): for orphan in Project.objects.filter(main_namespace=None):
repo = orphan.repo_set.filter(forge__source=SOURCES.gitlab).first() repo = orphan.repo_set.filter(forge__source=SOURCES.gitlab).first()
update_gitlab(self.api_data(f'/projects/{repo.forked_from}', paginate=False)) update_gitlab(self.api_data(f'/projects/{repo.forked_from}'))
def get_projects_redmine(self): def get_projects_redmine(self):
pass # TODO pass # TODO
...@@ -192,7 +199,6 @@ class Project(Links, NamedModel, TimeStampedModel): ...@@ -192,7 +199,6 @@ class Project(Links, NamedModel, TimeStampedModel):
return reverse('rainboard:project', kwargs={'slug': self.slug}) return reverse('rainboard:project', kwargs={'slug': self.slug})
def git_path(self): def git_path(self):
logger.info(f'git_path for {self} in {self.main_namespace}')
return settings.RAINBOARD_GITS / self.main_namespace.slug / self.slug return settings.RAINBOARD_GITS / self.main_namespace.slug / self.slug
def git(self): def git(self):
...@@ -223,7 +229,7 @@ class Project(Links, NamedModel, TimeStampedModel): ...@@ -223,7 +229,7 @@ class Project(Links, NamedModel, TimeStampedModel):
if branch.startswith('remotes/'): if branch.startswith('remotes/'):
branch = branch[8:] branch = branch[8:]
forge, namespace, name = branch.split('/', maxsplit=2) forge, namespace, name = branch.split('/', maxsplit=2)
namespace, _ = Namespace.objects.get_or_creat(slug=namespace) namespace, _ = Namespace.objects.get_or_create(slug=namespace)
repo, created = Repo.objects.get_or_create(forge__slug=forge, namespace=namespace, project=self, repo, created = Repo.objects.get_or_create(forge__slug=forge, namespace=namespace, project=self,
defaults={'name': self.name, 'default_branch': 'master', defaults={'name': self.name, 'default_branch': 'master',
'repo_id': 0}) 'repo_id': 0})
...@@ -304,25 +310,29 @@ class Repo(TimeStampedModel): ...@@ -304,25 +310,29 @@ class Repo(TimeStampedModel):
SOURCES.gitlab: f'{self.forge.api_url()}/projects/{self.repo_id}', SOURCES.gitlab: f'{self.forge.api_url()}/projects/{self.repo_id}',
}[self.forge.source] }[self.forge.source]
def api_data(self, url='', name=None, paginate=True): def api_req(self, url='', name=None, page=1):
logger.info(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}')
return requests.get(self.api_url() + url, {'page': page}, verify=self.forge.verify,
headers=self.forge.headers())
def api_data(self, url=''):
req = self.api_req(url)
return req.json() if req.status_code == 200 else [] # TODO
def api_list(self, url='', name=None):
page = 1 page = 1
while page: while page:
logger.info(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}') req = self.api_req(url, name, page)
req = requests.get(self.api_url() + url, {'page': page}, verify=self.forge.verify,
headers=self.forge.headers())
if req.status_code != 200: if req.status_code != 200:
return [] # TODO return [] # TODO
data = req.json() data = req.json()
if name is not None: if name is not None:
data = data[name] data = data[name]
if not paginate: yield from data
return data
for item in data:
yield item
page = api_next(self.forge.source, req) page = api_next(self.forge.source, req)
def api_update(self): def api_update(self):
data = self.api_data(paginate=False) data = self.api_data()
if data: if data:
return getattr(self, f'api_update_{self.forge.get_source_display()}')(data) return getattr(self, f'api_update_{self.forge.get_source_display()}')(data)
...@@ -383,11 +393,11 @@ class Repo(TimeStampedModel): ...@@ -383,11 +393,11 @@ class Repo(TimeStampedModel):
return getattr(self, f'get_builds_{self.forge.get_source_display()}')() return getattr(self, f'get_builds_{self.forge.get_source_display()}')()
def get_builds_gitlab(self): def get_builds_gitlab(self):
for pipeline in self.api_data('/pipelines'): for pipeline in self.api_list('/pipelines'):
pid, ref = pipeline['id'], pipeline['ref'] pid, ref = pipeline['id'], pipeline['ref']
if self.project.tag_set.filter(name=ref).exists(): if self.project.tag_set.filter(name=ref).exists():
continue continue
data = self.api_data(f'/pipelines/{pid}', paginate=False) data = self.api_data(f'/pipelines/{pid}')
branch_name = f'{self.forge.slug}/{self.namespace.slug}/{ref}' branch_name = f'{self.forge.slug}/{self.namespace.slug}/{ref}'
branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self) branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
if created: if created:
...@@ -401,7 +411,7 @@ class Repo(TimeStampedModel): ...@@ -401,7 +411,7 @@ class Repo(TimeStampedModel):
def get_builds_github(self): def get_builds_github(self):
if self.travis_id is not None: if self.travis_id is not None:
travis = Forge.objects.get(source=SOURCES.travis) travis = Forge.objects.get(source=SOURCES.travis)
for build in travis.api_data(f'/repo/{self.travis_id}/builds', name='builds'): for build in travis.api_list(f'/repo/{self.travis_id}/builds', name='builds'):
if self.project.tag_set.filter(name=build['branch']['name']).exists(): if self.project.tag_set.filter(name=build['branch']['name']).exists():
continue continue
branch_name = f'{self.forge.slug}/{self.namespace.slug}/{build["branch"]["name"]}' branch_name = f'{self.forge.slug}/{self.namespace.slug}/{build["branch"]["name"]}'
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment