Commit a3a9a11e authored by Guilhem Saurel's avatar Guilhem Saurel
Browse files

api lists

parent ca734fd7
...@@ -14,7 +14,7 @@ from autoslug import AutoSlugField ...@@ -14,7 +14,7 @@ from autoslug import AutoSlugField
from ndh.models import Links, NamedModel, TimeStampedModel from ndh.models import Links, NamedModel, TimeStampedModel
from ndh.utils import enum_to_choices from ndh.utils import enum_to_choices
from .utils import SOURCES, TARGETS, slugify_with_dots from .utils import SOURCES, TARGETS, slugify_with_dots, api_next
logger = logging.getLogger('rainboard.models') logger = logging.getLogger('rainboard.models')
...@@ -57,10 +57,21 @@ class Forge(Links, NamedModel): ...@@ -57,10 +57,21 @@ class Forge(Links, NamedModel):
def get_absolute_url(self): def get_absolute_url(self):
return self.url return self.url
def api_data(self, url=''): def api_data(self, url='', name=None, paginate=True):
logger.info(f'requesting api {self}{url}') page = 1
req = requests.get(self.api_url() + url, verify=self.verify, headers=self.headers()) while page:
return req.json() if req.status_code == 200 else [] logger.info(f'requesting api {self} {url}, page {page}')
req = requests.get(self.api_url() + url, {'page': page}, verify=self.verify, headers=self.headers())
if req.status_code != 200:
return [] # TODO
data = req.json()
if name is not None:
data = data[name]
if not paginate:
return data
for item in data:
yield item
page = api_next(self.source, req)
def headers(self): def headers(self):
return { return {
...@@ -101,14 +112,14 @@ class Forge(Links, NamedModel): ...@@ -101,14 +112,14 @@ class Forge(Links, NamedModel):
repo.default_branch = data['default_branch'] repo.default_branch = data['default_branch']
repo.open_issues = data['open_issues'] repo.open_issues = data['open_issues']
repo_data = repo.api_data() repo_data = repo.api_data(paginate=False)
if repo_data and 'license' in repo_data and repo_data['license']: if repo_data and 'license' in repo_data and repo_data['license']:
if 'spdx_id' in repo_data['license'] and repo_data['license']['spdx_id']: if 'spdx_id' in repo_data['license'] and repo_data['license']['spdx_id']:
license = License.objects.get(spdx_id=repo_data['license']['spdx_id']) license = License.objects.get(spdx_id=repo_data['license']['spdx_id'])
repo.license = license repo.license = license
if not project.license: if not project.license:
project.license = license project.license = license
repo.open_pr = len(repo.api_data('/pulls')) repo.open_pr = len(list(repo.api_data('/pulls')))
repo.save() repo.save()
project.save() project.save()
...@@ -148,7 +159,7 @@ class Forge(Links, NamedModel): ...@@ -148,7 +159,7 @@ class Forge(Links, NamedModel):
for orphan in Project.objects.filter(main_namespace=None): for orphan in Project.objects.filter(main_namespace=None):
repo = orphan.repo_set.filter(forge__source=SOURCES.gitlab).first() repo = orphan.repo_set.filter(forge__source=SOURCES.gitlab).first()
update_gitlab(self.api_data(f'/projects/{repo.forked_from}')) update_gitlab(self.api_data(f'/projects/{repo.forked_from}', paginate=False))
def get_projects_redmine(self): def get_projects_redmine(self):
pass # TODO pass # TODO
...@@ -293,13 +304,25 @@ class Repo(TimeStampedModel): ...@@ -293,13 +304,25 @@ class Repo(TimeStampedModel):
SOURCES.gitlab: f'{self.forge.api_url()}/projects/{self.repo_id}', SOURCES.gitlab: f'{self.forge.api_url()}/projects/{self.repo_id}',
}[self.forge.source] }[self.forge.source]
def api_data(self, url=''): def api_data(self, url='', name=None, paginate=True):
logger.info(f'requesting api {self.forge} {self.namespace} {self} {url}') page = 1
req = requests.get(self.api_url() + url, verify=self.forge.verify, headers=self.forge.headers()) while page:
return req.json() if req.status_code == 200 else [] logger.info(f'requesting api {self.forge} {self.namespace} {self} {url}, page {page}')
req = requests.get(self.api_url() + url, {'page': page}, verify=self.forge.verify,
headers=self.forge.headers())
if req.status_code != 200:
return [] # TODO
data = req.json()
if name is not None:
data = data[name]
if not paginate:
return data
for item in data:
yield item
page = api_next(self.forge.source, req)
def api_update(self): def api_update(self):
data = self.api_data() data = self.api_data(paginate=False)
if data: if data:
return getattr(self, f'api_update_{self.forge.get_source_display()}')(data) return getattr(self, f'api_update_{self.forge.get_source_display()}')(data)
...@@ -364,7 +387,7 @@ class Repo(TimeStampedModel): ...@@ -364,7 +387,7 @@ class Repo(TimeStampedModel):
pid, ref = pipeline['id'], pipeline['ref'] pid, ref = pipeline['id'], pipeline['ref']
if self.project.tag_set.filter(name=ref).exists(): if self.project.tag_set.filter(name=ref).exists():
continue continue
data = self.api_data(f'/pipelines/{pid}') data = self.api_data(f'/pipelines/{pid}', paginate=False)
branch_name = f'{self.forge.slug}/{self.namespace.slug}/{ref}' branch_name = f'{self.forge.slug}/{self.namespace.slug}/{ref}'
branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self) branch, created = Branch.objects.get_or_create(name=branch_name, project=self.project, repo=self)
if created: if created:
...@@ -378,7 +401,7 @@ class Repo(TimeStampedModel): ...@@ -378,7 +401,7 @@ class Repo(TimeStampedModel):
def get_builds_github(self): def get_builds_github(self):
if self.travis_id is not None: if self.travis_id is not None:
travis = Forge.objects.get(source=SOURCES.travis) travis = Forge.objects.get(source=SOURCES.travis)
for build in travis.api_data(f'/repo/{self.travis_id}/builds')['builds']: for build in travis.api_data(f'/repo/{self.travis_id}/builds', name='builds'):
if self.project.tag_set.filter(name=build['branch']['name']).exists(): if self.project.tag_set.filter(name=build['branch']['name']).exists():
continue continue
branch_name = f'{self.forge.slug}/{self.namespace.slug}/{build["branch"]["name"]}' branch_name = f'{self.forge.slug}/{self.namespace.slug}/{build["branch"]["name"]}'
......
...@@ -15,3 +15,14 @@ def slugify_with_dots(value): ...@@ -15,3 +15,14 @@ def slugify_with_dots(value):
value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii') value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore').decode('ascii')
value = re.sub(r'[^\w\s\.-]', '', value).strip().lower() value = re.sub(r'[^\w\s\.-]', '', value).strip().lower()
return mark_safe(re.sub(r'[-\s]+', '-', value)) return mark_safe(re.sub(r'[-\s]+', '-', value))
def api_next(source, req):
if source == SOURCES.github:
if 'Link' in req.headers:
for link in req.headers['Link'].split(','):
if 'next' in link:
return int(re.search('page=(\d+)', link).group(1))
if source == SOURCES.gitlab:
if 'X-Next-Page' in req.headers and req.headers['X-Next-Page']:
return int(req.headers['X-Next-Page'])
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment