fix tests

.gitlab-ci.yml

@@ -22,5 +22,5 @@ build:
   script:
     - docker build -t ${DOCKER_TAG} .
     - docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e GITHUB_PIPELINE_TOKEN -e GITLAB_PIPELINE_TOKEN -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py migrate
-    - docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e GITHUB_PIPELINE_TOKEN -e GITLAB_PIPELINE_TOKEN -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py test
+    - docker run --rm -e SECRET_KEY -e GITHUB_WEBHOOK_KEY -e GITLAB_WEBHOOK_KEY -e GITHUB_PIPELINE_TOKEN -e GITLAB_PIPELINE_TOKEN -e PYTHONWARNINGS ${DOCKER_TAG} ./manage.py test --parallel 1
     - docker push ${DOCKER_TAG}

dashboard/middleware.py

+import asyncio
 from ipaddress import ip_address, ip_network
 
 from django.conf import settings
 from django.http import HttpRequest, HttpResponse, HttpResponseRedirect
 from django.shortcuts import reverse
+from django.utils.decorators import sync_and_async_middleware
 
 from rest_framework import permissions
 
@@ -15,19 +17,30 @@ def ip_laas(request: HttpRequest) -> bool:
     return any(forwarded_for in ip_network(net) for net in settings.LAAS_NETWORKS)
 
 
-class LAASPermsMiddleware:
-    def __init__(self, get_response):
-        self.get_response = get_response
+def allowed(request: HttpRequest) -> bool:
+    """Allow access to pages protected at a higher application level,
+    or if the user is authenticated,
+    or if the request comes from a trusted IP.
+    """
+    return (any(request.path.startswith(f'/{url}/') for url in ALLOWED_URLS)
+            or request.user and request.user.is_authenticated
+            or request.method in permissions.SAFE_METHODS and ip_laas(request))
 
-    def __call__(self, request: HttpRequest) -> HttpResponse:
-        """Allow access to pages protected at a higher application level,
-        or if the user is authenticated,
-        or if the request comes from a trusted IP.
-        """
-        allowed = (any(request.path.startswith(f'/{url}/') for url in ALLOWED_URLS)
-                   or request.user and request.user.is_authenticated
-                   or request.method in permissions.SAFE_METHODS and ip_laas(request))
 
-        if allowed:
-            return self.get_response(request)
-        return HttpResponseRedirect(reverse('login'))
+@sync_and_async_middleware
+def laas_perms_middleware(get_response):
+    # One-time configuration and initialization goes here.
+    if asyncio.iscoroutinefunction(get_response):
+
+        async def middleware(request) -> HttpResponse:
+            if allowed(request):
+                return await get_response(request)
+            return HttpResponseRedirect(reverse('login'))
+    else:
+
+        def middleware(request) -> HttpResponse:
+            if allowed(request):
+                return get_response(request)
+            return HttpResponseRedirect(reverse('login'))
+
+    return middleware

dashboard/settings.py

@@ -58,7 +58,7 @@ MIDDLEWARE = [
     'django.contrib.auth.middleware.AuthenticationMiddleware',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
-    f'{PROJECT}.middleware.LAASPermsMiddleware',
+    f'{PROJECT}.middleware.laas_perms_middleware',
 ]
 
 ROOT_URLCONF = f'{PROJECT}.urls'

gh/views.py

@@ -7,6 +7,7 @@ from hashlib import sha1
 from ipaddress import ip_address, ip_network
 from json import loads
 
+from asgiref.sync import sync_to_async, async_to_sync
 from django.conf import settings
 from django.core.mail import mail_admins
 from django.http import HttpRequest
@@ -27,8 +28,11 @@ from . import models
 
 logger = logging.getLogger(__name__)
 
+PR_MASTER_MSG = """Hi ! This project doesn't usually accept pull requests on master. If this wasn't intentionnal, you
+can change the base branch of this pull request to devel (No need to close it for that). Best, a bot."""
 
-def check_suite(request: HttpRequest, rep: str) -> HttpResponse:
+
+async def check_suite(request: HttpRequest, rep: str) -> HttpResponse:
     """Manage Github's check suites."""
     data = loads(request.body.decode())
     slug = slugify(data['repository']['name'])
@@ -36,39 +40,43 @@ def check_suite(request: HttpRequest, rep: str) -> HttpResponse:
     if 'ros-release' in slug:  # Don't run check suites on ros-release repositories
         return HttpResponse(rep)
 
-    models.GithubCheckSuite.objects.get_or_create(id=data['check_suite']['id'])
+    await sync_to_async(models.GithubCheckSuite.objects.get_or_create)(id=data['check_suite']['id'])
     return HttpResponse(rep)
 
 
-def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
+async def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
     """Manage Github's Pull Requests."""
+    logger.info('process gh pr')
     data = loads(request.body.decode())
     event = data['action']
     branch = f'pr/{data["number"]}'
     login = slugify(data["pull_request"]["head"]["repo"]["owner"]["login"])
 
-    namespace = get_object_or_404(Namespace, slug_github=slugify(data['repository']['owner']['login']))
-    project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
-    git_repo = project.git()
+    namespace = await sync_to_async(get_object_or_404)(Namespace,
+                                                       slug_github=slugify(data['repository']['owner']['login']))
+    project = await sync_to_async(get_object_or_404)(Project,
+                                                     main_namespace=namespace,
+                                                     slug=slugify(data['repository']['name']))
+    git_repo = await sync_to_async(project.git)()
     logger.debug(f'{namespace.slug}/{project.slug}: Pull request on {branch}: {event}')
 
     # Prevent pull requests on master when necessary
     if event in ['opened', 'reopened']:
-        gh = project.github()
-        pr = gh.get_pull(data["number"])
+        gh = await sync_to_async(project.github)()
+        pr = await sync_to_async(gh.get_pull)(data["number"])
         pr_branch = pr.base.ref
-        if not project.accept_pr_to_master and pr_branch == 'master' \
-                and 'devel' in [b.name for b in gh.get_branches()] and login != namespace.slug_github:
+        if not project.accept_pr_to_master and pr_branch == 'master' and 'devel' in [
+                b.name for b in await sync_to_async(gh.get_branches)()
+        ] and login != namespace.slug_github:
             logger.info(f"{namespace.slug}/{project.slug}: New pr {data['number']} to master")
-            pr.create_issue_comment("Hi ! This project doesn't usually accept pull requests on master. If this wasn't "
-                                    "intentionnal, you can change the base branch of this pull request to devel "
-                                    "(No need to close it for that). Best, a bot.")
+            await sync_to_async(pr.create_issue_comment)(PR_MASTER_MSG)
 
     gh_remote_name = f'github/{login}'
     if gh_remote_name not in git_repo.remotes:
-        remote = git_repo.create_remote(gh_remote_name, data["pull_request"]["head"]["repo"]["clone_url"])
+        remote = await sync_to_async(git_repo.create_remote)(gh_remote_name,
+                                                             data["pull_request"]["head"]["repo"]["clone_url"])
     else:
-        remote = git_repo.remote(gh_remote_name)
+        remote = await sync_to_async(git_repo.remote)(gh_remote_name)
 
     # Sync the pull request with the pr/XX branch on Gitlab
     if event in ['opened', 'reopened', 'synchronize']:
@@ -79,12 +87,13 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
         if branch in git_repo.branches:
             git_repo.heads[branch].commit = commit
         else:
-            git_repo.create_head(branch, commit=commit)
+            await sync_to_async(git_repo.create_head)(branch, commit=commit)
 
         # Create a gitlab remote if it doesn't exist
         gl_remote_name = f'gitlab/{namespace.slug}'
         if gl_remote_name not in git_repo.remotes:
-            git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
+            url = await sync_to_async(project.remote_url_gitlab)()
+            await sync_to_async(git_repo.create_remote)(gl_remote_name, url=url)
 
         # Push the changes to gitlab
         logger.info(f'{namespace.slug}/{project.slug}: Pushing {commit} on {branch} on gitlab')
@@ -99,14 +108,16 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
         if branch in git_repo.branches:
             git_repo.delete_head(branch, force=True)
             git_repo.delete_remote(gh_remote_name)
-        project.gitlab().branches.delete(branch)
+        gitlab = await sync_to_async(project.gitlab)()
+        await sync_to_async(gitlab.branches.delete)(branch)
         logger.info(f'{namespace.slug}/{project.slug}: Deleted branch {branch}')
 
     return HttpResponse(rep)
 
 
-def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
+async def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
     """Someone pushed on github or gitlab. Synchronise local & remote repos."""
+    logger.debug('start gh.views.push')
     data = loads(request.body.decode())
     slug = slugify(data['repository']['name'])
 
@@ -114,18 +125,20 @@ def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
         return HttpResponse(rep)
 
     if source == SOURCES.gitlab:
-        namespace = get_object_or_404(Namespace,
-                                      slug_gitlab=slugify(data['project']['path_with_namespace'].split('/')[0]))
+        namespace = await sync_to_async(get_object_or_404)(Namespace,
+                                                           slug_gitlab=slugify(
+                                                               data['project']['path_with_namespace'].split('/')[0]))
     else:
-        namespace = get_object_or_404(Namespace, slug_github=slugify(data['repository']['owner']['login']))
+        namespace = await sync_to_async(get_object_or_404)(Namespace,
+                                                           slug_github=slugify(data['repository']['owner']['login']))
 
-    project = get_object_or_404(Project, main_namespace=namespace, slug=slug)
+    project = await sync_to_async(get_object_or_404)(Project, main_namespace=namespace, slug=slug)
 
     branch = data['ref'][11:]  # strip 'refs/heads/'
     commit = data['after']
     gl_remote_name = f'gitlab/{namespace.slug}'
     gh_remote_name = f'github/{namespace.slug}'
-    git_repo = project.git()
+    git_repo = await sync_to_async(project.git)()
     logger.debug(f'{namespace.slug}/{slug}: Push detected on {source.name} {branch} (commit {commit})')
 
     if branch.startswith('pr/'):  # Don't sync pr/XX branches here, they are already handled by pull_request()
@@ -136,16 +149,18 @@ def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
 
     # Fetch the latest commit from gitlab
     if gl_remote_name in git_repo.remotes:
-        gl_remote = git_repo.remote(gl_remote_name)
+        gl_remote = await sync_to_async(git_repo.remote)(gl_remote_name)
     else:
-        gl_remote = git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
+        url = await sync_to_async(project.remote_url_gitlab)()
+        gl_remote = await sync_to_async(git_repo.create_remote)(gl_remote_name, url=url)
     gl_remote.fetch()
 
     # Fetch the latest commit from github
     if gh_remote_name in git_repo.remotes:
-        gh_remote = git_repo.remote(gh_remote_name)
+        gh_remote = await sync_to_async(git_repo.remote)(gh_remote_name)
     else:
-        gh_remote = git_repo.create_remote(gh_remote_name, url=project.remote_url_github())
+        url = await sync_to_async(project.remote_url_github)()
+        gh_remote = await sync_to_async(git_repo.create_remote)(gh_remote_name, url=url)
     gh_remote.fetch()
 
     # The branch was deleted on one remote, delete the branch on the other remote as well
@@ -153,9 +168,11 @@ def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
         if branch in git_repo.branches:
             git_repo.delete_head(branch, force=True)
             if source == SOURCES.gitlab:
-                project.github().get_git_ref(f'heads/{branch}').delete()
+                github = await sync_to_async(project.github)()
+                github.get_git_ref(f'heads/{branch}').delete()
             else:
-                project.gitlab().branches.delete(branch)
+                gitlab = await sync_to_async(project.gitlab)()
+                gitlab.branches.delete(branch)
             logger.info(f'{namespace.slug}/{slug}: Deleted branch {branch}')
         return HttpResponse(rep)
 
@@ -170,16 +187,16 @@ def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
     if branch in git_repo.branches:
         git_repo.heads[branch].commit = commit
     else:
-        git_repo.create_head(branch, commit=commit)
+        await sync_to_async(git_repo.create_head)(branch, commit=commit)
 
     # Push the changes to other remote
     try:
         if source == SOURCES.gitlab and (branch not in gh_remote.refs or str(gh_remote.refs[branch].commit) != commit):
             logger.info(f'{namespace.slug}/{slug}: Pushing {commit} on {branch} on github')
-            git_repo.git.push(gh_remote_name, branch)
+            await sync_to_async(git_repo.git.push)(gh_remote_name, branch)
         elif branch not in gl_remote.refs or str(gl_remote.refs[branch].commit) != commit:
             logger.info(f'{namespace.slug}/{slug}: Pushing {commit} on {branch} on gitlab')
-            git_repo.git.push(gl_remote_name, branch)
+            await sync_to_async(git_repo.git.push)(gl_remote_name, branch)
         else:
             return HttpResponse('already synced')
     except git.exc.GitCommandError:
@@ -187,19 +204,24 @@ def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
         logger.exception(f'{namespace.slug}/{slug}: Forge sync failed')
         message = traceback.format_exc()
         message = re.sub(r'://.*@', '://[REDACTED]@', message)  # Hide access tokens in the mail
-        mail_admins(f'Forge sync failed for {namespace.slug}/{slug}', message)
+        await sync_to_async(mail_admins)(f'Forge sync failed for {namespace.slug}/{slug}', message)
 
     return HttpResponse(rep)
 
 
-def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
+async def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
     """Something happened on a Gitlab pipeline. Tell Github if necessary."""
+    logger.debug('start gh.views.pipeline')
     data = loads(request.body.decode())
     branch, commit, gl_status, pipeline_id = (data['object_attributes'][key] for key in ['ref', 'sha', 'status', 'id'])
-    namespace = get_object_or_404(Namespace, slug_gitlab=slugify(data['project']['path_with_namespace'].split('/')[0]))
-    project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['project']['name']))
-    gh_repo = project.github()
-    ci_web_url = project.url_gitlab() + '/pipelines/' + str(pipeline_id)
+    namespace = await sync_to_async(get_object_or_404)(Namespace,
+                                                       slug_gitlab=slugify(
+                                                           data['project']['path_with_namespace'].split('/')[0]))
+    project = await sync_to_async(get_object_or_404)(Project,
+                                                     main_namespace=namespace,
+                                                     slug=slugify(data['project']['name']))
+    gh_repo = await sync_to_async(project.github)()
+    ci_web_url = f'{project.url_gitlab()}/pipelines/{pipeline_id}'
     logger.debug(f'{namespace.slug}/{project.slug}: Pipeline #{pipeline_id} on commit {commit} for branch {branch}, '
                  f'status: {gl_status}')
 
@@ -207,11 +229,14 @@ def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
     if gl_status in ['pending', 'success', 'failed']:
         gh_status = gl_status if gl_status != 'failed' else 'failure'
         if branch.startswith('pr/'):
-            gh_repo.get_commit(sha=commit).create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
+            sha = await sync_to_async(gh_repo.get_commit)(sha=commit)
+            await sync_to_async(sha.create_status)(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
         else:
             try:
-                gh_commit = gh_repo.get_branch(branch).commit
-                gh_commit.create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
+                sha = await sync_to_async(gh_repo.get_branch)(branch)
+                await sync_to_async(sha.commit.create_status)(state=gh_status,
+                                                              target_url=ci_web_url,
+                                                              context='gitlab-ci')
             except github.GithubException as e:
                 if e.status == 404:
                     # Happens when a new branch is created on gitlab and the pipeline event comes before the push event
@@ -222,13 +247,16 @@ def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
     return HttpResponse(rep)
 
 
+@sync_to_async
 @csrf_exempt
-def webhook(request: HttpRequest) -> HttpResponse:
+@async_to_sync
+async def webhook(request: HttpRequest) -> HttpResponse:
     """
     Process request incoming from a github webhook.
 
     thx https://simpleisbetterthancomplex.com/tutorial/2016/10/31/how-to-handle-github-webhooks-using-django.html
     """
+    logger.info('GH webhook')
     # validate ip source
     forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR').split(', ')[0]
     # networks = httpx.get('https://api.github.com/meta').json()['hooks'] # Fails if API rate limit exceeded
@@ -257,17 +285,19 @@ def webhook(request: HttpRequest) -> HttpResponse:
     if event == 'ping':
         return HttpResponse('pong')
     if event == 'push':
-        return push(request, SOURCES.github, 'push event detected')
+        return await push(request, SOURCES.github, 'push event detected')
     if event == 'check_suite':
-        return check_suite(request, 'check_suite event detected')
+        return await check_suite(request, 'check_suite event detected')
     if event == 'pull_request':
-        return pull_request(request, 'pull_request event detected')
+        return await pull_request(request, 'pull_request event detected')
 
     return HttpResponseForbidden('event not found')
 
 
+@sync_to_async
 @csrf_exempt
-def gl_webhook(request: HttpRequest) -> HttpResponse:
+@async_to_sync
+async def gl_webhook(request: HttpRequest) -> HttpResponse:
     """Process request incoming from a gitlab webhook."""
 
     # validate ip source
@@ -288,8 +318,8 @@ def gl_webhook(request: HttpRequest) -> HttpResponse:
     if event == 'ping':
         return HttpResponse('pong')
     elif event == 'Pipeline Hook':
-        return pipeline(request, 'pipeline event detected')
+        return await pipeline(request, 'pipeline event detected')
     elif event == 'Push Hook':
-        return push(request, SOURCES.gitlab, 'push event detected')
+        return await push(request, SOURCES.gitlab, 'push event detected')
 
     return HttpResponseForbidden('event not found')