views.py 12.6 KB
Newer Older
Guilhem Saurel's avatar
Guilhem Saurel committed
1
2
"""Views for dashboard_apps."""
import hmac
Tom Pillot's avatar
Tom Pillot committed
3
4
5
import logging
import re
import traceback
Guilhem Saurel's avatar
Guilhem Saurel committed
6
7
from hashlib import sha1
from ipaddress import ip_address, ip_network
Guilhem Saurel's avatar
Guilhem Saurel committed
8
from json import loads
Guilhem Saurel's avatar
Guilhem Saurel committed
9
10

from django.conf import settings
Tom Pillot's avatar
Tom Pillot committed
11
from django.core.mail import mail_admins
Guilhem Saurel's avatar
Guilhem Saurel committed
12
from django.http import HttpRequest
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
13
14
15
from django.http.response import (HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseRedirect,
                                  HttpResponseServerError)
from django.shortcuts import get_object_or_404, reverse
Guilhem Saurel's avatar
Guilhem Saurel committed
16
17
18
from django.utils.encoding import force_bytes
from django.views.decorators.csrf import csrf_exempt

19
20
21
import git
import github
from autoslug.utils import slugify
Guilhem Saurel's avatar
Guilhem Saurel committed
22
from dashboard.middleware import ip_laas
Tom Pillot's avatar
Tom Pillot committed
23
24
from rainboard.models import Namespace, Project
from rainboard.utils import SOURCES
25

Guilhem Saurel's avatar
sync    
Guilhem Saurel committed
26
27
from . import models

Tom Pillot's avatar
Tom Pillot committed
28
29
logger = logging.getLogger(__name__)

Guilhem Saurel's avatar
Guilhem Saurel committed
30

Guilhem Saurel's avatar
sync    
Guilhem Saurel committed
31
32
33
def check_suite(request: HttpRequest, rep: str) -> HttpResponse:
    """Manage Github's check suites."""
    data = loads(request.body.decode())
Guilhem Saurel's avatar
Guilhem Saurel committed
34
35
36
37
38
    slug = slugify(data['repository']['name'])

    if 'ros-release' in slug:  # Don't run check suites on ros-release repositories
        return HttpResponse(rep)

Guilhem Saurel's avatar
sync    
Guilhem Saurel committed
39
40
41
42
    models.GithubCheckSuite.objects.get_or_create(id=data['check_suite']['id'])
    return HttpResponse(rep)


Guilhem Saurel's avatar
Guilhem Saurel committed
43
44
def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
    """Manage Github's Pull Requests."""
Guilhem Saurel's avatar
pr    
Guilhem Saurel committed
45
    data = loads(request.body.decode())
Tom Pillot's avatar
Tom Pillot committed
46
47
48
49
50
    event = data['action']
    branch = f'pr/{data["number"]}'
    login = slugify(data["pull_request"]["head"]["repo"]["owner"]["login"])

    namespace = get_object_or_404(Namespace, slug_github=slugify(data['repository']['owner']['login']))
Guilhem Saurel's avatar
pr    
Guilhem Saurel committed
51
52
    project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['repository']['name']))
    git_repo = project.git()
Tom Pillot's avatar
Tom Pillot committed
53
54
55
56
57
58
59
60
61
62
    logger.debug(f'{namespace.slug}/{project.slug}: Pull request on {branch}: {event}')

    # Prevent pull requests on master when necessary
    if event in ['opened', 'reopened']:
        gh = project.github()
        pr = gh.get_pull(data["number"])
        pr_branch = pr.base.ref
        if not project.accept_pr_to_master and pr_branch == 'master' \
                and 'devel' in [b.name for b in gh.get_branches()] and login != namespace.slug_github:
            logger.info(f"{namespace.slug}/{project.slug}: New pr {data['number']} to master")
63
64
65
            pr.create_issue_comment("Hi ! This project doesn't usually accept pull requests on master. If this wasn't "
                                    "intentionnal, you can change the base branch of this pull request to devel "
                                    "(No need to close it for that). Best, a bot.")
Tom Pillot's avatar
Tom Pillot committed
66
67
68
69

    gh_remote_name = f'github/{login}'
    if gh_remote_name not in git_repo.remotes:
        remote = git_repo.create_remote(gh_remote_name, data["pull_request"]["head"]["repo"]["clone_url"])
Guilhem Saurel's avatar
pr    
Guilhem Saurel committed
70
    else:
Tom Pillot's avatar
Tom Pillot committed
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
        remote = git_repo.remote(gh_remote_name)

    # Sync the pull request with the pr/XX branch on Gitlab
    if event in ['opened', 'reopened', 'synchronize']:
        remote.fetch()
        commit = data['pull_request']['head']['sha']

        # Update branch to the latest commit
        if branch in git_repo.branches:
            git_repo.heads[branch].commit = commit
        else:
            git_repo.create_head(branch, commit=commit)

        # Create a gitlab remote if it doesn't exist
        gl_remote_name = f'gitlab/{namespace.slug}'
        if gl_remote_name not in git_repo.remotes:
            git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())

        # Push the changes to gitlab
        logger.info(f'{namespace.slug}/{project.slug}: Pushing {commit} on {branch} on gitlab')
        try:
            git_repo.git.push(gl_remote_name, branch)
        except git.exc.GitCommandError:
            logger.warning(f'{namespace.slug}/{project.slug}: Failed to push on {branch} on gitlab, force pushing ...')
            git_repo.git.push(gl_remote_name, branch, force=True)

    # The pull request was closed, delete the branch pr/XX on Gitlab
    elif event == 'closed':
        if branch in git_repo.branches:
            git_repo.delete_head(branch, force=True)
            git_repo.delete_remote(gh_remote_name)
        project.gitlab().branches.delete(branch)
        logger.info(f'{namespace.slug}/{project.slug}: Deleted branch {branch}')

Guilhem Saurel's avatar
Guilhem Saurel committed
105
106
107
    return HttpResponse(rep)


Tom Pillot's avatar
Tom Pillot committed
108
109
def push(request: HttpRequest, source: SOURCES, rep: str) -> HttpResponse:
    """Someone pushed on github or gitlab. Synchronise local & remote repos."""
Guilhem Saurel's avatar
sync    
Guilhem Saurel committed
110
    data = loads(request.body.decode())
111
112
113
114
    slug = slugify(data['repository']['name'])

    if 'ros-release' in slug:  # Don't sync ros-release repositories
        return HttpResponse(rep)
Tom Pillot's avatar
Tom Pillot committed
115
116
117
118
119
120
121

    if source == SOURCES.gitlab:
        namespace = get_object_or_404(Namespace,
                                      slug_gitlab=slugify(data['project']['path_with_namespace'].split('/')[0]))
    else:
        namespace = get_object_or_404(Namespace, slug_github=slugify(data['repository']['owner']['login']))

122
    project = get_object_or_404(Project, main_namespace=namespace, slug=slug)
Guilhem Saurel's avatar
sync    
Guilhem Saurel committed
123

Tom Pillot's avatar
Tom Pillot committed
124
125
126
127
    branch = data['ref'][11:]  # strip 'refs/heads/'
    commit = data['after']
    gl_remote_name = f'gitlab/{namespace.slug}'
    gh_remote_name = f'github/{namespace.slug}'
Guilhem Saurel's avatar
sync    
Guilhem Saurel committed
128
    git_repo = project.git()
129
    logger.debug(f'{namespace.slug}/{slug}: Push detected on {source.name} {branch} (commit {commit})')
Guilhem Saurel's avatar
Guilhem Saurel committed
130

Tom Pillot's avatar
Tom Pillot committed
131
132
    if branch.startswith('pr/'):  # Don't sync pr/XX branches here, they are already handled by pull_request()
        return HttpResponse(rep)
Guilhem Saurel's avatar
sync    
Guilhem Saurel committed
133

134
135
136
    if branch.startswith('release/'):  # Don't sync release/X.Y.Z branches at all
        return HttpResponse(rep)

Tom Pillot's avatar
Tom Pillot committed
137
138
139
    # Fetch the latest commit from gitlab
    if gl_remote_name in git_repo.remotes:
        gl_remote = git_repo.remote(gl_remote_name)
Guilhem Saurel's avatar
sync    
Guilhem Saurel committed
140
    else:
Tom Pillot's avatar
Tom Pillot committed
141
142
143
144
145
146
        gl_remote = git_repo.create_remote(gl_remote_name, url=project.remote_url_gitlab())
    gl_remote.fetch()

    # Fetch the latest commit from github
    if gh_remote_name in git_repo.remotes:
        gh_remote = git_repo.remote(gh_remote_name)
Guilhem Saurel's avatar
sync    
Guilhem Saurel committed
147
    else:
Tom Pillot's avatar
Tom Pillot committed
148
149
        gh_remote = git_repo.create_remote(gh_remote_name, url=project.remote_url_github())
    gh_remote.fetch()
Guilhem Saurel's avatar
sync    
Guilhem Saurel committed
150

Tom Pillot's avatar
Tom Pillot committed
151
152
153
154
155
156
157
158
    # The branch was deleted on one remote, delete the branch on the other remote as well
    if commit == "0000000000000000000000000000000000000000":
        if branch in git_repo.branches:
            git_repo.delete_head(branch, force=True)
            if source == SOURCES.gitlab:
                project.github().get_git_ref(f'heads/{branch}').delete()
            else:
                project.gitlab().branches.delete(branch)
159
            logger.info(f'{namespace.slug}/{slug}: Deleted branch {branch}')
160
161
        return HttpResponse(rep)

Tom Pillot's avatar
Tom Pillot committed
162
163
164
165
    # Make sure we fetched the latest commit
    ref = gl_remote.refs[branch] if source == SOURCES.gitlab else gh_remote.refs[branch]
    if str(ref.commit) != commit:
        fail = f'Push: wrong commit: {ref.commit} vs {commit}'
166
        logger.error(f'{namespace.slug}/{slug}: ' + fail)
Tom Pillot's avatar
Tom Pillot committed
167
168
169
170
171
        return HttpResponseBadRequest(fail)

    # Update the branch to the latest commit
    if branch in git_repo.branches:
        git_repo.heads[branch].commit = commit
172
    else:
Tom Pillot's avatar
Tom Pillot committed
173
        git_repo.create_head(branch, commit=commit)
174

Tom Pillot's avatar
Tom Pillot committed
175
176
177
    # Push the changes to other remote
    try:
        if source == SOURCES.gitlab and (branch not in gh_remote.refs or str(gh_remote.refs[branch].commit) != commit):
178
            logger.info(f'{namespace.slug}/{slug}: Pushing {commit} on {branch} on github')
Tom Pillot's avatar
Tom Pillot committed
179
180
            git_repo.git.push(gh_remote_name, branch)
        elif branch not in gl_remote.refs or str(gl_remote.refs[branch].commit) != commit:
181
            logger.info(f'{namespace.slug}/{slug}: Pushing {commit} on {branch} on gitlab')
Tom Pillot's avatar
Tom Pillot committed
182
183
184
185
186
            git_repo.git.push(gl_remote_name, branch)
        else:
            return HttpResponse('already synced')
    except git.exc.GitCommandError:
        # Probably failed because of a force push
187
        logger.exception(f'{namespace.slug}/{slug}: Forge sync failed')
Tom Pillot's avatar
Tom Pillot committed
188
189
        message = traceback.format_exc()
        message = re.sub(r'://.*@', '://[REDACTED]@', message)  # Hide access tokens in the mail
190
        mail_admins(f'Forge sync failed for {namespace.slug}/{slug}', message)
Guilhem Saurel's avatar
sync    
Guilhem Saurel committed
191
192
193
194

    return HttpResponse(rep)


Guilhem Saurel's avatar
Guilhem Saurel committed
195
196
def pipeline(request: HttpRequest, rep: str) -> HttpResponse:
    """Something happened on a Gitlab pipeline. Tell Github if necessary."""
Guilhem Saurel's avatar
Guilhem Saurel committed
197
    data = loads(request.body.decode())
Tom Pillot's avatar
Tom Pillot committed
198
199
    branch, commit, gl_status, pipeline_id = (data['object_attributes'][key] for key in ['ref', 'sha', 'status', 'id'])
    namespace = get_object_or_404(Namespace, slug_gitlab=slugify(data['project']['path_with_namespace'].split('/')[0]))
Guilhem Saurel's avatar
Guilhem Saurel committed
200
    project = get_object_or_404(Project, main_namespace=namespace, slug=slugify(data['project']['name']))
Tom Pillot's avatar
Tom Pillot committed
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
    gh_repo = project.github()
    ci_web_url = project.url_gitlab() + '/pipelines/' + str(pipeline_id)
    logger.debug(f'{namespace.slug}/{project.slug}: Pipeline #{pipeline_id} on commit {commit} for branch {branch}, '
                 f'status: {gl_status}')

    # Report the status to Github
    if gl_status in ['pending', 'success', 'failed']:
        gh_status = gl_status if gl_status != 'failed' else 'failure'
        if branch.startswith('pr/'):
            gh_repo.get_commit(sha=commit).create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
        else:
            try:
                gh_commit = gh_repo.get_branch(branch).commit
                gh_commit.create_status(state=gh_status, target_url=ci_web_url, context='gitlab-ci')
            except github.GithubException as e:
                if e.status == 404:
                    # Happens when a new branch is created on gitlab and the pipeline event comes before the push event
                    logger.warning(f"Branch {branch} does not exist on github, unable to report the pipeline status.")
                else:
                    raise

Guilhem Saurel's avatar
Guilhem Saurel committed
222
223
224
    return HttpResponse(rep)


Guilhem Saurel's avatar
Guilhem Saurel committed
225
226
227
228
229
230
231
232
@csrf_exempt
def webhook(request: HttpRequest) -> HttpResponse:
    """
    Process request incoming from a github webhook.

    thx https://simpleisbetterthancomplex.com/tutorial/2016/10/31/how-to-handle-github-webhooks-using-django.html
    """
    # validate ip source
Tom Pillot's avatar
Tom Pillot committed
233
    forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR').split(', ')[0]
234
    # networks = httpx.get('https://api.github.com/meta').json()['hooks'] # Fails if API rate limit exceeded
Tom Pillot's avatar
Tom Pillot committed
235
    networks = ['185.199.108.0/22', '140.82.112.0/20']
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
236
    if not any(ip_address(forwarded_for) in ip_network(net) for net in networks):
Tom Pillot's avatar
Tom Pillot committed
237
        logger.warning('not from github IP')
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
238
        return HttpResponseRedirect(reverse('login'))
Guilhem Saurel's avatar
Guilhem Saurel committed
239
240
241
242

    # validate signature
    signature = request.META.get('HTTP_X_HUB_SIGNATURE')
    if signature is None:
Tom Pillot's avatar
Tom Pillot committed
243
        logger.warning('no signature')
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
244
245
246
        return HttpResponseRedirect(reverse('login'))
    algo, signature = signature.split('=')
    if algo != 'sha1':
Tom Pillot's avatar
Tom Pillot committed
247
        logger.warning('signature not sha-1')
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
248
        return HttpResponseServerError('I only speak sha1.', status=501)
Guilhem Saurel's avatar
Guilhem Saurel committed
249

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
250
251
    mac = hmac.new(force_bytes(settings.GITHUB_WEBHOOK_KEY), msg=force_bytes(request.body), digestmod=sha1)
    if not hmac.compare_digest(force_bytes(mac.hexdigest()), force_bytes(signature)):
Tom Pillot's avatar
Tom Pillot committed
252
        logger.warning('wrong signature')
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
253
        return HttpResponseForbidden('wrong signature.')
Guilhem Saurel's avatar
Guilhem Saurel committed
254
255
256
257

    # process event
    event = request.META.get('HTTP_X_GITHUB_EVENT', 'ping')
    if event == 'ping':
Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
258
        return HttpResponse('pong')
Guilhem Saurel's avatar
Guilhem Saurel committed
259
    if event == 'push':
Tom Pillot's avatar
Tom Pillot committed
260
        return push(request, SOURCES.github, 'push event detected')
Guilhem Saurel's avatar
sync    
Guilhem Saurel committed
261
262
    if event == 'check_suite':
        return check_suite(request, 'check_suite event detected')
Guilhem Saurel's avatar
Guilhem Saurel committed
263
    if event == 'pull_request':
Tom Pillot's avatar
Tom Pillot committed
264
        return pull_request(request, 'pull_request event detected')
Guilhem Saurel's avatar
Guilhem Saurel committed
265

Guilhem Saurel's avatar
clean    
Guilhem Saurel committed
266
    return HttpResponseForbidden('event not found')
Guilhem Saurel's avatar
Guilhem Saurel committed
267
268
269
270


@csrf_exempt
def gl_webhook(request: HttpRequest) -> HttpResponse:
Tom Pillot's avatar
Tom Pillot committed
271
272
    """Process request incoming from a gitlab webhook."""

Guilhem Saurel's avatar
Guilhem Saurel committed
273
274
    # validate ip source
    if not ip_laas(request):
Tom Pillot's avatar
Tom Pillot committed
275
        logger.warning('not from LAAS IP')
Guilhem Saurel's avatar
Guilhem Saurel committed
276
277
278
279
280
        return HttpResponseRedirect(reverse('login'))

    # validate token
    token = request.META.get('HTTP_X_GITLAB_TOKEN')
    if token is None:
Tom Pillot's avatar
Tom Pillot committed
281
        logger.warning('no token')
Guilhem Saurel's avatar
Guilhem Saurel committed
282
        return HttpResponseRedirect(reverse('login'))
Guilhem Saurel's avatar
Guilhem Saurel committed
283
    if token != settings.GITLAB_WEBHOOK_KEY:
Tom Pillot's avatar
Tom Pillot committed
284
        logger.warning('wrong token')
Guilhem Saurel's avatar
Guilhem Saurel committed
285
286
        return HttpResponseForbidden('wrong token.')

Tom Pillot's avatar
Tom Pillot committed
287
    event = request.META.get('HTTP_X_GITLAB_EVENT')
Guilhem Saurel's avatar
Guilhem Saurel committed
288
289
    if event == 'ping':
        return HttpResponse('pong')
Tom Pillot's avatar
Tom Pillot committed
290
    elif event == 'Pipeline Hook':
Guilhem Saurel's avatar
Guilhem Saurel committed
291
        return pipeline(request, 'pipeline event detected')
Tom Pillot's avatar
Tom Pillot committed
292
293
    elif event == 'Push Hook':
        return push(request, SOURCES.gitlab, 'push event detected')
Guilhem Saurel's avatar
Guilhem Saurel committed
294
295

    return HttpResponseForbidden('event not found')