Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Gepetto
dashboard
Commits
3492c7d3
Commit
3492c7d3
authored
Jul 24, 2020
by
Tom Pillot
Browse files
gitlab pipeline
parent
4004cb11
Changes
6
Hide whitespace changes
Inline
Side-by-side
Dockerfile
View file @
3492c7d3
...
...
@@ -20,9 +20,7 @@ RUN apt-get update -qq && apt-get install -qqy \
netcat-openbsd
\
make
\
msmtp
\
&&
curl
-fsSL
https://download.docker.com/linux/debian/gpg | apt-key add -
\
&&
echo
"deb [arch=armhf] https://download.docker.com/linux/debian buster stable"
>>
/etc/apt/sources.list
\
&&
apt-get update
-qq
&&
apt-get
install
-qqy
docker-ce
&&
curl
-sSL
https://get.docker.com/ | sh
RUN
pip3
install
--no-cache-dir
\
gunicorn
\
...
...
@@ -36,10 +34,8 @@ RUN pipenv install --system --deploy
ADD
. .
CMD
rm -f /opt/openrobots/etc/robotpkg.conf \
&& git clone git://git.openrobots.org/robots/robotpkg /srv/dashboard/robotpkg || true \
&& git clone git://git.openrobots.org/robots/robotpkg/robotpkg-wip /srv/dashboard/robotpkg/wip || true \
&& /srv/dashboard/robotpkg/bootstrap/bootstrap \
CMD
mkdir -p /srv/dashboard/robotpkg \
&& git clone https://github.com/Ozon2/example-adder.git /srv/dashboard/robotpkg/example-adder || true \
&& while ! nc -z postgres 5432; do sleep 1; done \
&& ./manage.py migrate \
&& ./manage.py collectstatic --no-input \
...
...
gh/views.py
View file @
3492c7d3
"""Views for dashboard_apps."""
import
hmac
import
git
from
hashlib
import
sha1
from
ipaddress
import
ip_address
,
ip_network
from
json
import
loads
from
pprint
import
pprint
import
git
import
requests
from
autoslug.utils
import
slugify
from
django.conf
import
settings
from
django.http
import
HttpRequest
from
django.http.response
import
(
HttpResponse
,
HttpResponseBadRequest
,
HttpResponseForbidden
,
HttpResponseRedirect
,
...
...
@@ -14,12 +16,8 @@ from django.shortcuts import get_object_or_404, reverse
from
django.utils.encoding
import
force_bytes
from
django.views.decorators.csrf
import
csrf_exempt
import
requests
from
autoslug.utils
import
slugify
from
dashboard.middleware
import
ip_laas
from
rainboard.models
import
Forge
,
Namespace
,
Project
from
rainboard.models
import
Namespace
,
Project
from
.
import
models
...
...
@@ -37,7 +35,8 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
branch
=
f
'pr/
{
data
[
"number"
]
}
'
print
(
f
'Pull request:
{
event
}
'
)
namespace
=
get_object_or_404
(
Namespace
,
slug
=
slugify
(
data
[
'repository'
][
'owner'
][
'login'
]))
# namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['login']))
namespace
=
get_object_or_404
(
Namespace
,
slug
=
'tpillot'
)
project
=
get_object_or_404
(
Project
,
main_namespace
=
namespace
,
slug
=
slugify
(
data
[
'repository'
][
'name'
]))
git_repo
=
project
.
git
()
...
...
@@ -81,8 +80,8 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
if
branch
in
git_repo
.
branches
:
git_repo
.
delete_head
(
branch
,
force
=
True
)
git_repo
.
delete_remote
(
gh_remote_name
)
project
.
gitlab
().
branches
.
delete
(
branch
)
print
(
f
'Deleted branch
{
branch
}
'
)
project
.
gitlab
().
branches
.
delete
(
branch
)
print
(
f
'Deleted branch
{
branch
}
'
)
return
HttpResponse
(
rep
)
...
...
@@ -90,67 +89,81 @@ def pull_request(request: HttpRequest, rep: str) -> HttpResponse:
def
push
(
request
:
HttpRequest
,
rep
:
str
)
->
HttpResponse
:
"""Someone pushed on github. Synchronise local repo & gitlab."""
data
=
loads
(
request
.
body
.
decode
())
namespace
=
get_object_or_404
(
Namespace
,
slug
=
slugify
(
data
[
'repository'
][
'owner'
][
'name'
]))
# namespace = get_object_or_404(Namespace, slug=slugify(data['repository']['owner']['name']))
namespace
=
get_object_or_404
(
Namespace
,
slug
=
'tpillot'
)
project
=
get_object_or_404
(
Project
,
main_namespace
=
namespace
,
slug
=
slugify
(
data
[
'repository'
][
'name'
]))
branch_name
=
data
[
'ref'
][
11
:]
# strip 'refs/heads/'
print
(
f
'Push detected on github:
{
branch_name
}
'
)
branch
=
data
[
'ref'
][
11
:]
# strip 'refs/heads/'
gh_remote_name
=
f
'github/
{
namespace
.
slug
}
'
gl_remote_name
=
f
'gitlab/
{
namespace
.
slug
}
'
gh_branch_name
=
f
'
{
gh_remote_name
}
/
{
branch_name
}
'
gl_branch_name
=
f
'
{
gl_remote_name
}
/
{
branch_name
}
'
git_repo
=
project
.
git
()
gh_remote
=
git_repo
.
remote
(
gh_remote_name
)
print
(
f
'Push detected on github:
{
branch
}
'
)
# TODO: hard reset if necessary ?
if
gh_remote_name
in
git_repo
.
remotes
:
gh_remote
=
git_repo
.
remote
(
gh_remote_name
)
else
:
gh_remote
=
git_repo
.
create_remote
(
gh_remote_name
,
url
=
project
.
url_github
())
gh_remote
.
fetch
()
# The branch was deleted, delete the branch on Gitlab
if
data
[
'after'
]
==
"0000000000000000000000000000000000000000"
:
print
(
"Branch deleted"
)
for
branch
in
[
gh_branch_name
,
gl_branch_name
,
branch_name
]:
if
branch
in
git_repo
.
branches
:
git_repo
.
delete_head
(
branch
,
force
=
True
)
project
.
gitlab
().
branches
.
delete
(
branch_name
)
if
branch
in
git_repo
.
branches
:
git_repo
.
delete_head
(
branch
,
force
=
True
)
project
.
gitlab
().
branches
.
delete
(
branch
)
return
HttpResponse
(
rep
)
# TODO: Is that necessary ?
gh_ref
=
gh_remote
.
refs
[
branch_name
]
gh_ref
=
gh_remote
.
refs
[
branch
]
if
str
(
gh_ref
.
commit
)
!=
data
[
'after'
]:
fail
=
f
'
p
ush: wrong commit:
{
gh_ref
.
commit
}
vs
{
data
[
"after"
]
}
'
fail
=
f
'
P
ush: wrong commit:
{
gh_ref
.
commit
}
vs
{
data
[
"after"
]
}
'
print
(
fail
)
return
HttpResponseBadRequest
(
fail
)
# TODO: Is that necessary ?
if
gl_remote_name
not
in
git_repo
.
remotes
:
print
(
f
'project
{
project
}
not available on
{
gl_remote_name
}
'
)
return
HttpResponse
(
rep
)
# Update branch to the last commit
if
branch
in
git_repo
.
branches
:
git_repo
.
branches
[
branch
].
commit
=
data
[
'after'
]
else
:
git_repo
.
create_head
(
branch
,
commit
=
data
[
'after'
])
# Update branches to the last commit
for
branch
in
[
gh_branch_name
,
gl_branch_name
,
branch_name
]:
if
branch
in
git_repo
.
branches
:
git_repo
.
branches
[
branch
].
commit
=
data
[
'after'
]
else
:
git_repo
.
create_head
(
branch
,
commit
=
data
[
'after'
])
if
gl_remote_name
in
git_repo
.
remotes
:
gl_remote
=
git_repo
.
remote
(
gl_remote_name
)
else
:
gl_remote
=
git_repo
.
create_remote
(
gl_remote_name
,
url
=
project
.
url_gitlab
())
# Push the changes to gitlab
# TODO: force push
gl_remote
=
git_repo
.
remote
(
gl_remote_name
)
# TODO: Force push
gl_remote
.
fetch
()
if
branch
_name
not
in
gl_remote
.
refs
or
str
(
gl_remote
.
refs
[
branch
_name
].
commit
)
!=
data
[
'after'
]:
print
(
f
'pushing
{
data
[
"after"
]
}
on
{
branch
_name
}
on gitlab'
)
gl_remote
.
push
(
branch
_name
)
if
branch
not
in
gl_remote
.
refs
or
str
(
gl_remote
.
refs
[
branch
].
commit
)
!=
data
[
'after'
]:
print
(
f
'pushing
{
data
[
"after"
]
}
on
{
branch
}
on gitlab'
)
gl_remote
.
push
(
branch
)
return
HttpResponse
(
rep
)
def
pipeline
(
request
:
HttpRequest
,
rep
:
str
)
->
HttpResponse
:
"""Something happened on a Gitlab pipeline. Tell Github if necessary."""
print
(
'pipeline'
)
data
=
loads
(
request
.
body
.
decode
())
namespace
=
get_object_or_404
(
Namespace
,
slug
=
slugify
(
data
[
'project'
][
'namespace'
]))
branch
,
commit
,
gl_status
,
pipeline_id
=
(
data
[
'object_attributes'
][
key
]
for
key
in
[
'ref'
,
'sha'
,
'status'
,
'id'
])
print
(
f
'Pipeline id #
{
pipeline_id
}
on commit
{
commit
}
for branch
{
branch
}
, status:
{
gl_status
}
'
)
# namespace = get_object_or_404(Namespace, slug=slugify(data['project']['namespace']))
namespace
=
get_object_or_404
(
Namespace
,
slug
=
'tpillot'
)
project
=
get_object_or_404
(
Project
,
main_namespace
=
namespace
,
slug
=
slugify
(
data
[
'project'
][
'name'
]))
branch
,
commit
,
status
=
(
data
[
'object_attributes'
][
key
]
for
key
in
[
'ref'
,
'sha'
,
'status'
])
# status in ['pending', 'running', 'success', 'failed']
print
(
namespace
,
project
,
branch
,
commit
,
status
)
gh_repo
=
project
.
github
()
ci_web_url
=
project
.
url_gitlab
()
+
'/pipelines/'
+
str
(
pipeline_id
)
# Report the status to Github
if
gl_status
in
[
'pending'
,
'success'
,
'failed'
]:
gh_status
=
gl_status
if
gl_status
!=
'failed'
else
'failure'
if
branch
.
startswith
(
'pr/'
):
pr_number
=
int
(
branch
[
3
:])
pr_commit_sha
=
gh_repo
.
get_pull
(
pr_number
).
head
.
sha
gh_repo
.
get_commit
(
sha
=
pr_commit_sha
).
create_status
(
state
=
gh_status
,
target_url
=
ci_web_url
,
context
=
'gitlab-ci'
)
else
:
gh_commit
=
gh_repo
.
get_branch
(
branch
).
commit
gh_commit
.
create_status
(
state
=
gh_status
,
target_url
=
ci_web_url
,
context
=
'gitlab-ci'
)
return
HttpResponse
(
rep
)
...
...
@@ -162,7 +175,7 @@ def webhook(request: HttpRequest) -> HttpResponse:
thx https://simpleisbetterthancomplex.com/tutorial/2016/10/31/how-to-handle-github-webhooks-using-django.html
"""
# validate ip source
forwarded_for
=
request
.
META
.
get
(
'HTTP_X_FORWARDED_FOR'
)
forwarded_for
=
request
.
META
.
get
(
'HTTP_X_FORWARDED_FOR'
)
.
split
(
', '
)[
0
]
networks
=
requests
.
get
(
'https://api.github.com/meta'
).
json
()[
'hooks'
]
if
not
any
(
ip_address
(
forwarded_for
)
in
ip_network
(
net
)
for
net
in
networks
):
print
(
'not from github IP'
)
...
...
@@ -200,6 +213,8 @@ def webhook(request: HttpRequest) -> HttpResponse:
@
csrf_exempt
def
gl_webhook
(
request
:
HttpRequest
)
->
HttpResponse
:
"""Process request incoming from a gitlab webhook."""
# validate ip source
if
not
ip_laas
(
request
):
print
(
'not from LAAS IP'
)
...
...
rainboard/migrations/0045_project_accept_pr_to_master.py
0 → 100644
View file @
3492c7d3
# Generated by Django 3.0.8 on 2020-07-24 13:45
from
django.db
import
migrations
,
models
class
Migration
(
migrations
.
Migration
):
dependencies
=
[
(
'rainboard'
,
'0044_focal'
),
]
operations
=
[
migrations
.
AddField
(
model_name
=
'project'
,
name
=
'accept_pr_to_master'
,
field
=
models
.
BooleanField
(
default
=
False
),
),
]
rainboard/migrations/004
5
_example-adder.py
→
rainboard/migrations/004
6
_example-adder.py
View file @
3492c7d3
...
...
@@ -3,6 +3,11 @@
from
django.db
import
migrations
def
add_namespace
(
apps
,
schema_editor
):
Namespace
=
apps
.
get_model
(
'rainboard'
,
'Namespace'
)
Namespace
.
objects
.
create
(
name
=
'tpillot'
,
group
=
False
)
def
add_project
(
apps
,
schema_editor
):
Project
=
apps
.
get_model
(
'rainboard'
,
'Project'
)
Namespace
=
apps
.
get_model
(
'rainboard'
,
'Namespace'
)
...
...
@@ -11,28 +16,7 @@ def add_project(apps, schema_editor):
git_project
=
Project
(
name
=
'example-adder'
,
public
=
True
,
main_namespace
=
Namespace
.
objects
.
get
(
name
=
'Gepetto'
),
main_forge
=
Forge
.
objects
.
get
(
name
=
'Gitlab'
),
license
=
None
,
homepage
=
None
,
description
=
None
,
version
=
None
,
updated
=
None
,
tests
=
True
,
docs
=
True
,
from_gepetto
=
True
,
cmake_name
=
None
,
archived
=
False
,
suffix
=
''
,
allow_format_failure
=
True
,
has_python
=
True
)
git_project
.
save
()
git_project
=
Project
(
name
=
'talos-data'
,
public
=
True
,
main_namespace
=
Namespace
.
objects
.
get
(
name
=
'Gepetto'
),
main_namespace
=
Namespace
.
objects
.
get
(
name
=
'tpillot'
),
main_forge
=
Forge
.
objects
.
get
(
name
=
'Gitlab'
),
license
=
None
,
homepage
=
None
,
...
...
@@ -53,9 +37,10 @@ def add_project(apps, schema_editor):
class
Migration
(
migrations
.
Migration
):
dependencies
=
[
(
'rainboard'
,
'004
4_focal
'
),
(
'rainboard'
,
'004
5_project_accept_pr_to_master
'
),
]
operations
=
[
migrations
.
RunPython
(
add_namespace
),
migrations
.
RunPython
(
add_project
),
]
rainboard/migrations/004
6
_robotpkg.py
→
rainboard/migrations/004
7
_robotpkg.py
View file @
3492c7d3
...
...
@@ -7,22 +7,22 @@ path = settings.RAINBOARD_RPKG
def
add_robotpkg
(
apps
,
schema_editor
):
for
project
in
[
Project
.
objects
.
get
(
slug
=
'example-adder'
)
,
Project
.
objects
.
get
(
slug
=
'talos-data'
)]:
project
=
Project
.
objects
.
get
(
slug
=
'example-adder'
)
for
slug
in
[
project
.
slug
,
project
.
slug
.
replace
(
'_'
,
'-'
)]:
for
pkg
in
itertools
.
chain
(
path
.
glob
(
f
'*/
{
slug
}{
project
.
suffix
}
'
),
path
.
glob
(
f
'*/py-
{
slug
}{
project
.
suffix
}
'
)):
obj
,
created
=
Robotpkg
.
objects
.
get_or_create
(
name
=
pkg
.
name
,
category
=
pkg
.
parent
.
name
,
project
=
project
)
if
created
:
print
(
project
,
pkg
)
obj
.
update
(
pull
=
False
)
for
slug
in
[
project
.
slug
,
project
.
slug
.
replace
(
'_'
,
'-'
)]:
for
pkg
in
itertools
.
chain
(
path
.
glob
(
f
'*/
{
slug
}{
project
.
suffix
}
'
),
path
.
glob
(
f
'*/py-
{
slug
}{
project
.
suffix
}
'
)):
obj
,
created
=
Robotpkg
.
objects
.
get_or_create
(
name
=
pkg
.
name
,
category
=
pkg
.
parent
.
name
,
project
=
project
)
if
created
:
print
(
project
,
pkg
)
obj
.
update
(
pull
=
False
)
for
rpkg
in
project
.
robotpkg_set
.
all
():
rpkg
.
update_images
()
for
rpkg
in
project
.
robotpkg_set
.
all
():
rpkg
.
update_images
()
class
Migration
(
migrations
.
Migration
):
dependencies
=
[
(
'rainboard'
,
'004
5
_example-adder'
),
(
'rainboard'
,
'004
6
_example-adder'
),
]
operations
=
[
...
...
rainboard/models.py
View file @
3492c7d3
...
...
@@ -407,6 +407,10 @@ class Project(Links, NamedModel, TimeStampedModel):
def
url_gitlab
(
self
):
return
f
'https://gitlab.laas.fr/
{
self
.
main_namespace
.
slug
}
/
{
self
.
slug
}
'
def
url_github
(
self
):
# return f'https://github.com/{self.main_namespace.slug}/{self.slug}'
return
f
'https://github.com/Ozon2/
{
self
.
slug
}
'
def
badge
(
self
,
link
,
img
,
alt
):
return
mark_safe
(
f
'<a href="
{
link
}
"><img src="
{
img
}
" alt="
{
alt
}
" /></a> '
)
...
...
@@ -919,11 +923,15 @@ class Image(models.Model):
},
auth
=
(
'gsaurel'
,
self
.
robotpkg
.
project
.
main_forge
.
token
)).
json
()[
'token'
]
headers
[
'Authorization'
]
=
f
'Bearer
{
token
}
'
r
=
requests
.
get
(
self
.
get_image_url
(),
headers
=
headers
)
if
r
.
status_code
==
200
:
self
.
image
=
r
.
json
()[
'fsLayers'
][
0
][
'blobSum'
].
split
(
':'
)[
1
][:
12
]
self
.
created
=
parse_datetime
(
json
.
loads
(
r
.
json
()[
'history'
][
0
][
'v1Compatibility'
])[
'created'
])
self
.
save
()
# r = requests.get(self.get_image_url(), headers=headers)
# if r.status_code == 200:
# self.image = r.json()['fsLayers'][0]['blobSum'].split(':')[1][:12]
# self.created = parse_datetime(json.loads(r.json()['history'][0]['v1Compatibility'])['created'])
# self.save()
self
.
image
=
'7db86b55ec14'
self
.
created
=
parse_datetime
(
'2020-07-20T04:27:31.857218193+02:00'
)
self
.
save
()
if
not
self
.
allow_failure
and
self
.
created
and
(
timezone
.
now
()
-
self
.
created
).
days
>
7
:
self
.
allow_failure
=
True
self
.
save
()
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment