Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Gepetto
dashboard
Commits
12057875
Commit
12057875
authored
Jan 12, 2018
by
Guilhem Saurel
Browse files
spdx, api_update_github
parent
b328e845
Changes
3
Hide whitespace changes
Inline
Side-by-side
rainboard/management/commands/populate.py
View file @
12057875
...
...
@@ -6,19 +6,24 @@ from rainboard.models import Forge, License, Repo
import
requests
LICENSES
=
'https://raw.githubusercontent.com/spdx/license-list-data/master/json/licenses.json'
logger
=
logging
.
getLogger
(
'rainboard.management.populate'
)
class
Command
(
BaseCommand
):
help
=
'populates licenses, projets, namespaces and repos from forges'
def
handle
(
self
,
*
args
,
**
options
):
github
=
Forge
.
objects
.
get
(
name
=
'Github'
)
#
github = Forge.objects.get(name='Github')
logger
.
info
(
f
'updating licenses'
)
for
data
in
requests
.
get
(
f
'
{
github
.
api_url
()
}
/licenses'
,
headers
=
github
.
headers
()).
json
()
:
for
data
in
requests
.
get
(
LICENSES
).
json
()[
'licenses'
]
:
logger
.
info
(
f
' updating license
{
data
[
"name"
]
}
'
)
License
.
objects
.
get_or_create
(
github_key
=
data
[
'key'
],
defaults
=
{
key
:
data
[
key
]
for
key
in
[
'name'
,
'spdx_id'
,
'url'
]})
License
.
objects
.
get_or_create
(
spdx_id
=
data
[
'licenseId'
],
defaults
=
{
'name'
:
data
[
'name'
],
'url'
:
data
[
'detailsUrl'
]})
# for data in requests.get(f'{github.api_url()}/licenses', headers=github.headers()).json():
# logger.info(f' updating license {data["name"]}')
# License.objects.get_or_create(spdx_id=data['spdx_id'],
# defaults={key: data[key] for key in ['name', 'url']})
logger
.
info
(
f
'updating forges'
)
for
forge
in
Forge
.
objects
.
all
():
...
...
rainboard/migrations/0001_initial.py
View file @
12057875
# Generated by Django 2.0.1 on 2018-01-11 1
3
:4
0
# Generated by Django 2.0.1 on 2018-01-11 1
5
:4
6
import
autoslug.fields
from
django.db
import
migrations
,
models
...
...
@@ -60,15 +60,10 @@ class Migration(migrations.Migration):
name
=
'License'
,
fields
=
[
(
'id'
,
models
.
AutoField
(
auto_created
=
True
,
primary_key
=
True
,
serialize
=
False
,
verbose_name
=
'ID'
)),
(
'name'
,
models
.
CharField
(
max_length
=
200
,
unique
=
True
)),
(
'slug'
,
autoslug
.
fields
.
AutoSlugField
(
editable
=
False
,
populate_from
=
'name'
,
unique
=
True
)),
(
'github_key'
,
models
.
CharField
(
max_length
=
50
)),
(
'spdx_id'
,
models
.
CharField
(
max_length
=
50
)),
(
'name'
,
models
.
CharField
(
max_length
=
200
)),
(
'spdx_id'
,
models
.
CharField
(
max_length
=
50
,
unique
=
True
)),
(
'url'
,
models
.
URLField
()),
],
options
=
{
'abstract'
:
False
,
},
),
migrations
.
CreateModel
(
name
=
'Namespace'
,
...
...
rainboard/models.py
View file @
12057875
...
...
@@ -18,9 +18,9 @@ class Namespace(NamedModel):
group
=
models
.
BooleanField
(
default
=
False
)
class
License
(
Named
Model
):
github_key
=
models
.
CharField
(
max_length
=
5
0
)
spdx_id
=
models
.
CharField
(
max_length
=
50
)
class
License
(
models
.
Model
):
name
=
models
.
CharField
(
max_length
=
20
0
)
spdx_id
=
models
.
CharField
(
max_length
=
50
,
unique
=
True
)
url
=
models
.
URLField
(
max_length
=
200
)
def
__str__
(
self
):
...
...
@@ -94,13 +94,13 @@ class Forge(Links, NamedModel):
repo
.
open_issues
=
data
[
'open_issues'
]
repo_data
=
repo
.
api_data
()
if
'license'
in
repo_data
and
repo_data
[
'license'
]:
license_data
=
repo_data
[
'license'
]
license
,
_
=
License
.
objects
.
get_or_create
(
name
=
license_data
[
'name'
],
defaults
=
{
'github_key'
:
license_data
[
'key
'
]
}
)
repo
.
license
=
license
if
not
project
.
license
:
project
.
license
=
license
if
repo_data
and
'license'
in
repo_data
and
repo_data
[
'license'
]:
print
(
repo_data
[
'license'
]
)
if
'spdx_id'
in
repo_data
[
'license'
]
and
repo_data
[
'license'
][
'spdx_id'
]:
license
=
License
.
objects
.
get
(
spdx_id
=
repo_data
[
'license'
][
'spdx_id
'
])
repo
.
license
=
license
if
not
project
.
license
:
project
.
license
=
license
repo
.
open_pr
=
len
(
repo
.
api_data
(
'/pulls'
))
repo
.
save
()
project
.
save
()
...
...
@@ -177,11 +177,17 @@ class Repo(TimeStampedModel):
return
f
'
{
self
.
forge
.
api_url
()
}
/projects/
{
self
.
repo_id
}
'
def
api_data
(
self
,
url
=
''
):
return
requests
.
get
(
self
.
api_url
()
+
url
,
verify
=
self
.
forge
.
verify
,
headers
=
self
.
forge
.
headers
()).
json
()
logger
.
info
(
f
'requesting api
{
self
.
forge
}
{
self
.
namespace
}
{
self
}
{
url
}
'
)
req
=
requests
.
get
(
self
.
api_url
()
+
url
,
verify
=
self
.
forge
.
verify
,
headers
=
self
.
forge
.
headers
())
return
req
.
json
()
if
req
.
status_code
==
200
else
[]
def
api_update
(
self
):
if
self
.
forge
.
source
==
SOURCES
.
gitlab
:
self
.
api_update_gitlab
(
self
.
api_data
())
data
=
self
.
api_data
()
if
data
:
if
self
.
forge
.
source
==
SOURCES
.
gitlab
:
return
self
.
api_update_gitlab
(
data
)
if
self
.
forge
.
source
==
SOURCES
.
github
:
return
self
.
api_update_github
(
data
)
def
api_update_gitlab
(
self
,
data
):
# TODO Missing: license, homepage, open_pr
...
...
@@ -194,6 +200,19 @@ class Repo(TimeStampedModel):
self
.
forked_from
=
data
[
'forked_from_project'
][
'id'
]
self
.
save
()
def
api_update_github
(
self
,
data
):
# TODO Missing: open_pr
self
.
name
=
data
[
'name'
]
if
data
[
'license'
]
is
not
None
:
self
.
license
=
License
.
objects
.
filter
(
spdx_id
=
data
[
'license'
][
'spdx_id'
]).
first
()
self
.
homepage
=
data
[
'homepage'
]
self
.
url
=
data
[
'url'
]
self
.
default_branch
=
data
[
'default_branch'
]
self
.
open_issues
=
data
[
'open_issues_count'
]
self
.
repo_id
=
data
[
'id'
]
if
'source'
in
data
:
self
.
forked_from
=
data
[
'source'
][
'id'
]
class
Commit
(
NamedModel
,
TimeStampedModel
):
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment