Skip to content

Commit 08c9ac2

Browse files
LilykosLilykos
committed
services: create zenodo deposit through CAP
* creates a Zenodo deposit, with files from CAP * saves metadata about the Zenodo deposit, and attaches it to a CAP deposit * integration tests Signed-off-by: Ilias Koutsakis <ilias.koutsakis@cern.ch>
1 parent 4ff923c commit 08c9ac2

File tree

6 files changed

+423
-53
lines changed

6 files changed

+423
-53
lines changed

‎cap/config.py‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -720,10 +720,7 @@ def _(x):
720720

721721
# Zenodo
722722
# ======
723-
ZENODO_SERVER_URL = os.environ.get('APP_ZENODO_SERVER_URL',
724-
'https://zenodo.org/api')
725-
726-
ZENODO_ACCESS_TOKEN = os.environ.get('APP_ZENODO_ACCESS_TOKEN', 'CHANGE_ME')
723+
ZENODO_SERVER_URL = os.environ.get('APP_ZENODO_SERVER_URL', 'https://zenodo.org/api') # noqa
727724

728725
# Endpoints
729726
# =========

‎cap/modules/deposit/api.py‎

Lines changed: 60 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575
UpdateDepositPermission)
7676

7777
from .review import Reviewable
78+
from .tasks import upload_to_zenodo
7879

7980
_datastore = LocalProxy(lambda: current_app.extensions['security'].datastore)
8081

@@ -254,53 +255,65 @@ def upload(self, pid, *args, **kwargs):
254255
_, rec = request.view_args.get('pid_value').data
255256
record_uuid = str(rec.id)
256257
data = request.get_json()
257-
webhook = data.get('webhook', False)
258-
event_type = data.get('event_type', 'release')
259-
260-
try:
261-
url = data['url']
262-
except KeyError:
263-
raise FileUploadError('Missing url parameter.')
264-
265-
try:
266-
host, owner, repo, branch, filepath = parse_git_url(url)
267-
api = create_git_api(host, owner, repo, branch,
268-
current_user.id)
269-
270-
if filepath:
271-
if webhook:
272-
raise FileUploadError(
273-
'You cannot create a webhook on a file')
274-
275-
download_repo_file(
276-
record_uuid,
277-
f'repositories/{host}/{owner}/{repo}/{api.branch or api.sha}/{filepath}', # noqa
278-
*api.get_file_download(filepath),
279-
api.auth_headers,
280-
)
281-
elif webhook:
282-
if event_type == 'release':
283-
if branch:
284-
raise FileUploadError(
285-
'You cannot create a release webhook'
286-
' for a specific branch or sha.')
287-
288-
if event_type == 'push' and \
289-
api.branch is None and api.sha:
290-
raise FileUploadError(
291-
'You cannot create a push webhook'
292-
' for a specific sha.')
258+
target = data.get('target')
293259

294-
create_webhook(record_uuid, api, event_type)
260+
if target == 'zenodo':
261+
files = data.get('files')
262+
bucket = data.get('bucket')
263+
if files and bucket:
264+
upload_to_zenodo.delay(record_uuid, files, bucket)
295265
else:
296-
download_repo.delay(
297-
record_uuid,
298-
f'repositories/{host}/{owner}/{repo}/{api.branch or api.sha}.tar.gz', # noqa
299-
api.get_repo_download(),
300-
api.auth_headers)
266+
raise FileUploadError(
267+
'You cannot create an empty Zenodo deposit. '
268+
'Please add some files.')
269+
else:
270+
webhook = data.get('webhook', False)
271+
event_type = data.get('event_type', 'release')
272+
273+
try:
274+
url = data['url']
275+
except KeyError:
276+
raise FileUploadError('Missing url parameter.')
277+
278+
try:
279+
host, owner, repo, branch, filepath = parse_git_url(url) # noqa
280+
api = create_git_api(host, owner, repo, branch,
281+
current_user.id)
282+
283+
if filepath:
284+
if webhook:
285+
raise FileUploadError(
286+
'You cannot create a webhook on a file')
287+
288+
download_repo_file(
289+
record_uuid,
290+
f'repositories/{host}/{owner}/{repo}/{api.branch or api.sha}/{filepath}', # noqa
291+
*api.get_file_download(filepath),
292+
api.auth_headers,
293+
)
294+
elif webhook:
295+
if event_type == 'release':
296+
if branch:
297+
raise FileUploadError(
298+
'You cannot create a release webhook'
299+
' for a specific branch or sha.')
300+
301+
if event_type == 'push' and \
302+
api.branch is None and api.sha:
303+
raise FileUploadError(
304+
'You cannot create a push webhook'
305+
' for a specific sha.')
306+
307+
create_webhook(record_uuid, api, event_type)
308+
else:
309+
download_repo.delay(
310+
record_uuid,
311+
f'repositories/{host}/{owner}/{repo}/{api.branch or api.sha}.tar.gz', # noqa
312+
api.get_repo_download(),
313+
api.auth_headers)
301314

302-
except GitError as e:
303-
raise FileUploadError(str(e))
315+
except GitError as e:
316+
raise FileUploadError(str(e))
304317

305318
return self
306319

@@ -584,16 +597,15 @@ def validate(self, **kwargs):
584597

585598
validator = NoRequiredValidator(schema, resolver=resolver)
586599

587-
result = {}
588-
result['errors'] = [
600+
errors = [
589601
FieldError(
590602
list(error.path)+error.validator_value,
591603
str(error.message))
592604
for error in validator.iter_errors(self)
593605
]
594606

595-
if result['errors']:
596-
raise DepositValidationError(None, errors=result['errors'])
607+
if errors:
608+
raise DepositValidationError(None, errors=errors)
597609
except RefResolutionError:
598610
raise DepositValidationError('Schema {} not found.'.format(
599611
self['$schema']))

‎cap/modules/deposit/tasks.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# This file is part of CERN Analysis Preservation Framework.
4+
# Copyright (C) 2018 CERN.
5+
#
6+
# CERN Analysis Preservation Framework is free software; you can redistribute
7+
# it and/or modify it under the terms of the GNU General Public License as
8+
# published by the Free Software Foundation; either version 2 of the
9+
# License, or (at your option) any later version.
10+
#
11+
# CERN Analysis Preservation Framework is distributed in the hope that it will
12+
# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
# General Public License for more details.
15+
#
16+
# You should have received a copy of the GNU General Public License
17+
# along with CERN Analysis Preservation Framework; if not, write to the
18+
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
19+
# MA 02111-1307, USA.
20+
#
21+
# In applying this license, CERN does not
22+
# waive the privileges and immunities granted to it by virtue of its status
23+
# as an Intergovernmental Organization or submit itself to any jurisdiction.
24+
"""Tasks."""
25+
26+
from __future__ import absolute_import, print_function
27+
28+
import requests
29+
from flask import current_app
30+
from celery import shared_task
31+
from invenio_db import db
32+
from invenio_files_rest.models import FileInstance, ObjectVersion
33+
34+
from cap.modules.auth.ext import _fetch_token
35+
from cap.modules.deposit.errors import FileUploadError
36+
37+
38+
@shared_task(autoretry_for=(Exception, ),
39+
retry_kwargs={
40+
'max_retries': 5,
41+
'countdown': 10
42+
})
43+
def upload_to_zenodo(record_uuid, files, bucket):
44+
"""Upload code to zenodo."""
45+
from cap.modules.deposit.api import CAPDeposit
46+
record = CAPDeposit.get_record(record_uuid)
47+
48+
zenodo_server_url = current_app.config.get('ZENODO_SERVER_URL')
49+
token = _fetch_token('zenodo')
50+
51+
if not token:
52+
raise FileUploadError(
53+
'Token not found, please connect your Zenodo '
54+
'account before creating a deposit.')
55+
56+
# in order to upload first create an empty deposit
57+
deposit = requests.post(
58+
url=f'{zenodo_server_url}/deposit/depositions',
59+
params=dict(access_token=token),
60+
json={},
61+
headers={'Content-Type': 'application/json'}
62+
)
63+
64+
if not deposit.ok:
65+
raise FileUploadError(
66+
'Something went wrong, Zenodo deposit not created.')
67+
68+
data = deposit.json()
69+
bucket_url = data['links']['bucket']
70+
depid = data['id']
71+
72+
# TODO: fix with serializers
73+
zenodo_deposit = {
74+
'id': depid,
75+
'links': {
76+
'self': data['links']['self'],
77+
'html': data['links']['html'],
78+
'publish': data['links']['publish'],
79+
},
80+
'files': []
81+
}
82+
83+
for filename in files:
84+
file_obj = ObjectVersion.get(bucket, filename)
85+
file_ins = FileInstance.get(file_obj.file_id)
86+
87+
# upload each file in the deposit
88+
with open(file_ins.uri, 'rb') as fp:
89+
file = requests.put(
90+
url=f'{bucket_url}/{filename}',
91+
data=fp,
92+
params=dict(access_token=token),
93+
)
94+
95+
if not file.ok:
96+
raise FileUploadError(
97+
'Files could not be attached to Zenodo deposit.')
98+
99+
data = file.json()
100+
zenodo_deposit['files'].append({
101+
'self': data['links']['self'],
102+
'key': data['key'],
103+
'size': data['size']
104+
})
105+
106+
# optionally add metadata
107+
# resp = requests.put(
108+
# url=f'{zenodo_server_url}/deposit/depositions/{depid}',
109+
# params=dict(access_token=token),
110+
# data=json.dumps({}),
111+
# headers={'Content-Type': 'application/json'}
112+
# )
113+
114+
# get the `_zenodo` field or add it with default []
115+
record.setdefault('_zenodo', []).append(zenodo_deposit)
116+
record.commit()
117+
db.session.commit()

‎docker-services.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ services:
2727
- "INVENIO_RATELIMIT_STORAGE_URL=redis://cache:6379/3"
2828
- "INVENIO_CERN_APP_CREDENTIALS_KEY=CHANGE_ME"
2929
- "INVENIO_CERN_APP_CREDENTIALS_SECRET=CHANGE_ME"
30+
- "INVENIO_ZENODO_CLIENT_ID=CHANGE_ME"
31+
- "INVENIO_ZENODO_CLIENT_SECRET=CHANGE_ME"
3032
- "DEV_HOST=CHANGE_ME"
3133
lb:
3234
build: ./docker/haproxy/

‎tests/conftest.py‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import tempfile
2929
from datetime import datetime, timedelta
3030
from uuid import uuid4
31+
from six import BytesIO
3132

3233
import pytest
3334
from flask import current_app
@@ -108,7 +109,8 @@ def default_config():
108109
DEBUG=False,
109110
TESTING=True,
110111
APP_GITLAB_OAUTH_ACCESS_TOKEN='testtoken',
111-
MAIL_DEFAULT_SENDER="analysis-preservation-support@cern.ch")
112+
MAIL_DEFAULT_SENDER="analysis-preservation-support@cern.ch",
113+
ZENODO_SERVER_URL='https://zenodo-test.org')
112114

113115

114116
@pytest.fixture(scope='session')
@@ -401,6 +403,21 @@ def deposit(example_user, create_deposit):
401403
)
402404

403405

406+
@pytest.fixture
407+
def deposit_with_file(example_user, create_schema, create_deposit):
408+
"""New deposit with files."""
409+
create_schema('test-schema', experiment='CMS')
410+
return create_deposit(
411+
example_user,
412+
'test-schema',
413+
{
414+
'$ana_type': 'test-schema',
415+
'title': 'test title'
416+
},
417+
files={'test-file.txt': BytesIO(b'Hello world!')},
418+
experiment='CMS')
419+
420+
404421
@pytest.fixture
405422
def record(example_user, create_deposit):
406423
"""Example record."""

0 commit comments

Comments
 (0)