-
+
${_("Index:")}
diff --git a/addons/weko/tests/test_client.py b/addons/weko/tests/test_client.py
index d616196ec50..a624e293adc 100644
--- a/addons/weko/tests/test_client.py
+++ b/addons/weko/tests/test_client.py
@@ -19,6 +19,36 @@ def mock_requests_get(url, **kwargs):
return utils.mock_response_404
+def _fake_truncated_indices(expanded_ids):
+ grandchildren = [{'id': '300', 'name': 'Grandchild 1', 'children': []}]
+ if '201' in expanded_ids:
+ grandchildren.append({'id': '301', 'name': 'Grandchild 2', 'children': []})
+ else:
+ grandchildren.append({'id': 'more'})
+ children = [{'id': '200', 'name': 'Child 1', 'children': []}]
+ if '100' in expanded_ids:
+ children.append({'id': '201', 'name': 'Child 2', 'children': grandchildren})
+ else:
+ children.append({'id': 'more'})
+ return [{'id': '100', 'name': 'Sample Index', 'children': children}]
+
+
+def mock_requests_get_truncated(url, **kwargs):
+ prefix = 'https://test.sample.nii.ac.jp/api/tree?action=browsing'
+ if url == prefix:
+ return utils.MockResponse(_fake_truncated_indices([]), 200)
+ if url.startswith(prefix + '&more_ids='):
+ expanded_ids = url[len(prefix + '&more_ids='):].split('/')
+ return utils.MockResponse(_fake_truncated_indices(expanded_ids), 200)
+ return utils.mock_response_404
+
+
+def mock_requests_get_never_expanded(url, **kwargs):
+ if url.startswith('https://test.sample.nii.ac.jp/api/tree?action=browsing'):
+ return utils.MockResponse(_fake_truncated_indices([]), 200)
+ return utils.mock_response_404
+
+
class TestWEKOClient(OsfTestCase):
def setUp(self):
self.host = utils.fake_weko_host
@@ -35,6 +65,37 @@ def test_weko_get_indices(self, get_req_mock):
assert_equal(indices[0].title, 'Sample Index')
assert_equal(indices[0].identifier, 100)
+ @mock.patch('requests.get', side_effect=mock_requests_get_truncated)
+ def test_weko_get_indices_expands_truncated_children(self, get_req_mock):
+ indices = self.conn.get_indices()
+
+ titles = [i.title for i in client._flatten_indices(indices)]
+ assert_equal(titles, ['Sample Index', 'Child 1', 'Child 2', 'Grandchild 1', 'Grandchild 2'])
+ assert_equal(
+ [c[0][0] for c in get_req_mock.call_args_list],
+ [
+ 'https://test.sample.nii.ac.jp/api/tree?action=browsing',
+ 'https://test.sample.nii.ac.jp/api/tree?action=browsing&more_ids=100',
+ 'https://test.sample.nii.ac.jp/api/tree?action=browsing&more_ids=100/201',
+ ]
+ )
+
+ @mock.patch('requests.get', side_effect=mock_requests_get_never_expanded)
+ def test_weko_get_indices_fails_when_weko_keeps_children_truncated(self, get_req_mock):
+ with assert_raises(ValueError):
+ self.conn.get_indices()
+
+ @mock.patch('requests.get', side_effect=mock_requests_get)
+ def test_weko_get_indices_asks_for_the_given_language(self, get_req_mock):
+ self.conn.get_indices(accept_language='ja')
+
+ get_req_mock.assert_called_once_with(
+ 'https://test.sample.nii.ac.jp/api/tree?action=browsing',
+ auth=(None, None),
+ headers={'Accept-Language': 'ja'},
+ timeout=client.DEFAULT_TIMEOUT,
+ )
+
@mock.patch('requests.get', side_effect=mock_requests_get)
def test_weko_get_index_by_id(self, get_req_mock):
index = self.conn.get_index_by_id(100)
diff --git a/addons/weko/tests/test_serializer.py b/addons/weko/tests/test_serializer.py
index 51d56e40930..6fbf5e5a1fe 100644
--- a/addons/weko/tests/test_serializer.py
+++ b/addons/weko/tests/test_serializer.py
@@ -82,6 +82,42 @@ def test_serialize_settings_authorized(self):
assert_equal(serialized['ownerName'], self.user_settings.owner.fullname)
assert_in('savedIndex', serialized)
+ def test_serialize_settings_names_indices_in_the_page_language(self):
+ def requests_get(url, **kwargs):
+ if kwargs.get('headers', {}).get('Accept-Language') == 'ja':
+ return utils.MockResponse([{'id': 100, 'name': 'サンプルインデックス'}], 200)
+ return utils.MockResponse([{'id': 100, 'name': 'Sample Index'}], 200)
+
+ with mock.patch.object(type(self.node_settings), 'has_auth', return_value=True), \
+ mock.patch.object(WEKOSerializer, 'credentials_are_valid', return_value=True), \
+ mock.patch('requests.get', side_effect=requests_get), \
+ self.app.app.test_request_context(headers={'Accept-Language': 'ja-JP'}):
+ serialized = self.ser.serialize_settings(self.node_settings, self.user, self.client)
+
+ assert_equal(serialized['indices'], [{
+ 'title': 'Sample Index',
+ 'localizedTitle': 'サンプルインデックス',
+ 'id': 100,
+ 'children': [],
+ }])
+
+ def test_serialize_settings_falls_back_to_the_default_name_and_warns(self):
+ def requests_get(url, **kwargs):
+ if kwargs.get('headers', {}).get('Accept-Language') == 'ja':
+ return utils.MockResponse([{'id': 100, 'name': 'サンプルインデックス'}], 200)
+ return utils.MockResponse([{'id': 100, 'name': 'Sample Index'}, {'id': 101, 'name': 'New Index'}], 200)
+
+ with mock.patch.object(type(self.node_settings), 'has_auth', return_value=True), \
+ mock.patch.object(WEKOSerializer, 'credentials_are_valid', return_value=True), \
+ mock.patch('requests.get', side_effect=requests_get), \
+ self.app.app.test_request_context(headers={'Accept-Language': 'ja'}), \
+ self.assertLogs('addons.weko.serializer', level='WARNING') as logs:
+ serialized = self.ser.serialize_settings(self.node_settings, self.user, self.client)
+
+ assert_equal([i['localizedTitle'] for i in serialized['indices']], ['サンプルインデックス', 'New Index'])
+ assert_equal(len(logs.records), 1)
+ assert_in('101', logs.records[0].getMessage())
+
def test_serialize_settings_authorized_folder_is_set(self):
pass
diff --git a/addons/weko/tests/utils.py b/addons/weko/tests/utils.py
index 076064d897a..f04a75ae656 100644
--- a/addons/weko/tests/utils.py
+++ b/addons/weko/tests/utils.py
@@ -11,9 +11,6 @@
'id': 100,
'name': 'Sample Index',
'children': [
- {
- 'id': 'more',
- },
{
'id': 'dummy',
}
diff --git a/addons/wiki/models.py b/addons/wiki/models.py
index e20f40dd730..f8c6a1bf75f 100644
--- a/addons/wiki/models.py
+++ b/addons/wiki/models.py
@@ -266,7 +266,16 @@ def get_for_node(self, node, name=None, id=None):
deleted__isnull=True,
node=node,
).order_by('created', 'id').first()
- return WikiPage.load(id)
+ if id:
+ wiki_page = WikiPage.load(id)
+ # Restrict by node so wid cannot leak another project's page name.
+ if (
+ wiki_page
+ and wiki_page.deleted is None
+ and wiki_page.node_id == node.id
+ ):
+ return wiki_page
+ return None
def get_for_child_nodes(self, node, parent=None):
if parent:
diff --git a/addons/wiki/settings/defaults.py b/addons/wiki/settings/defaults.py
index 9fadefc5f61..bbf8ed0f0de 100644
--- a/addons/wiki/settings/defaults.py
+++ b/addons/wiki/settings/defaults.py
@@ -8,6 +8,11 @@
SHAREJS_PORT = 7007
SHAREJS_URL = '{}:{}'.format(SHAREJS_HOST, SHAREJS_PORT)
+Y_WEBSOCKET_SECRET = os.environ.get('Y_WEBSOCKET_SECRET', '')
+Y_WEBSOCKET_JWT_ALGORITHM = 'HS256'
+# Token validity for WebSocket connection (seconds). Not enforced after connection is established.
+Y_WEBSOCKET_TOKEN_TTL = int(os.environ.get('Y_WEBSOCKET_TOKEN_TTL', 8 * 60 * 60))
+
Y_WEBSOCKET_HOST = 'localhost'
Y_WEBSOCKET_PORT = 1234
Y_WEBSOCKET_URL = '{}:{}'.format(Y_WEBSOCKET_HOST, Y_WEBSOCKET_PORT)
diff --git a/addons/wiki/templates/edit.mako b/addons/wiki/templates/edit.mako
index c4fd10d92da..6356b6f15e6 100644
--- a/addons/wiki/templates/edit.mako
+++ b/addons/wiki/templates/edit.mako
@@ -522,6 +522,7 @@ ${parent.javascript_bottom()}
metadata: {
registration: true,
docId: ${ sharejs_uuid | sjson, n },
+ yWebsocketToken: ${ y_websocket_token | sjson, n },
userId: ${user_id | sjson, n },
userName: ${ user_full_name | sjson, n },
userUrl: ${ user_url | sjson, n },
diff --git a/addons/wiki/tests/test_wiki.py b/addons/wiki/tests/test_wiki.py
index 6daffd6dead..3be7918e3e3 100644
--- a/addons/wiki/tests/test_wiki.py
+++ b/addons/wiki/tests/test_wiki.py
@@ -35,7 +35,7 @@
from addons.wiki.utils import (
get_sharejs_uuid, generate_private_uuid, share_db, delete_share_doc,
migrate_uuid, format_wiki_version, serialize_wiki_settings, serialize_wiki_widget,
- check_file_object_in_node
+ check_file_object_in_node, generate_y_websocket_token
)
from addons.wiki.views import WIKI_IMPORT_TASK_ALREADY_EXISTS
from addons.wiki import tasks
@@ -448,6 +448,17 @@ def test_wiki_id_url_get_returns_404(self):
res = self.app.get(url, expect_errors=True)
assert_equal(res.status_code, 404)
+ def test_wiki_id_url_get_returns_404_for_other_node_wiki(self):
+ other = ProjectFactory(is_public=True, creator=self.user)
+ other_page = WikiPage.objects.create_for_node(
+ other, 'SecretFromOther', 'content', Auth(self.user)
+ )
+ url = self.project.web_url_for(
+ 'project_wiki_id_page', wid=other_page._primary_key, _guid=True
+ )
+ res = self.app.get(url, expect_errors=True)
+ assert_equal(res.status_code, 404)
+
def test_home_is_capitalized_in_web_view(self):
url = self.project.web_url_for('project_wiki_home', wid='home', _guid=True)
res = self.app.get(url, auth=self.user.auth).follow(auth=self.user.auth)
@@ -1861,11 +1872,21 @@ def test_project_wiki_validate_for_import(self, mock_check_file_object_in_node):
mock_check_file_object_in_node.return_value = True
dir_id = self.root_import_folder1._id
url = self.project.api_url_for('project_wiki_validate_for_import', dir_id=dir_id)
- res = self.app.get(url)
+ res = self.app.get(url, auth=self.user.auth)
response_json = res.json
task_id = response_json['taskId']
uuid_obj = uuid.UUID(task_id)
assert uuid_obj
+ task = WikiImportTask.objects.get(task_id=task_id)
+ self.assertEqual(task.node, self.project)
+ self.assertEqual(task.status, WikiImportTask.STATUS_COMPLETED)
+ self.assertIsNotNone(task.process_end)
+ self.assertEqual(
+ WikiImportTask.objects.filter(
+ node=self.project, status=WikiImportTask.STATUS_RUNNING
+ ).count(),
+ 0,
+ )
@mock.patch('addons.wiki.utils.check_file_object_in_node')
def test_project_wiki_validate_for_import_error(self, mock_check_file_object_in_node):
@@ -1875,7 +1896,7 @@ def test_project_wiki_validate_for_import_error(self, mock_check_file_object_in_
))
dir_id = self.root_import_folder1._id
url = self.project.api_url_for('project_wiki_validate_for_import', dir_id=dir_id)
- res = self.app.get(url, expect_errors=True)
+ res = self.app.get(url, auth=self.user.auth, expect_errors=True)
self.assertEqual(res.status_code, 400)
self.assertEqual(res.json['message_short'], 'directory id does not exist')
self.assertEqual(res.json['message_long'], 'directory id does not exist')
@@ -1964,6 +1985,9 @@ def test_project_wiki_import(self, mock_check_file_object_in_node, mock_project_
response_json = res.json
task_id = response_json['taskId']
uuid_obj = uuid.UUID(task_id)
+ task = WikiImportTask.objects.get(task_id=task_id)
+ self.assertEqual(task.node, self.project)
+ self.assertEqual(task.status, WikiImportTask.STATUS_RUNNING)
@mock.patch('addons.wiki.utils.check_file_object_in_node')
def test_project_wiki_import_error(self, mock_check_file_object_in_node):
@@ -1983,6 +2007,43 @@ def test_project_wiki_import_with_no_admin_permission(self):
res = self.app.post_json(url, { 'data': [{'test': 'test1'}] }, expect_errors=True)
assert_equal(res.status_code, 401)
+ @mock.patch('addons.wiki.views.AsyncResult')
+ def test_project_get_task_result_with_admin_permission(self, mock_async_result):
+ mock_res = mock.MagicMock()
+ mock_res.ready.return_value = True
+ mock_res.get.return_value = {'canStartImport': True, 'data': []}
+ mock_async_result.return_value = mock_res
+ WikiImportTask.objects.create(
+ node=self.project,
+ task_id='task-id',
+ status=WikiImportTask.STATUS_COMPLETED,
+ creator=self.user,
+ )
+ url = self.project.api_url_for('project_get_task_result', task_id='task-id')
+ res = self.app.get(url, auth=self.user.auth)
+ assert_equal(res.status_code, http_status.HTTP_200_OK)
+ assert_equal(res.json, {'canStartImport': True, 'data': []})
+
+ @mock.patch('addons.wiki.views.AsyncResult')
+ def test_project_get_task_result_rejects_task_for_other_node(self, mock_async_result):
+ other_project = ProjectFactory(creator=self.user)
+ WikiImportTask.objects.create(
+ node=other_project,
+ task_id='other-node-task-id',
+ status=WikiImportTask.STATUS_COMPLETED,
+ creator=self.user,
+ )
+ url = self.project.api_url_for('project_get_task_result', task_id='other-node-task-id')
+ res = self.app.get(url, auth=self.user.auth, expect_errors=True)
+ assert_equal(res.status_code, http_status.HTTP_404_NOT_FOUND)
+ mock_async_result.assert_not_called()
+
+ def test_project_get_task_result_rejects_unknown_task_id(self):
+ url = self.project.api_url_for('project_get_task_result', task_id='unknown-task-id')
+ res = self.app.get(url, auth=self.user.auth, expect_errors=True)
+ assert_equal(res.status_code, http_status.HTTP_404_NOT_FOUND)
+
+
@mock.patch('celery.contrib.abortable.AbortableAsyncResult')
def test_wiki_import_create_or_update_aborted(self, mock_task):
mock_task.is_aborted.return_value = True
@@ -3340,6 +3401,24 @@ def test_bulk_update_wiki_sort(self):
self.assertEqual(result_wiki_child_page2, {'parent_id': wiki_page2_id, 'sort_order': 1})
self.assertEqual(result_wiki_child_page3, {'parent_id': wiki_child_page2_id, 'sort_order': 1})
+ def test_bulk_update_wiki_sort_rejects_cross_node_parent(self):
+ other = ProjectFactory(is_public=True, creator=self.user)
+ other_page = WikiPage.objects.create_for_node(
+ other, 'other project wiki', 'content', self.consolidate_auth
+ )
+ other_guid = other_page.guids.first()._id
+ parent_before = self.wiki_page1.parent_id
+ sort_id_list = [self.guid1]
+ sort_num_list = [1]
+ parent_wiki_id_list = [other_guid]
+ with self.assertRaises(HTTPError) as cm:
+ views._bulk_update_wiki_sort(
+ self.project, sort_id_list, sort_num_list, parent_wiki_id_list
+ )
+ self.assertEqual(cm.exception.code, http_status.HTTP_400_BAD_REQUEST)
+ self.wiki_page1.reload()
+ self.assertEqual(self.wiki_page1.parent_id, parent_before)
+
def _wiki_sort_payload(self):
return {
'sortedData': [
@@ -3380,3 +3459,112 @@ def test_project_update_wiki_page_sort(self):
self.assertEqual(result_wiki_child_page1, {'parent_id': wiki_page2_id, 'sort_order': 1})
self.assertEqual(result_wiki_child_page2, {'parent_id': wiki_page2_id, 'sort_order': 2})
self.assertEqual(result_wiki_child_page3, {'parent_id': wiki_child_page2_id, 'sort_order': 1})
+
+ def test_project_update_wiki_page_sort_rejects_cross_node_parent(self):
+ other = ProjectFactory(is_public=True, creator=self.user)
+ other_page = WikiPage.objects.create_for_node(
+ other, 'other project wiki', 'content', self.consolidate_auth
+ )
+ other_guid = other_page.guids.first()._id
+ parent_before = self.wiki_page1.parent_id
+ url = self.project.api_url_for('project_update_wiki_page_sort')
+ # Nest a local wiki under another project's wiki id as parent.
+ payload = {
+ 'sortedData': [
+ {
+ 'name': 'other as parent',
+ 'id': other_guid,
+ 'sortOrder': 1,
+ 'children': [
+ {
+ 'name': 'wiki page1',
+ 'id': self.guid1,
+ 'sortOrder': 1,
+ 'children': [],
+ 'fold': False,
+ }
+ ],
+ 'fold': False,
+ }
+ ]
+ }
+ res = self.app.post_json(url, payload, auth=self.user.auth, expect_errors=True)
+ assert_equal(res.status_code, http_status.HTTP_400_BAD_REQUEST)
+ self.wiki_page1.reload()
+ self.assertEqual(self.wiki_page1.parent_id, parent_before)
+
+@pytest.mark.enable_bookmark_creation
+class TestYWebsocketToken(OsfTestCase):
+
+ def setUp(self):
+ super(TestYWebsocketToken, self).setUp()
+ # Class-level @mock.patch is unreliable under pytest here; patch in setUp
+ # so generate_y_websocket_token sees a non-empty secret during the request.
+ self._y_websocket_secret = 'test-y-websocket-secret'
+ self._secret_patcher = mock.patch(
+ 'addons.wiki.settings.Y_WEBSOCKET_SECRET',
+ self._y_websocket_secret,
+ )
+ self._secret_patcher.start()
+ self.addCleanup(self._secret_patcher.stop)
+
+ self.user = AuthUserFactory()
+ self.project = ProjectFactory(is_public=True, creator=self.user)
+ self.wname = 'foo.bar'
+ self.wkey = to_mongo_key(self.wname)
+
+ def test_token_generated_for_editor(self):
+ import jwt
+ from addons.wiki import settings as wiki_settings
+
+ url = self.project.web_url_for('project_wiki_view', wname=self.wname)
+ res = self.app.get(url, auth=self.user.auth)
+ assert_equal(res.status_code, 200)
+
+ self.project.reload()
+ body = res.body.decode()
+ sharejs_uuid = get_sharejs_uuid(self.project, self.wname)
+ assert_true(sharejs_uuid)
+ assert_in(sharejs_uuid, body)
+
+ # edit.mako renders `yWebsocketToken: "..."`, not JSON `"yWebsocketToken": "..."`.
+ token_match = re.search(r'yWebsocketToken:\s*"([^"]+)"', body)
+ assert_true(token_match)
+ token = token_match.group(1)
+ assert_true(token)
+
+ payload = jwt.decode(
+ token,
+ self._y_websocket_secret,
+ algorithms=[wiki_settings.Y_WEBSOCKET_JWT_ALGORITHM],
+ )
+ assert_equal(payload['doc_id'], sharejs_uuid)
+ assert_equal(payload['sub'], self.user._id)
+
+ def test_token_not_visible_without_write_permission(self):
+ WikiPage.objects.create_for_node(self.project, self.wname, 'some content', Auth(self.user))
+
+ url = self.project.web_url_for('project_wiki_view', wname=self.wname)
+ # Generate sharejs uuid / token path as an editor first (same pattern as TestWikiUuid).
+ res = self.app.get(url, auth=self.user.auth)
+ assert_equal(res.status_code, 200)
+ self.project.reload()
+ sharejs_uuid = get_sharejs_uuid(self.project, self.wname)
+ assert_true(sharejs_uuid)
+
+ # Users without write permission should not receive uuid or a non-empty token.
+ res = self.app.get(url)
+ assert_equal(res.status_code, 200)
+ body = res.body.decode()
+ assert_not_in(sharejs_uuid, body)
+ token_match = re.search(r'yWebsocketToken:\s*"([^"]+)"', body)
+ assert_false(token_match)
+
+ def test_generate_y_websocket_token_without_secret(self):
+ with mock.patch('addons.wiki.settings.Y_WEBSOCKET_SECRET', ''):
+ assert_equal(generate_y_websocket_token('doc-id', 'user-id'), '')
+
+ def test_generate_y_websocket_token_without_user_id(self):
+ assert_equal(generate_y_websocket_token('doc-id', ''), '')
+ assert_equal(generate_y_websocket_token('doc-id', None), '')
+
diff --git a/addons/wiki/tests/test_wiki_import.py b/addons/wiki/tests/test_wiki_import.py
index 73fe2c3a941..b57bf6545c4 100644
--- a/addons/wiki/tests/test_wiki_import.py
+++ b/addons/wiki/tests/test_wiki_import.py
@@ -1245,11 +1245,15 @@ def test_project_wiki_validate_for_import(self, mock_check_file_object_in_node):
mock_check_file_object_in_node.return_value = True
dir_id = self.root_import_folder1._id
url = self.project.api_url_for('project_wiki_validate_for_import', dir_id=dir_id)
- res = self.app.get(url)
+ res = self.app.get(url, auth=self.user.auth)
response_json = res.json
task_id = response_json['taskId']
uuid_obj = uuid.UUID(task_id)
assert uuid_obj
+ task = WikiImportTask.objects.get(task_id=task_id)
+ assert_equal(task.node, self.project)
+ assert_equal(task.status, WikiImportTask.STATUS_COMPLETED)
+ assert_is_not_none(task.process_end)
def test_project_wiki_validate_for_import_process(self):
result = views.project_wiki_validate_for_import_process(
@@ -1323,6 +1327,9 @@ def test_project_wiki_import(self, mock_check_file_object_in_node, mock_project_
task_id = response_json['taskId']
uuid_obj = uuid.UUID(task_id)
assert_is_not_none(uuid_obj)
+ task = WikiImportTask.objects.get(task_id=task_id)
+ assert_equal(task.node, self.project)
+ assert_equal(task.status, WikiImportTask.STATUS_RUNNING)
@mock.patch('addons.wiki.views._get_md_content_from_wb')
@mock.patch('addons.wiki.views._get_or_create_wiki_folder')
diff --git a/addons/wiki/utils.py b/addons/wiki/utils.py
index 39af0660fb0..947cf31b834 100644
--- a/addons/wiki/utils.py
+++ b/addons/wiki/utils.py
@@ -4,10 +4,13 @@
import uuid
import unicodedata
import ssl
+import datetime
from future.moves.urllib.parse import quote
+import jwt
from pymongo import MongoClient
import requests
+from django.utils import timezone
from bs4 import BeautifulSoup
from django.apps import apps
@@ -65,6 +68,32 @@ def get_sharejs_uuid(node, wname):
)) if private_uuid else None
+def generate_y_websocket_token(doc_id, user_id):
+ """
+ Generate a signed JWT for y-websocket connection authorization.
+
+ Includes ``sub`` (OSF user GUID) for connection traceability on the
+ y-websocket side. Returns an empty string when Y_WEBSOCKET_SECRET is not
+ configured or required claims are missing.
+ """
+ secret = wiki_settings.Y_WEBSOCKET_SECRET
+ if not secret or not doc_id or not user_id:
+ return ''
+
+ payload = {
+ 'doc_id': doc_id,
+ 'sub': user_id,
+ 'exp': timezone.now() + datetime.timedelta(seconds=wiki_settings.Y_WEBSOCKET_TOKEN_TTL),
+ }
+ token = jwt.encode(
+ payload,
+ secret,
+ algorithm=wiki_settings.Y_WEBSOCKET_JWT_ALGORITHM,
+ )
+ if isinstance(token, bytes):
+ return token.decode()
+ return token
+
def delete_share_doc(node, wname):
"""Deletes share document and removes namespace from model."""
diff --git a/addons/wiki/views.py b/addons/wiki/views.py
index a1051ba9110..bcf8b905095 100644
--- a/addons/wiki/views.py
+++ b/addons/wiki/views.py
@@ -93,6 +93,10 @@
message_short='Invalid request',
message_long='The requested version of this wiki page does not exist.'
))
+WIKI_INVALID_PARENT_ERROR = HTTPError(http_status.HTTP_400_BAD_REQUEST, data=dict(
+ message_short='Invalid parent wiki',
+ message_long='Parent wiki page does not exist on this project.',
+))
WIKI_IMPORT_TASK_ALREADY_EXISTS = HTTPError(http_status.HTTP_400_BAD_REQUEST, data=dict(
message_short='Running Task exists',
@@ -325,10 +329,15 @@ def project_wiki_view(auth, wname, path=None, **kwargs):
rendered_before_update = False
markdown = ''
+ y_websocket_token = ''
if can_edit:
if wiki_key not in node.wiki_private_uuids:
wiki_utils.generate_private_uuid(node, wiki_name)
sharejs_uuid = wiki_utils.get_sharejs_uuid(node, wiki_name)
+ y_websocket_token = wiki_utils.generate_y_websocket_token(
+ sharejs_uuid,
+ auth.user._id,
+ )
else:
if not wiki_page and wiki_key != 'home':
raise WIKI_PAGE_NOT_FOUND_ERROR
@@ -366,6 +375,7 @@ def project_wiki_view(auth, wname, path=None, **kwargs):
'sharejs_uuid': sharejs_uuid or '',
'sharejs_url': settings.SHAREJS_URL,
'y_websocket_url': settings.Y_WEBSOCKET_URL,
+ 'y_websocket_token': y_websocket_token,
'is_current': is_current,
'version_settings': version_settings,
'pages_current': _get_wiki_pages_latest(node),
@@ -771,12 +781,40 @@ def serialize_component_wiki(node, auth):
return component
return None
+
+def _bind_wiki_task_to_node(task_id, node, auth, status, set_process_end=False):
+ """Record task_id as belonging to node for get_task_result authorization.
+
+ Validate tasks must not use STATUS_RUNNING so they do not affect
+ "importing" UI, abort, or concurrent-import checks.
+ """
+ defaults = {
+ 'node': node,
+ 'status': status,
+ 'creator': auth.user if auth else None,
+ }
+ if set_process_end:
+ defaults['process_end'] = timezone.make_naive(timezone.now(), timezone.utc)
+ WikiImportTask.objects.get_or_create(task_id=task_id, defaults=defaults)
+
+
+def _wiki_task_belongs_to_node(task_id, node):
+ return WikiImportTask.objects.filter(task_id=task_id, node=node).exists()
+
+
@must_be_valid_project
-def project_wiki_validate_for_import(dir_id, node, **kwargs):
+@must_have_permission(ADMIN)
+@must_not_be_registration
+@must_have_addon('wiki', 'node')
+def project_wiki_validate_for_import(dir_id, auth, node, **kwargs):
wiki_utils.check_file_object_in_node(dir_id, node)
node_id = node.guids.first()._id
task = tasks.run_project_wiki_validate_for_import.delay(dir_id, node_id)
task_id = task.id
+ # Completed + process_end: ownership only; not treated as a running import.
+ _bind_wiki_task_to_node(
+ task_id, node, auth, WikiImportTask.STATUS_COMPLETED, set_process_end=True
+ )
return {'taskId': task_id}
def project_wiki_validate_for_import_process(dir_id, node):
@@ -904,11 +942,15 @@ def project_wiki_import(dir_id, auth, node, **kwargs):
data_json = json.dumps(data)
task = tasks.run_project_wiki_import.delay(data_json, dir_id, current_user_id, node_id)
task_id = task.id
+ # Bind before returning so polling get_task_result does not race the worker.
+ _bind_wiki_task_to_node(task_id, node, auth, WikiImportTask.STATUS_RUNNING)
+ check_running_task(task_id, node)
return {'taskId': task_id}
def project_wiki_import_process(data, dir_id, task_id, auth, node):
logger.info('----WIKI IMPORT DIRECTORY_ID: {}, PROJECT_NAME: {} ----'.format(dir_id, node.title))
- WikiImportTask.objects.create(node=node, task_id=task_id, status=WikiImportTask.STATUS_RUNNING, creator=auth.user)
+ # May already exist if project_wiki_import bound the task before the worker ran.
+ _bind_wiki_task_to_node(task_id, node, auth, WikiImportTask.STATUS_RUNNING)
check_running_task(task_id, node)
ret = []
wiki_id_list = []
@@ -1306,8 +1348,15 @@ def _create_import_error_list(wiki_infos, imported_list):
return import_errors
@must_be_valid_project
+@must_have_permission(ADMIN)
+@must_not_be_registration
@must_have_addon('wiki', 'node')
def project_get_task_result(task_id, node, **kwargs):
+ if not _wiki_task_belongs_to_node(task_id, node):
+ raise HTTPError(http_status.HTTP_404_NOT_FOUND, data=dict(
+ message_short='Not found',
+ message_long='Task not found.',
+ ))
res = AsyncResult(task_id, app=celery_app)
result = None
if not res.ready():
@@ -1408,7 +1457,10 @@ def _bulk_update_wiki_sort(node, sort_id_list, sort_num_list, parent_wiki_id_lis
parent_wiki_id = parent_wiki_id_list[idx]
setattr(page, 'sort_order', sort_order_number)
if parent_wiki_id is not None:
- parent = WikiPage.objects.get(guids___id=parent_wiki_id)
+ # Same-node only; global guid lookup would allow cross-project parents.
+ parent = WikiPage.objects.get_for_node(node, id=parent_wiki_id)
+ if parent is None:
+ raise WIKI_INVALID_PARENT_ERROR
setattr(page, 'parent', parent)
else:
setattr(page, 'parent', None)
diff --git a/admin/base/urls.py b/admin/base/urls.py
index c63c9d20fbd..d201a3e5b38 100644
--- a/admin/base/urls.py
+++ b/admin/base/urls.py
@@ -1,62 +1,84 @@
from django.conf.urls import include, url
from django.contrib import admin
-from admin.base.settings import ADMIN_BASE, DEBUG
+from django.http import Http404
+from admin.base.settings import ADMIN_BASE, DEBUG, ENABLE_LOGIN_FORM
from admin.base import views
base_pattern = '^{}'.format(ADMIN_BASE)
+
+def admin_login_disabled(request):
+ # admin.site.urls provides its own password-based login (AdminSite.login),
+ # which must not exist while password-based login is disabled.
+ raise Http404
+
+admin_urls = [
+ url(r'^$', views.home, name='home'),
+]
+
+if not ENABLE_LOGIN_FORM:
+ # must be registered before admin.site.urls, otherwise ^admin/ matches first
+ admin_urls.append(url(r'^admin/login/$', admin_login_disabled, name='admin_login_disabled'))
+
+admin_urls += [
+ url(r'^admin/', admin.site.urls),
+ url(r'^asset_files/', include('admin.asset_files.urls', namespace='asset_files')),
+ url(r'^banners/', include('admin.banners.urls', namespace='banners')),
+ url(r'^brands/', include('admin.brands.urls', namespace='brands')),
+ url(r'^spam/', include('admin.spam.urls', namespace='spam')),
+ url(r'^institutions/', include('admin.institutions.urls', namespace='institutions')),
+ url(r'^entitlements/', include('admin.entitlements.urls', namespace='entitlements')),
+ url(r'^quota_recalc/', include('admin.quota_recalc.urls', namespace='quota_recalc')),
+ url(r'^preprint_providers/', include('admin.preprint_providers.urls', namespace='preprint_providers')),
+ url(r'^collection_providers/', include('admin.collection_providers.urls', namespace='collection_providers')),
+ url(r'^registration_providers/', include('admin.registration_providers.urls', namespace='registration_providers')),
+ url(r'^account/', include('admin.common_auth.urls', namespace='auth')),
+]
+
+if ENABLE_LOGIN_FORM:
+ # password recovery/reset only makes sense while password-based login itself is enabled
+ admin_urls.append(url(r'^password/', include('password_reset.urls')))
+
+admin_urls += [
+ url(r'^nodes/', include('admin.nodes.urls', namespace='nodes')),
+ url(r'^preprints/', include('admin.preprints.urls', namespace='preprints')),
+ url(r'^subjects/', include('admin.subjects.urls', namespace='subjects')),
+ url(r'^users/', include('admin.users.urls', namespace='users')),
+ url(r'^user-emails/', include('admin.user_emails.urls', namespace='user-emails')),
+ url(r'^maintenance/', include('admin.maintenance.urls', namespace='maintenance')),
+ url(r'^meetings/', include('admin.meetings.urls',
+ namespace='meetings')),
+ url(r'^metrics/', include('admin.metrics.urls',
+ namespace='metrics')),
+ url(r'^desk/', include('admin.desk.urls',
+ namespace='desk')),
+ url(r'^osf_groups/', include('admin.osf_groups.urls', namespace='osf_groups')),
+ url(r'^management/', include('admin.management.urls', namespace='management')),
+ url(r'^announcement/', include('admin.rdm_announcement.urls', namespace='announcement')),
+ url(r'^addons/', include('admin.rdm_addons.urls', namespace='addons')),
+ url(r'^oauth/', include('admin.rdm_addons.oauth.urls', namespace='oauth')),
+ url(r'^statistics/', include('admin.rdm_statistics.urls', namespace='statistics')),
+ url(r'^timestampadd/', include('admin.rdm_timestampadd.urls', namespace='timestampadd')),
+ url(r'^keymanagement/', include('admin.rdm_keymanagement.urls', namespace='keymanagement')),
+ url(r'^timestampsettings/', include('admin.rdm_timestampsettings.urls', namespace='timestampsettings')),
+ url(r'^custom_storage_location/', include('admin.rdm_custom_storage_location.urls', namespace='custom_storage_location')),
+ url(r'^institutional_storage_quota_control/', include('admin.institutional_storage_quota_control.urls',
+ namespace='institutional_storage_quota_control')),
+ url(r'^metadata/', include('admin.rdm_metadata.urls', namespace='metadata')),
+ url(r'^user_identification_information/',
+ include('admin.user_identification_information.urls', namespace='user_identification_information')),
+ url(r'^user_identification_information_admin/',
+ include('admin.user_identification_information_admin.urls', namespace='user_identification_information_admin')),
+ url(r'^project_limit_number/', include('admin.project_limit_number.urls', namespace='project_limit_number')),
+ url(r'^rdm_workflow/', include('admin.rdm_workflow.urls', namespace='rdm_workflow')),
+ url(r'^loa/', include('admin.loa.urls', namespace='loa')),
+]
+
urlpatterns = [
### ADMIN ###
url(
base_pattern,
- include([
- url(r'^$', views.home, name='home'),
- url(r'^admin/', admin.site.urls),
- url(r'^asset_files/', include('admin.asset_files.urls', namespace='asset_files')),
- url(r'^banners/', include('admin.banners.urls', namespace='banners')),
- url(r'^brands/', include('admin.brands.urls', namespace='brands')),
- url(r'^spam/', include('admin.spam.urls', namespace='spam')),
- url(r'^institutions/', include('admin.institutions.urls', namespace='institutions')),
- url(r'^entitlements/', include('admin.entitlements.urls', namespace='entitlements')),
- url(r'^quota_recalc/', include('admin.quota_recalc.urls', namespace='quota_recalc')),
- url(r'^preprint_providers/', include('admin.preprint_providers.urls', namespace='preprint_providers')),
- url(r'^collection_providers/', include('admin.collection_providers.urls', namespace='collection_providers')),
- url(r'^registration_providers/', include('admin.registration_providers.urls', namespace='registration_providers')),
- url(r'^account/', include('admin.common_auth.urls', namespace='auth')),
- url(r'^password/', include('password_reset.urls')),
- url(r'^nodes/', include('admin.nodes.urls', namespace='nodes')),
- url(r'^preprints/', include('admin.preprints.urls', namespace='preprints')),
- url(r'^subjects/', include('admin.subjects.urls', namespace='subjects')),
- url(r'^users/', include('admin.users.urls', namespace='users')),
- url(r'^user-emails/', include('admin.user_emails.urls', namespace='user-emails')),
- url(r'^maintenance/', include('admin.maintenance.urls', namespace='maintenance')),
- url(r'^meetings/', include('admin.meetings.urls',
- namespace='meetings')),
- url(r'^metrics/', include('admin.metrics.urls',
- namespace='metrics')),
- url(r'^desk/', include('admin.desk.urls',
- namespace='desk')),
- url(r'^osf_groups/', include('admin.osf_groups.urls', namespace='osf_groups')),
- url(r'^management/', include('admin.management.urls', namespace='management')),
- url(r'^announcement/', include('admin.rdm_announcement.urls', namespace='announcement')),
- url(r'^addons/', include('admin.rdm_addons.urls', namespace='addons')),
- url(r'^oauth/', include('admin.rdm_addons.oauth.urls', namespace='oauth')),
- url(r'^statistics/', include('admin.rdm_statistics.urls', namespace='statistics')),
- url(r'^timestampadd/', include('admin.rdm_timestampadd.urls', namespace='timestampadd')),
- url(r'^keymanagement/', include('admin.rdm_keymanagement.urls', namespace='keymanagement')),
- url(r'^timestampsettings/', include('admin.rdm_timestampsettings.urls', namespace='timestampsettings')),
- url(r'^custom_storage_location/', include('admin.rdm_custom_storage_location.urls', namespace='custom_storage_location')),
- url(r'^institutional_storage_quota_control/', include('admin.institutional_storage_quota_control.urls',
- namespace='institutional_storage_quota_control')),
- url(r'^metadata/', include('admin.rdm_metadata.urls', namespace='metadata')),
- url(r'^user_identification_information/',
- include('admin.user_identification_information.urls', namespace='user_identification_information')),
- url(r'^user_identification_information_admin/',
- include('admin.user_identification_information_admin.urls', namespace='user_identification_information_admin')),
- url(r'^project_limit_number/', include('admin.project_limit_number.urls', namespace='project_limit_number')),
- url(r'^rdm_workflow/', include('admin.rdm_workflow.urls', namespace='rdm_workflow')),
- url(r'^loa/', include('admin.loa.urls', namespace='loa')),
- ]),
+ include(admin_urls),
),
]
diff --git a/admin/base/utils.py b/admin/base/utils.py
index 6ee8780be71..ff86f9f7900 100644
--- a/admin/base/utils.py
+++ b/admin/base/utils.py
@@ -1,6 +1,8 @@
"""
Utility functions and classes
"""
+from functools import wraps
+
from osf.models import Subject, NodeLicense, Brand
from django.core.exceptions import ValidationError, PermissionDenied
@@ -35,6 +37,29 @@ def osf_staff_check(user):
return user.is_authenticated and user.is_staff
+def osf_superuser_check(user):
+ """An inactive or unregistered superuser is not an Integrated Admin."""
+ return bool(
+ user.is_authenticated and
+ user.is_active and
+ user.is_registered and
+ user.is_superuser
+ )
+
+
+def superuser_required(view_func):
+ """Function-view decorator that returns 403 (PermissionDenied) instead of
+ redirecting to login, unlike django.contrib.auth.decorators.user_passes_test."""
+
+ @wraps(view_func)
+ def wrapped_view(request, *args, **kwargs):
+ if not osf_superuser_check(request.user):
+ raise PermissionDenied
+ return view_func(request, *args, **kwargs)
+
+ return wrapped_view
+
+
def get_subject_rules(subjects_selected):
"""
Take a list of subjects, and parse them into rules consistent with preprpint provider
diff --git a/admin/collection_providers/views.py b/admin/collection_providers/views.py
index cafb9476f9e..4cfd366d990 100644
--- a/admin/collection_providers/views.py
+++ b/admin/collection_providers/views.py
@@ -4,7 +4,7 @@
from django.core import serializers
from django.core.exceptions import ValidationError
from django.urls import reverse_lazy
-from django.shortcuts import redirect
+from django.shortcuts import redirect, get_object_or_404
from django.views.generic import View, CreateView, ListView, DetailView, UpdateView, DeleteView, TemplateView
from django.contrib import messages
from django.contrib.auth.mixins import PermissionRequiredMixin
@@ -228,12 +228,14 @@ def get_object(self, queryset=None):
return CollectionProvider.objects.get(id=self.kwargs['collection_provider_id'])
-class CannotDeleteProvider(TemplateView):
+class CannotDeleteProvider(PermissionRequiredMixin, TemplateView):
+ permission_required = 'osf.delete_collectionprovider'
+ raise_exception = True
template_name = 'collection_providers/cannot_delete.html'
def get_context_data(self, **kwargs):
context = super(CannotDeleteProvider, self).get_context_data(**kwargs)
- context['provider'] = CollectionProvider.objects.get(id=self.kwargs['collection_provider_id'])
+ context['provider'] = get_object_or_404(CollectionProvider, id=self.kwargs['collection_provider_id'])
return context
diff --git a/admin/common_auth/views.py b/admin/common_auth/views.py
index d2dc3993090..dd03b30dedb 100644
--- a/admin/common_auth/views.py
+++ b/admin/common_auth/views.py
@@ -39,10 +39,12 @@ def dispatch(self, request, *args, **kwargs):
return super(LoginView, self).dispatch(request, *args, **kwargs)
def form_valid(self, form):
- user = authenticate(
- username=form.cleaned_data.get('email').strip(),
- password=form.cleaned_data.get('password').strip()
- )
+ user = None
+ if ENABLE_LOGIN_FORM:
+ user = authenticate(
+ username=form.cleaned_data.get('email').strip(),
+ password=form.cleaned_data.get('password').strip()
+ )
if user is not None:
login(self.request, user)
else:
diff --git a/admin/institutional_storage_quota_control/views.py b/admin/institutional_storage_quota_control/views.py
index 7b028da5986..725d464053d 100644
--- a/admin/institutional_storage_quota_control/views.py
+++ b/admin/institutional_storage_quota_control/views.py
@@ -1,16 +1,19 @@
+import logging
from django.contrib.auth.mixins import UserPassesTestMixin
from django.db import connection, transaction, IntegrityError
from django.db.models import Subquery, OuterRef
from django.http import Http404
-
from admin.institutions.views import QuotaUserList
-from osf.models import Institution, OSFUser, UserQuota
+from osf.models import Institution, OSFUser, UserQuota, InstitutionDefaultMaxQuota
from admin.base import settings
from addons.osfstorage.models import Region
from django.views.generic import ListView, View
from django.shortcuts import redirect
from admin.rdm.utils import RdmPermissionMixin
from django.core.urlresolvers import reverse
+from api.base import settings as api_settings
+
+logger = logging.getLogger(__name__)
class InstitutionStorageList(RdmPermissionMixin, UserPassesTestMixin, ListView):
@@ -122,6 +125,22 @@ def get_institution(self):
raise Http404
return institution
+ def get_default_max_quota(self):
+ """Get default max quota for the institution, fallback to DEFAULT_MAX_QUOTA when not found."""
+ try:
+ institution_default_max_quota = InstitutionDefaultMaxQuota.objects.get(
+ institution_id=self.institution_id
+ )
+ return institution_default_max_quota.default_max_quota
+ except InstitutionDefaultMaxQuota.DoesNotExist:
+ return api_settings.DEFAULT_MAX_QUOTA
+
+ def get_context_data(self, **kwargs):
+ """ Add default_max_quota to template context """
+ context = super().get_context_data(**kwargs)
+ context['default_max_quota'] = self.get_default_max_quota()
+ return context
+
class UpdateQuotaUserListByInstitutionStorageID(RdmPermissionMixin, UserPassesTestMixin, View):
""" Change max quota for an institution's users if that institution is not using NII Storage. """
@@ -163,17 +182,23 @@ def post(self, request, *args, **kwargs):
min_value, max_value = connection.ops.integer_field_range('PositiveIntegerField')
if min_value <= max_quota <= max_value:
# If max quota value is between 0 and 2147483647, update or create used quota for each user in the institution
- for user in OSFUser.objects.filter(
- affiliated_institutions=self.institution_id):
- try:
- with transaction.atomic():
- UserQuota.objects.update_or_create(
- user=user,
- storage_type=UserQuota.CUSTOM_STORAGE,
- defaults={'max_quota': max_quota}
- )
- except IntegrityError:
- UserQuota.objects.filter(user=user, storage_type=UserQuota.CUSTOM_STORAGE).update(max_quota=max_quota)
+ with transaction.atomic():
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution_id,
+ defaults={'default_max_quota': max_quota}
+ )
+ for user in OSFUser.objects.filter(
+ affiliated_institutions=self.institution_id):
+ try:
+ with transaction.atomic():
+ UserQuota.objects.update_or_create(
+ user=user,
+ storage_type=UserQuota.CUSTOM_STORAGE,
+ defaults={'max_quota': max_quota}
+ )
+ except IntegrityError as e:
+ logger.warning(u'IntegrityError while updating UserQuota: user={}, storage_type={}, max_quota={}: {}.'.format(user.id, UserQuota.CUSTOM_STORAGE, max_quota, str(e)))
+ UserQuota.objects.filter(user=user, storage_type=UserQuota.CUSTOM_STORAGE).update(max_quota=max_quota)
return redirect(
'institutional_storage_quota_control:institution_user_list',
institution_id=self.institution_id
diff --git a/admin/management/views.py b/admin/management/views.py
index fffd1bbef4f..b1522991ad2 100644
--- a/admin/management/views.py
+++ b/admin/management/views.py
@@ -1,15 +1,29 @@
+from django.contrib.auth.mixins import UserPassesTestMixin
from django.views.generic import TemplateView, View
from osf.management.commands.manage_switch_flags import manage_waffle
from django.core.urlresolvers import reverse
from django.shortcuts import redirect
-class ManagementCommands(TemplateView):
+from admin.rdm.utils import RdmPermissionMixin
+
+class ManagementCommands(RdmPermissionMixin, UserPassesTestMixin, TemplateView):
""" Basic form to trigger various management commands
"""
template_name = 'management/commands.html'
object_type = 'management'
+ raise_exception = True
+
+ def test_func(self):
+ """check user permissions"""
+ return self.is_super_admin
+
+
+class WaffleFlag(RdmPermissionMixin, UserPassesTestMixin, View):
+ raise_exception = True
-class WaffleFlag(View):
+ def test_func(self):
+ """check user permissions"""
+ return self.is_super_admin
def post(self, request, *args, **kwargs):
manage_waffle()
diff --git a/admin/preprint_providers/views.py b/admin/preprint_providers/views.py
index 159a0a409f5..8d2ecbf6d29 100644
--- a/admin/preprint_providers/views.py
+++ b/admin/preprint_providers/views.py
@@ -14,7 +14,7 @@
from django.contrib import messages
from django.contrib.auth.mixins import PermissionRequiredMixin
from django.forms.models import model_to_dict
-from django.shortcuts import redirect, render
+from django.shortcuts import redirect, render, get_object_or_404
from admin.base import settings
from admin.base.forms import ImportFileForm
@@ -319,12 +319,14 @@ def get_object(self, queryset=None):
return PreprintProvider.objects.get(id=self.kwargs['preprint_provider_id'])
-class CannotDeleteProvider(TemplateView):
+class CannotDeleteProvider(PermissionRequiredMixin, TemplateView):
+ permission_required = 'osf.delete_preprintprovider'
+ raise_exception = True
template_name = 'preprint_providers/cannot_delete.html'
def get_context_data(self, **kwargs):
context = super(CannotDeleteProvider, self).get_context_data(**kwargs)
- context['provider'] = PreprintProvider.objects.get(id=self.kwargs['preprint_provider_id'])
+ context['provider'] = get_object_or_404(PreprintProvider, id=self.kwargs['preprint_provider_id'])
return context
diff --git a/admin/quota_recalc/views.py b/admin/quota_recalc/views.py
index 29484c3d559..0ef1d740788 100644
--- a/admin/quota_recalc/views.py
+++ b/admin/quota_recalc/views.py
@@ -1,12 +1,15 @@
+import logging
from django.http import JsonResponse
from django.db import transaction, IntegrityError
+from admin.base.utils import superuser_required
from addons.osfstorage.models import Region
-from api.base import settings as api_settings
from osf.models import OSFUser, UserQuota, Node
from osf.models.node import set_project_storage_type
from osf.utils.requests import check_select_for_update
-from website.util.quota import used_quota
+from website.util.quota import get_default_max_quota, used_quota
+
+logger = logging.getLogger(__name__)
def calculate_quota(user):
@@ -23,6 +26,7 @@ def calculate_quota(user):
with transaction.atomic():
for storage_type in storage_type_list:
used = used_quota(user._id, storage_type)
+ max_quota = get_default_max_quota(user, storage_type)
try:
if check_select_for_update():
user_quota = UserQuota.objects.filter(
@@ -42,10 +46,11 @@ def calculate_quota(user):
UserQuota.objects.create(
user=user,
storage_type=storage_type,
- max_quota=api_settings.DEFAULT_MAX_QUOTA,
+ max_quota=max_quota,
used=used,
)
- except IntegrityError:
+ except IntegrityError as e:
+ logger.warning(u'IntegrityError while creating UserQuota in calculate_quota: user={}, storage_type={}: {}.'.format(user.id, storage_type, str(e)))
used = used_quota(user._id, storage_type)
if check_select_for_update():
user_quota = UserQuota.objects.filter(
@@ -60,6 +65,7 @@ def calculate_quota(user):
user_quota.used = used
user_quota.save()
+@superuser_required
def all_users(request, **kwargs):
c = 0
for osf_user in OSFUser.objects.exclude(deleted__isnull=False):
@@ -70,6 +76,7 @@ def all_users(request, **kwargs):
'message': str(c) + ' users\' quota successfully recalculated!'
})
+@superuser_required
def user(request, guid, **kwargs):
user = OSFUser.load(guid)
if user is None:
diff --git a/admin/rdm_custom_storage_location/cron.sh b/admin/rdm_custom_storage_location/cron.sh
index 10cb90c4250..81c65448528 100644
--- a/admin/rdm_custom_storage_location/cron.sh
+++ b/admin/rdm_custom_storage_location/cron.sh
@@ -1 +1 @@
-*/30 * * * * curl http://localhost:8001/custom_storage_location/external_acc_update/d610ef95f0b0f5868f13919b8ed64070b9acb9c19b8da9f2c514ed938203ec3e236c9cad4f4146bdf22b4e79cf0d92f6d4f4c996d236c6b0ee79a1336b26afb7/ | jq -c .
+*/30 * * * * TOKEN=$(cd /code && python3 manage.py generate_signed_cron_token | tail -n1) && [ -n "$TOKEN" ] && curl "http://localhost:8001/custom_storage_location/external_acc_update/$TOKEN/" | jq -c .
diff --git a/admin/rdm_custom_storage_location/urls.py b/admin/rdm_custom_storage_location/urls.py
index 6a62d690294..c51f34c148e 100644
--- a/admin/rdm_custom_storage_location/urls.py
+++ b/admin/rdm_custom_storage_location/urls.py
@@ -3,7 +3,7 @@
urlpatterns = [
- url(r'^external_acc_update/(?P
-?\w+)/$', views.external_acc_update, name='external_acc_update'),
+ url(r'^external_acc_update/(?P\d+)/(?P[0-9a-fA-F]+)/$', views.external_acc_update, name='external_acc_update'),
url(r'^institutional_storage/$', views.InstitutionalStorageView.as_view(), name='institutional_storage'),
url(r'^institutional_storage/institutions/$', views.InstitutionalStorageListView.as_view(), name='institutional_storage_institutions'),
url(r'^institutional_storage/institutions/(?P[0-9]+)/$', views.InstitutionalStorageView.as_view(), name='institutional_storage_list'),
diff --git a/admin/rdm_custom_storage_location/utils.py b/admin/rdm_custom_storage_location/utils.py
index df350ed2ae1..9959cda827d 100644
--- a/admin/rdm_custom_storage_location/utils.py
+++ b/admin/rdm_custom_storage_location/utils.py
@@ -13,6 +13,7 @@
import os
import owncloud
from django.core.exceptions import ValidationError
+from django.db import IntegrityError, transaction
from addons.dropboxbusiness.models import node_post_save as dropboxbusiness_post_save
from addons.onedrivebusiness.models import node_post_save as onedrivebusiness_post_save
@@ -45,7 +46,8 @@
from framework.exceptions import HTTPError
from osf.models import AbstractNode
from website import settings as osf_settings
-from osf.models import Node, OSFUser, ProjectStorageType, UserQuota
+from osf.models import Node, OSFUser, ProjectStorageType, UserQuota, InstitutionDefaultMaxQuota
+from api.base import settings as api_settings
from osf.models.external import ExternalAccountTemporary, ExternalAccount
from osf.utils import external_util
import datetime
@@ -261,7 +263,10 @@ def test_s3_connection(access_key, secret_key, bucket):
s3_response = {
'id': user_info.id,
'display_name': user_info.display_name,
- 'Owner': user_info.Owner,
+ 'Owner': {
+ 'ID': user_info.id,
+ 'DisplayName': user_info.display_name,
+ },
}
return ({
@@ -1387,3 +1392,51 @@ def add_node_settings_to_projects(institution, provider_name):
onedrivebusiness_post_save(None, project, created=project_has_no_node_settings)
else:
node_post_save(None, project, created=project_has_no_node_settings)
+
+def is_institution_using_nii_storage(institution):
+ """Return True if the institution is currently using NII storage (or has no region configured)."""
+ old_region = Region.objects.filter(_id=institution._id).first()
+ return (
+ old_region is None or
+ old_region.waterbutler_settings.get('storage', {}).get('type') == Region.NII_STORAGE
+ )
+
+
+def upsert_user_quota_for_institution(institution, provider_short_name, old_is_nii, old_quota_type=None):
+ """Determine quota migration params and upsert UserQuota for all users affiliated with the institution."""
+ new_quota_type = institution.get_user_quota_type_for_nii_storage()
+ users = OSFUser.objects.filter(affiliated_institutions=institution.id)
+
+ if provider_short_name == 'osfstorage' and not old_is_nii:
+ # 機関ストレージ → NII Storage: reset all users to default quota
+ storage_type = new_quota_type
+ quota_map = {}
+ elif provider_short_name != 'osfstorage' and old_is_nii:
+ # NII Storage → 機関ストレージ: set all users to institution default quota
+ default_max_quota = InstitutionDefaultMaxQuota.objects.filter(institution=institution).values_list('default_max_quota', flat=True).first()
+ fixed_quota = default_max_quota if default_max_quota is not None else api_settings.DEFAULT_MAX_QUOTA
+ storage_type = UserQuota.CUSTOM_STORAGE
+ quota_map = {user.id: fixed_quota for user in users}
+ elif old_quota_type is not None and old_quota_type != new_quota_type and new_quota_type is not None:
+ # NII Storage quota type changed (e.g. no-region → has-region): preserve each user's existing quota
+ storage_type = new_quota_type
+ quota_map = {
+ q.user_id: q.max_quota
+ for q in UserQuota.objects.filter(user__in=users, storage_type=old_quota_type)
+ }
+ else:
+ return
+
+ with transaction.atomic():
+ for user in users:
+ max_quota = quota_map.get(user.id, api_settings.DEFAULT_MAX_QUOTA)
+ try:
+ with transaction.atomic():
+ UserQuota.objects.update_or_create(
+ user=user,
+ storage_type=storage_type,
+ defaults={'max_quota': max_quota}
+ )
+ except IntegrityError as e:
+ logger.warning(u'IntegrityError while updating UserQuota: user={}, storage_type={}, max_quota={}: {}.'.format(user.id, storage_type, max_quota, str(e)))
+ UserQuota.objects.filter(user=user, storage_type=storage_type).update(max_quota=max_quota)
diff --git a/admin/rdm_custom_storage_location/views.py b/admin/rdm_custom_storage_location/views.py
index dff08d2b3ec..0837a5c1085 100644
--- a/admin/rdm_custom_storage_location/views.py
+++ b/admin/rdm_custom_storage_location/views.py
@@ -5,7 +5,6 @@
from django.views.generic import TemplateView, View, ListView
import codecs
import json
-import hashlib
from rest_framework import status as http_status
from mimetypes import MimeTypes
import os
@@ -19,13 +18,13 @@
from osf.models import Institution, OSFUser
from osf.models.external import ExternalAccountTemporary
from scripts import refresh_addon_tokens
+from framework.auth import cron_signed_url
from website import settings as osf_settings
from distutils.util import strtobool
from admin.base import settings
logger = logging.getLogger(__name__)
-SITE_KEY = 'rdm_custom_storage_location'
INSTITUTION_NOT_FOUND_MESSAGE = 'Institution does not exist'
class InstitutionalStorageBaseView(RdmPermissionMixin, UserPassesTestMixin):
@@ -291,6 +290,9 @@ def post(self, request, *args, **kwargs):
'message': 'Storage name is missing.'
}, status=http_status.HTTP_400_BAD_REQUEST)
+ old_is_nii = utils.is_institution_using_nii_storage(institution)
+ old_quota_type = institution.get_user_quota_type_for_nii_storage()
+
result = None
if provider_short_name == 's3':
@@ -436,6 +438,7 @@ def post(self, request, *args, **kwargs):
utils.change_allowed_for_institutions(
institution, provider_short_name)
utils.add_node_settings_to_projects(institution, provider_short_name)
+ utils.upsert_user_quota_for_institution(institution, provider_short_name, old_is_nii, old_quota_type)
return JsonResponse(result[0], status=status)
@@ -522,8 +525,8 @@ def post(self, request, *args, **kwargs):
'message': 'Garbage data removed!!'
}, status=http_status.HTTP_200_OK)
-def external_acc_update(request, access_token):
- if hashlib.sha512(SITE_KEY.encode('utf-8')).hexdigest() != access_token.lower():
+def external_acc_update(request, ts, signature):
+ if not cron_signed_url.verify_signed_params(ts, signature):
return HttpResponse(
json.dumps({'state': 'fail', 'error': 'access forbidden'}),
content_type='application/json',
diff --git a/admin/rdm_statistics/cron.sh b/admin/rdm_statistics/cron.sh
index 62a80709268..a3d434e7808 100644
--- a/admin/rdm_statistics/cron.sh
+++ b/admin/rdm_statistics/cron.sh
@@ -1 +1 @@
-0 2 * * 1 curl http://localhost:8001/statistics/gather/2A85563B2B0F7D3168199F475365F57DA1D56E4BB2CE2B7044EB058AE5E287637E7C636A772682D92C8D6B1830B9A97C5A5DC3DE7016C60BDE4BAA7CC3B38AEB/ | jq -c .
+0 2 * * 1 TOKEN=$(cd /code && python3 manage.py generate_signed_cron_token | tail -n1) && [ -n "$TOKEN" ] && curl "http://localhost:8001/statistics/gather/$TOKEN/" | jq -c .
diff --git a/admin/rdm_statistics/urls.py b/admin/rdm_statistics/urls.py
index 5775b6bf71e..443de572baf 100644
--- a/admin/rdm_statistics/urls.py
+++ b/admin/rdm_statistics/urls.py
@@ -8,7 +8,7 @@
url(r'^index$', views.IndexView.as_view(), name='index'),
url(r'^(?P-?[0-9]+)/graph/(?P\w+)_(?P\w+)\.(\w+)$',
views.ImageView.as_view(), name='graph'),
- url(r'^gather/(?P-?\w+)/$', views.GatherView.as_view(), name='gather'),
+ url(r'^gather/(?P\d+)/(?P[0-9a-fA-F]+)/$', views.GatherView.as_view(), name='gather'),
url(r'^report/(?P-?[0-9]+)/$', views.create_pdf, name='report'),
url(r'^csv/(?P-?[0-9]+)/$', views.create_csv, name='csv'),
url(r'^mail/(?P-?[0-9]+)/$', views.SendView.as_view(), name='mail'),
diff --git a/admin/rdm_statistics/views.py b/admin/rdm_statistics/views.py
index d4522cf1a9c..20f8dc23017 100644
--- a/admin/rdm_statistics/views.py
+++ b/admin/rdm_statistics/views.py
@@ -12,7 +12,6 @@
import csv
import pandas as pd
import numpy as np
-import hashlib
from django.apps import apps
from django.views.generic import TemplateView, View
@@ -41,8 +40,10 @@
import seaborn as sns
import pdfkit
from admin.base import settings
+from admin.base.utils import superuser_required
from admin.rdm.utils import RdmPermissionMixin, get_dummy_institution
from admin.rdm_addons import utils
+from framework.auth import cron_signed_url
import logging
logger = logging.getLogger(__name__)
@@ -51,7 +52,6 @@
STATISTICS_IMAGE_HEIGHT = 4
RECURSIVE_LIMIT = 10000
WB_MAX_RETRY = 3
-SITE_KEY = 'rdm_statistics'
class InstitutionListViewStat(RdmPermissionMixin, UserPassesTestMixin, TemplateView):
"""institlutions list view for statistics"""
@@ -113,8 +113,6 @@ def get_context_data(self, **kwargs):
ctx['current_date'] = current_date
ctx['user'] = user
ctx['provider_data_array'] = provider_data_array
- digest = hashlib.sha512(SITE_KEY.encode('utf-8')).hexdigest()
- ctx['token'] = digest.upper()
return ctx
@@ -447,9 +445,10 @@ class GatherView(TemplateView):
raise_exception = True
def get(self, request, *args, **kwargs):
- # simple authentication
- access_token = self.kwargs.get('access_token')
- if not simple_auth(access_token):
+ # signed-URL authentication
+ ts = self.kwargs.get('ts')
+ signature = self.kwargs.get('signature')
+ if not cron_signed_url.verify_signed_params(ts, signature):
response_hash = {'state': 'fail', 'error': 'access forbidden'}
response_json = json.dumps(response_hash)
response = HttpResponse(response_json, content_type='application/json')
@@ -484,7 +483,7 @@ def get(self, request, *args, **kwargs):
response_json = json.dumps(self.stat_list)
response = HttpResponse(response_json, content_type='application/json')
# statistics mail send
- send_stat_mail(request)
+ send_stat_mail_core(request)
except Exception as err:
response_hash = {'state': 'fail', 'error': str(err)}
response_json = json.dumps(response_hash)
@@ -589,15 +588,15 @@ def count_project_files(self, node_id, provider, path, cookies):
pass
self.count_project_files(provider=provider, node_id=node_id, path='/' + path, cookies=cookies)
-def simple_auth(access_token):
- digest = hashlib.sha512(SITE_KEY.encode('utf-8')).hexdigest()
- if digest == access_token.lower():
- return True
- else:
- return False
-def send_stat_mail(request, **kwargs):
- """send statistics information email"""
+def send_stat_mail_core(request, **kwargs):
+ """Send statistics information email.
+
+ No authorization check here by design - this is the shared logic called
+ both by the send_stat_mail view (protected below) and internally by
+ GatherView.get(), which runs unauthenticated over localhost from
+ cron and is authorized separately via its own access token.
+ """
current_date = get_current_date()
all_institutions = Institution.objects.order_by('id').all()
all_staff_users = OSFUser.objects.filter(is_staff=True)
@@ -630,6 +629,11 @@ def send_stat_mail(request, **kwargs):
response = HttpResponse(response_json, content_type='application/json')
return response
+@superuser_required
+def send_stat_mail(request, **kwargs):
+ """URL-exposed view for /statistics/test/mail/ - Integrated Admin only."""
+ return send_stat_mail_core(request, **kwargs)
+
def send_error_mail(err):
"""send error email"""
current_date = get_current_date()
@@ -776,14 +780,18 @@ def approximate_size(size, a_kilobyte_is_1024_bytes=True):
### views or funcs for development and test
############################################
-class IndexView(TemplateView):
+class IndexView(RdmPermissionMixin, UserPassesTestMixin, TemplateView):
"""index view of statistics module."""
template_name = 'rdm_statistics/index.html'
raise_exception = True
+ def test_func(self):
+ """check user permissions"""
+ return self.is_authenticated and (self.is_super_admin or self.is_admin)
+
def find_bookmark_collection(self, user):
collection = apps.get_model('osf.Collection')
- return collection.objects.get(creator=user, is_deleted=False, is_bookmark_collection=True)
+ return collection.objects.get(creator=user, deleted__isnull=True, is_bookmark_collection=True)
def get(self, request, *args, **kwargs):
user = self.request.user
diff --git a/admin/registration_providers/views.py b/admin/registration_providers/views.py
index 2466bfae63a..c576ed7db16 100644
--- a/admin/registration_providers/views.py
+++ b/admin/registration_providers/views.py
@@ -6,7 +6,7 @@
from django.core.exceptions import ValidationError
from django.core.management import call_command
from django.urls import reverse_lazy
-from django.shortcuts import redirect
+from django.shortcuts import redirect, get_object_or_404
from django.views.generic import View, CreateView, ListView, DetailView, UpdateView, DeleteView, TemplateView
from django.contrib.auth.mixins import PermissionRequiredMixin
from django.contrib import messages
@@ -204,12 +204,14 @@ def get_context_data(self, *args, **kwargs):
return super(DeleteRegistrationProvider, self).get_context_data(*args, **kwargs)
-class CannotDeleteProvider(TemplateView):
+class CannotDeleteProvider(PermissionRequiredMixin, TemplateView):
+ permission_required = 'osf.delete_registrationprovider'
+ raise_exception = True
template_name = 'registration_providers/cannot_delete.html'
def get_context_data(self, **kwargs):
context = super(CannotDeleteProvider, self).get_context_data(**kwargs)
- context['provider'] = RegistrationProvider.objects.get(id=self.kwargs['registration_provider_id'])
+ context['provider'] = get_object_or_404(RegistrationProvider, id=self.kwargs['registration_provider_id'])
return context
@@ -376,7 +378,7 @@ def get(self, request, *args, **kwargs):
return redirect(reverse_lazy('registration_providers:detail', kwargs={'registration_provider_id': provider.id}))
-class ChangeSchema(TemplateView):
+class ChangeSchema(PermissionRequiredMixin, TemplateView):
permission_required = 'osf.change_registrationprovider'
template_name = 'registration_providers/change_schema.html'
diff --git a/admin/templates/institutional_storage_quota_control/user_list_by_institute.html b/admin/templates/institutional_storage_quota_control/user_list_by_institute.html
index 2e2050514b0..3d747299e1c 100644
--- a/admin/templates/institutional_storage_quota_control/user_list_by_institute.html
+++ b/admin/templates/institutional_storage_quota_control/user_list_by_institute.html
@@ -121,7 +121,7 @@ {% trans "Institutional Storage" %} > {{ institution_name }}
https://code.djangoproject.com/ticket/17971
+ setattr(self.request, 'session', 'session')
+ setattr(self.request, '_messages', FallbackStorage(self.request))
+ self.form = LoginForm(data={'email': 'user@example.com', 'password': 'secret'})
+ nt.assert_true(self.form.is_valid())
+ self.view = setup_form_view(self.view, self.request, self.form)
+
+ @mock.patch('admin.common_auth.views.ENABLE_LOGIN_FORM', False)
+ @mock.patch('admin.common_auth.views.authenticate')
+ def test_authenticate_skipped_when_login_form_disabled(self, mock_authenticate):
+ response = self.view.form_valid(self.form)
+ nt.assert_false(mock_authenticate.called)
+ nt.assert_equal(response.status_code, 302)
+ nt.assert_true(response.url.startswith('/account/login'))
+
+ @mock.patch('admin.common_auth.views.ENABLE_LOGIN_FORM', True)
+ @mock.patch('admin.common_auth.views.authenticate', return_value=None)
+ def test_authenticate_called_and_denied_when_login_form_enabled(self, mock_authenticate):
+ response = self.view.form_valid(self.form)
+ nt.assert_true(mock_authenticate.called)
+ nt.assert_equal(response.status_code, 302)
+ nt.assert_true(response.url.startswith('/account/login'))
+
+ @mock.patch('admin.common_auth.views.ENABLE_LOGIN_FORM', True)
+ @mock.patch('admin.common_auth.views.login')
+ @mock.patch('admin.common_auth.views.authenticate')
+ def test_authenticate_called_and_login_succeeds_when_login_form_enabled(self, mock_authenticate, mock_login):
+ mock_authenticate.return_value = AuthUserFactory()
+ response = self.view.form_valid(self.form)
+ nt.assert_true(mock_authenticate.called)
+ nt.assert_true(mock_login.called)
+ nt.assert_equal(response.status_code, 302)
diff --git a/admin_tests/institutional_storage_quota_control/test_views.py b/admin_tests/institutional_storage_quota_control/test_views.py
index e5996209d36..9f55b0ce523 100644
--- a/admin_tests/institutional_storage_quota_control/test_views.py
+++ b/admin_tests/institutional_storage_quota_control/test_views.py
@@ -253,6 +253,121 @@ def test_post__max_quota_too_large(self):
nt.assert_is_not_none(new_user_quota)
nt.assert_equal(new_user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+ def test_post__institution_default_max_quota_created(self):
+ """Test that InstitutionDefaultMaxQuota is created/updated on POST"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+
+ new_max_quota = 200
+ region = RegionFactory(_id=self.institution01._id)
+ region.waterbutler_settings['storage']['type'] = Region.INSTITUTIONS
+ region.save()
+
+ request = RequestFactory().post(
+ reverse(self.view_name,
+ kwargs={'institution_id': self.institution01.id}),
+ {'maxQuota': new_max_quota})
+ request.user = self.superuser
+ response = self.view(request, institution_id=self.institution01.id)
+
+ nt.assert_equal(response.status_code, 302)
+
+ # Verify InstitutionDefaultMaxQuota record was created
+ quota = InstitutionDefaultMaxQuota.objects.filter(
+ institution_id=self.institution01.id
+ ).first()
+ nt.assert_is_not_none(quota)
+ nt.assert_equal(quota.default_max_quota, new_max_quota)
+
+ def test_post__institution_default_max_quota_updated(self):
+ """Test that InstitutionDefaultMaxQuota is updated on subsequent POST"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+
+ # Create initial record
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution01.id,
+ defaults={'default_max_quota': 100}
+ )
+
+ # Update via POST
+ new_max_quota = 250
+ region = RegionFactory(_id=self.institution01._id)
+ region.waterbutler_settings['storage']['type'] = Region.INSTITUTIONS
+ region.save()
+
+ request = RequestFactory().post(
+ reverse(self.view_name,
+ kwargs={'institution_id': self.institution01.id}),
+ {'maxQuota': new_max_quota})
+ request.user = self.superuser
+ response = self.view(request, institution_id=self.institution01.id)
+
+ nt.assert_equal(response.status_code, 302)
+
+ # Verify InstitutionDefaultMaxQuota record was updated
+ quota = InstitutionDefaultMaxQuota.objects.filter(
+ institution_id=self.institution01.id
+ ).first()
+ nt.assert_is_not_none(quota)
+ nt.assert_equal(quota.default_max_quota, new_max_quota)
+
+ # Verify only one record exists for this institution
+ count = InstitutionDefaultMaxQuota.objects.filter(
+ institution_id=self.institution01.id
+ ).count()
+ nt.assert_equal(count, 1)
+
+ def test_post__integrity_error_handles_fallback_update(self):
+ """Test that IntegrityError in update_or_create triggers fallback update logic"""
+ from unittest import mock
+ from django.db import IntegrityError, transaction
+
+ new_max_quota = 300
+ region = RegionFactory(_id=self.institution01.guid)
+ region.waterbutler_settings['storage']['type'] = Region.INSTITUTIONS
+ region.save()
+
+ # Create initial UserQuota
+ initial_quota = UserQuota.objects.create(
+ user=self.institution01_admin,
+ storage_type=UserQuota.CUSTOM_STORAGE,
+ max_quota=100
+ )
+
+ def raise_integrity_error(*args, **kwargs):
+ transaction.set_rollback(True)
+ raise IntegrityError('Simulated race condition')
+
+ # Mock update_or_create to raise IntegrityError and mark transaction for rollback
+ with mock.patch.object(UserQuota.objects, 'update_or_create') as mock_update:
+ mock_update.side_effect = raise_integrity_error
+
+ request = RequestFactory().post(
+ reverse(
+ self.view_name,
+ kwargs={'institution_id': self.institution01.id}
+ ),
+ {'maxQuota': new_max_quota}
+ )
+ request.user = self.institution01_admin
+
+ response = self.view(
+ request,
+ institution_id=self.institution01.id
+ )
+
+ # Verify response is redirect
+ nt.assert_equal(response.status_code, 302)
+
+ # Verify max_quota was updated via fallback logic
+ updated_quota = UserQuota.objects.filter(
+ user=self.institution01_admin,
+ storage_type=UserQuota.CUSTOM_STORAGE
+ ).first()
+
+ nt.assert_is_not_none(updated_quota)
+ nt.assert_equal(updated_quota.max_quota, new_max_quota)
+ nt.assert_equal(updated_quota.id, initial_quota.id)
+
class TestUserListByInstitutionStorageID(AdminTestCase):
def setUp(self):
@@ -261,7 +376,11 @@ def setUp(self):
self.institution02 = InstitutionFactory(name='inst02')
self.region01 = RegionFactory(_id=self.institution01._id, name='Storage 01')
+ self.region01.waterbutler_settings['storage']['type'] = Region.INSTITUTIONS
+ self.region01.save()
self.region02 = RegionFactory(_id=self.institution02._id, name='Storage 02')
+ self.region02.waterbutler_settings['storage']['type'] = Region.INSTITUTIONS
+ self.region02.save()
self.anon = AnonymousUser()
self.user = AuthUserFactory(fullname='user')
@@ -441,18 +560,17 @@ def test_get_institution__not_found(self):
view.get_institution()
# Institution use NII Storage
- self.institution01._id = ''
- self.institution01.save()
+ nii_institution = InstitutionFactory(_id='', name='nii_inst')
request = RequestFactory().get(
reverse(
self.view_name,
- kwargs={'institution_id': self.institution01.id}
+ kwargs={'institution_id': nii_institution.id}
)
)
request.user = self.superuser
view = setup_view(self.view_instance, request,
- institution_id=self.institution01.id)
+ institution_id=nii_institution.id)
with nt.assert_raises(Http404):
view.get_institution()
@@ -476,6 +594,95 @@ def test__institution_id_not_valid(self):
)
)
+ def test_get_default_max_quota__institution_has_default(self):
+ """Test get_default_max_quota returns value from DB when record exists"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+
+ # Create default max quota for institution
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution01.id,
+ defaults={'default_max_quota': 500}
+ )
+
+ request = RequestFactory().get(
+ reverse(
+ self.view_name,
+ kwargs={'institution_id': self.institution01.id}
+ )
+ )
+ request.user = self.institution01_admin
+
+ view = setup_view(self.view_instance, request,
+ institution_id=self.institution01.id)
+ view.institution_id = self.institution01.id
+ default_max_quota = view.get_default_max_quota()
+
+ nt.assert_equal(default_max_quota, 500)
+
+ def test_get_default_max_quota__institution_no_default(self):
+ """Test get_default_max_quota returns DEFAULT_MAX_QUOTA when record doesn't exist"""
+ request = RequestFactory().get(
+ reverse(
+ self.view_name,
+ kwargs={'institution_id': self.institution01.id}
+ )
+ )
+ request.user = self.institution01_admin
+
+ view = setup_view(self.view_instance, request,
+ institution_id=self.institution01.id)
+ view.institution_id = self.institution01.id
+ default_max_quota = view.get_default_max_quota()
+
+ nt.assert_equal(default_max_quota, api_settings.DEFAULT_MAX_QUOTA)
+
+ def test_get_context_data__includes_default_max_quota(self):
+ """Test get_context_data includes default_max_quota in context"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+
+ # Create default max quota for institution
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution01.id,
+ defaults={'default_max_quota': 300}
+ )
+
+ request = RequestFactory().get(
+ reverse(
+ self.view_name,
+ kwargs={'institution_id': self.institution01.id}
+ )
+ )
+ request.user = self.institution01_admin
+
+ view = setup_view(self.view_instance, request,
+ institution_id=self.institution01.id)
+ view.institution_id = self.institution01.id
+ view.object_list = view.get_queryset()
+ context = view.get_context_data()
+
+ nt.assert_in('default_max_quota', context)
+ nt.assert_equal(context['default_max_quota'], 300)
+
+ def test_get_context_data__default_max_quota_fallback(self):
+ """Test get_context_data uses DEFAULT_MAX_QUOTA as fallback"""
+ request = RequestFactory().get(
+ reverse(
+ self.view_name,
+ kwargs={'institution_id': self.institution01.id}
+ )
+ )
+ request.user = self.institution01_admin
+
+ view = setup_view(self.view_instance, request,
+ institution_id=self.institution01.id)
+ view.institution_id = self.institution01.id
+ view.object_list = view.get_queryset()
+ context = view.get_context_data()
+
+ nt.assert_in('default_max_quota', context)
+ nt.assert_equal(context['default_max_quota'], api_settings.DEFAULT_MAX_QUOTA)
+
+
class TestAccessInstitutionStorageList(AdminTestCase):
def setUp(self):
super(TestAccessInstitutionStorageList, self).setUp()
diff --git a/admin_tests/management/__init__.py b/admin_tests/management/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/admin_tests/management/test_views.py b/admin_tests/management/test_views.py
new file mode 100644
index 00000000000..8e93b66af0f
--- /dev/null
+++ b/admin_tests/management/test_views.py
@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+from nose import tools as nt
+import mock
+from django.test import RequestFactory
+from django.contrib.auth.models import AnonymousUser
+from django.core.exceptions import PermissionDenied
+
+from admin.management import views
+from osf_tests.factories import AuthUserFactory
+from tests.base import AdminTestCase
+
+
+class TestManagementCommandsPermission(AdminTestCase):
+ """ManagementCommands must be restricted to Integrated Admin (is_super_admin)."""
+
+ def setUp(self):
+ super(TestManagementCommandsPermission, self).setUp()
+ self.superuser = AuthUserFactory()
+ self.superuser.is_superuser = True
+ self.superuser.save()
+ self.general_user = AuthUserFactory()
+
+ def test_get_denied_for_general_user(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = self.general_user
+ with nt.assert_raises(PermissionDenied):
+ views.ManagementCommands.as_view()(request)
+
+ def test_get_denied_for_anonymous(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = AnonymousUser()
+ with nt.assert_raises(PermissionDenied):
+ views.ManagementCommands.as_view()(request)
+
+ def test_get_allowed_for_superuser(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = self.superuser
+ response = views.ManagementCommands.as_view()(request)
+ nt.assert_equal(response.status_code, 200)
+
+
+class TestWaffleFlagPermission(AdminTestCase):
+ """WaffleFlag must be restricted to Integrated Admin (is_super_admin)."""
+
+ def setUp(self):
+ super(TestWaffleFlagPermission, self).setUp()
+ self.superuser = AuthUserFactory()
+ self.superuser.is_superuser = True
+ self.superuser.save()
+ self.general_user = AuthUserFactory()
+
+ def test_post_denied_for_general_user(self):
+ request = RequestFactory().post('/fake_path')
+ request.user = self.general_user
+ with nt.assert_raises(PermissionDenied):
+ views.WaffleFlag.as_view()(request)
+
+ def test_post_denied_for_anonymous(self):
+ request = RequestFactory().post('/fake_path')
+ request.user = AnonymousUser()
+ with nt.assert_raises(PermissionDenied):
+ views.WaffleFlag.as_view()(request)
+
+ @mock.patch('admin.management.views.manage_waffle')
+ def test_post_allowed_for_superuser(self, mock_manage_waffle):
+ request = RequestFactory().post('/fake_path')
+ request.user = self.superuser
+ response = views.WaffleFlag.as_view()(request)
+ nt.assert_equal(response.status_code, 302)
+ nt.assert_true(mock_manage_waffle.called)
diff --git a/admin_tests/preprint_providers/test_views.py b/admin_tests/preprint_providers/test_views.py
index 66403b89536..204107770a3 100644
--- a/admin_tests/preprint_providers/test_views.py
+++ b/admin_tests/preprint_providers/test_views.py
@@ -6,6 +6,9 @@
import responses
from nose import tools as nt
from django.test import RequestFactory
+from django.contrib.auth.models import AnonymousUser, Permission
+from django.core.exceptions import PermissionDenied
+from django.http import Http404
from django.core.files.uploadedfile import InMemoryUploadedFile
from scripts.update_taxonomies import update_taxonomies
@@ -369,6 +372,48 @@ def test_cannot_get_if_preprints_present(self, req, view, preprint, provider_wit
assert redirect.url == '/preprint_providers/{}/cannot_delete/'.format(provider_with_preprint.id)
assert redirect.status_code == 302
+@pytest.mark.urls('admin.base.urls')
+class TestCannotDeleteProviderPermission:
+ """CannotDeleteProvider must be restricted to users with 'osf.delete_preprintprovider'."""
+
+ @pytest.fixture()
+ def provider(self):
+ return PreprintProviderFactory()
+
+ def test_denied_for_general_user(self, req, provider):
+ with pytest.raises(PermissionDenied):
+ views.CannotDeleteProvider.as_view()(req, preprint_provider_id=provider.id)
+
+ def test_denied_for_anonymous(self, provider):
+ req = RequestFactory().get('/fake_path')
+ req.user = AnonymousUser()
+ with pytest.raises(PermissionDenied):
+ views.CannotDeleteProvider.as_view()(req, preprint_provider_id=provider.id)
+
+ def test_allowed_for_user_with_permission(self, req, user, provider):
+ permission = Permission.objects.get(codename='delete_preprintprovider')
+ user.user_permissions.add(permission)
+ user.save()
+
+ res = views.CannotDeleteProvider.as_view()(req, preprint_provider_id=provider.id)
+ assert res.status_code == 200
+
+ def test_allowed_for_superuser(self, req, user, provider):
+ user.is_superuser = True
+ user.save()
+
+ res = views.CannotDeleteProvider.as_view()(req, preprint_provider_id=provider.id)
+ assert res.status_code == 200
+
+ def test_404_for_missing_provider(self, req, user):
+ permission = Permission.objects.get(codename='delete_preprintprovider')
+ user.user_permissions.add(permission)
+ user.save()
+
+ with pytest.raises(Http404):
+ views.CannotDeleteProvider.as_view()(req, preprint_provider_id=999999)
+
+
class TestProcessCustomTaxonomy(ProcessCustomTaxonomyMixinBase):
@pytest.fixture(autouse=True)
diff --git a/admin_tests/quota_recalc/test_views.py b/admin_tests/quota_recalc/test_views.py
index cbaa876e015..cdc2b89903d 100644
--- a/admin_tests/quota_recalc/test_views.py
+++ b/admin_tests/quota_recalc/test_views.py
@@ -1,24 +1,35 @@
# -*- coding: utf-8 -*-
from django.test import RequestFactory
+from django.contrib.auth.models import AnonymousUser
+from django.core.exceptions import PermissionDenied
import json
import mock
from nose import tools as nt
from admin.quota_recalc import views
from api.base import settings as api_settings
-from osf.models import UserQuota
+from osf.models import OSFUser, UserQuota
from osf_tests.factories import AuthUserFactory, InstitutionFactory, RegionFactory
from tests.base import AdminTestCase
class TestQuotaRecalcView(AdminTestCase):
- @staticmethod
- def get_request(view, **kwargs):
- return view(RequestFactory().get('/fake_path'), **kwargs)
+ def setUp(self):
+ super(TestQuotaRecalcView, self).setUp()
+ self.superuser = AuthUserFactory()
+ self.superuser.is_superuser = True
+ self.superuser.save()
+
+ def get_request(self, view, **kwargs):
+ request = RequestFactory().get('/fake_path')
+ request.user = self.superuser
+ return view(request, **kwargs)
+ @mock.patch('admin.quota_recalc.views.get_default_max_quota')
@mock.patch('admin.quota_recalc.views.used_quota')
- def test_user_create_userquota_record(self, mock_usedquota):
+ def test_user_create_userquota_record(self, mock_usedquota, mock_get_default_max_quota):
mock_usedquota.return_value = 1500
+ mock_get_default_max_quota.return_value = api_settings.DEFAULT_MAX_QUOTA
user = AuthUserFactory()
UserQuota.objects.filter(user=user).delete()
@@ -31,9 +42,11 @@ def test_user_create_userquota_record(self, mock_usedquota):
nt.assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
nt.assert_equal(user_quota.used, 1500)
+ @mock.patch('admin.quota_recalc.views.get_default_max_quota')
@mock.patch('admin.quota_recalc.views.used_quota')
- def test_user_update_userquota_record(self, mock_usedquota):
+ def test_user_update_userquota_record(self, mock_usedquota, mock_get_default_max_quota):
mock_usedquota.return_value = 7000
+ mock_get_default_max_quota.return_value = 200
user = AuthUserFactory()
UserQuota.objects.create(
@@ -51,9 +64,11 @@ def test_user_update_userquota_record(self, mock_usedquota):
nt.assert_equal(user_quota.max_quota, 200)
nt.assert_equal(user_quota.used, 7000)
+ @mock.patch('admin.quota_recalc.views.get_default_max_quota')
@mock.patch('admin.quota_recalc.views.used_quota')
- def test_user_invalid_guid(self, mock_usedquota):
+ def test_user_invalid_guid(self, mock_usedquota, mock_get_default_max_quota):
mock_usedquota.return_value = 3000
+ mock_get_default_max_quota.return_value = api_settings.DEFAULT_MAX_QUOTA
response = self.get_request(views.user, guid='cuzidontcare')
res_json = json.loads(response.content)
@@ -61,9 +76,11 @@ def test_user_invalid_guid(self, mock_usedquota):
nt.assert_equal(res_json['status'], 'failed')
nt.assert_equal(res_json['message'], 'User not found.')
+ @mock.patch('admin.quota_recalc.views.get_default_max_quota')
@mock.patch('admin.quota_recalc.views.used_quota')
- def test_users_create_userquota_record(self, mock_usedquota):
+ def test_users_create_userquota_record(self, mock_usedquota, mock_get_default_max_quota):
mock_usedquota.return_value = 1500
+ mock_get_default_max_quota.return_value = api_settings.DEFAULT_MAX_QUOTA
user = AuthUserFactory()
user2 = AuthUserFactory()
UserQuota.objects.filter(user=user).delete()
@@ -85,7 +102,88 @@ def test_users_create_userquota_record(self, mock_usedquota):
res_json3 = json.loads(response3.content)
nt.assert_equal(response3.status_code, 200)
nt.assert_equal(res_json3['status'], 'OK')
- nt.assert_true('2' in res_json3['message'])
+ nt.assert_true('3' in res_json3['message'])
+
+
+class TestQuotaRecalcPermission(AdminTestCase):
+ """all_users/user must be restricted to Integrated Admin (is_superuser)."""
+
+ def setUp(self):
+ super(TestQuotaRecalcPermission, self).setUp()
+ self.general_user = AuthUserFactory()
+ self.superuser = AuthUserFactory()
+ self.superuser.is_superuser = True
+ self.superuser.save()
+ self.target_user = AuthUserFactory()
+
+ def test_all_users_denied_for_general_user(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = self.general_user
+ with nt.assert_raises(PermissionDenied):
+ views.all_users(request)
+
+ def test_all_users_denied_for_anonymous(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = AnonymousUser()
+ with nt.assert_raises(PermissionDenied):
+ views.all_users(request)
+
+ @mock.patch('admin.quota_recalc.views.used_quota')
+ def test_all_users_allowed_for_superuser(self, mock_usedquota):
+ mock_usedquota.return_value = 0
+ request = RequestFactory().get('/fake_path')
+ request.user = self.superuser
+ response = views.all_users(request)
+ nt.assert_equal(response.status_code, 200)
+
+ def test_user_denied_for_general_user(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = self.general_user
+ with nt.assert_raises(PermissionDenied):
+ views.user(request, guid=self.target_user._id)
+
+ def test_user_denied_for_anonymous(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = AnonymousUser()
+ with nt.assert_raises(PermissionDenied):
+ views.user(request, guid=self.target_user._id)
+
+ @mock.patch('admin.quota_recalc.views.used_quota')
+ def test_user_allowed_for_superuser(self, mock_usedquota):
+ mock_usedquota.return_value = 0
+ request = RequestFactory().get('/fake_path')
+ request.user = self.superuser
+ response = views.user(request, guid=self.target_user._id)
+ nt.assert_equal(response.status_code, 200)
+
+ def _invalid_superuser(self, **fields):
+ # queryset.update() skips OSFUser.save(), which would otherwise recompute is_active
+ OSFUser.objects.filter(pk=self.superuser.pk).update(**fields)
+ self.superuser.refresh_from_db()
+ nt.assert_true(self.superuser.is_superuser)
+ return self.superuser
+
+ def _assert_denied(self, view, user, **kwargs):
+ request = RequestFactory().get('/fake_path')
+ request.user = user
+ with nt.assert_raises(PermissionDenied):
+ view(request, **kwargs)
+
+ def test_all_users_denied_for_inactive_superuser(self):
+ user = self._invalid_superuser(is_active=False)
+ self._assert_denied(views.all_users, user)
+
+ def test_all_users_denied_for_unregistered_superuser(self):
+ user = self._invalid_superuser(is_registered=False)
+ self._assert_denied(views.all_users, user)
+
+ def test_user_denied_for_inactive_superuser(self):
+ user = self._invalid_superuser(is_active=False)
+ self._assert_denied(views.user, user, guid=self.target_user._id)
+
+ def test_user_denied_for_unregistered_superuser(self):
+ user = self._invalid_superuser(is_registered=False)
+ self._assert_denied(views.user, user, guid=self.target_user._id)
class TestCalculateQuota(AdminTestCase):
@@ -94,19 +192,24 @@ def setUp(self):
super(TestCalculateQuota, self).setUp()
self.user = AuthUserFactory()
+ @mock.patch('admin.quota_recalc.views.get_default_max_quota')
@mock.patch('admin.quota_recalc.views.used_quota')
- def test_user_without_institution(self, mock_usedquota):
+ def test_user_without_institution(self, mock_usedquota, mock_get_default_max_quota):
mock_usedquota.return_value = 5000
+ mock_get_default_max_quota.return_value = 1000
views.calculate_quota(self.user)
user_quota = UserQuota.objects.filter(user=self.user).all()
nt.assert_equal(len(user_quota), 1)
nt.assert_equal(user_quota[0].used, 5000)
+ nt.assert_equal(user_quota[0].max_quota, 1000)
+ @mock.patch('admin.quota_recalc.views.get_default_max_quota')
@mock.patch('admin.quota_recalc.views.used_quota')
- def test_user_institution_without_custom_storage(self, mock_usedquota):
+ def test_user_institution_without_custom_storage(self, mock_usedquota, mock_get_default_max_quota):
mock_usedquota.return_value = 6000
+ mock_get_default_max_quota.return_value = 2000
institution = InstitutionFactory()
self.user.affiliated_institutions.add(institution)
@@ -116,12 +219,17 @@ def test_user_institution_without_custom_storage(self, mock_usedquota):
user_quota = UserQuota.objects.filter(user=self.user).all()
nt.assert_equal(len(user_quota), 1)
nt.assert_equal(user_quota[0].used, 6000)
+ nt.assert_equal(user_quota[0].max_quota, 2000)
+ @mock.patch('admin.quota_recalc.views.get_default_max_quota')
@mock.patch('admin.quota_recalc.views.used_quota')
- def test_user_institution_with_custom_storage(self, mock_usedquota):
+ def test_user_institution_with_custom_storage(self, mock_usedquota, mock_get_default_max_quota):
+
mock_usedquota.side_effect = \
lambda uid, storage_type: 300 if storage_type == UserQuota.NII_STORAGE else 7000
+ mock_get_default_max_quota.side_effect = \
+ lambda _, storage_type: 1000 if storage_type == UserQuota.NII_STORAGE else 2000
institution = InstitutionFactory()
self.user.affiliated_institutions.add(institution)
RegionFactory(_id=institution._id)
@@ -136,5 +244,11 @@ def test_user_institution_with_custom_storage(self, mock_usedquota):
UserQuota.CUSTOM_STORAGE: 7000,
}
- nt.assert_equal(user_quota[0].used, expected[user_quota[0].storage_type])
- nt.assert_equal(user_quota[1].used, expected[user_quota[1].storage_type])
+ expected_max = {
+ UserQuota.NII_STORAGE: 1000,
+ UserQuota.CUSTOM_STORAGE: 2000,
+ }
+
+ for uq in user_quota:
+ nt.assert_equal(uq.used, expected[uq.storage_type])
+ nt.assert_equal(uq.max_quota, expected_max[uq.storage_type])
diff --git a/admin_tests/rdm_custom_storage_location/test_utils.py b/admin_tests/rdm_custom_storage_location/test_utils.py
index 9aa9c9a2d79..55ca340482a 100644
--- a/admin_tests/rdm_custom_storage_location/test_utils.py
+++ b/admin_tests/rdm_custom_storage_location/test_utils.py
@@ -19,18 +19,24 @@
get_ociinstitutions_info,
get_nextcloudinstitutions_info,
get_dropboxbusiness_info,
- get_institutional_storage_information
+ get_institutional_storage_information,
+ is_institution_using_nii_storage,
+ upsert_user_quota_for_institution,
)
from mock import patch, MagicMock
from osf_tests.factories import (
InstitutionFactory,
ProjectFactory,
RegionFactory,
+ UserFactory,
bulkmount_waterbutler_settings,
addon_waterbutler_settings,
AuthUserFactory
)
from tests.base import AdminTestCase
+from api.base import settings as api_settings
+from osf.models import UserQuota
+from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
from rest_framework import status as http_status
@@ -438,3 +444,292 @@ def test_get_institutional_storage_information_unknown_provider(self):
)
nt.assert_equal(result, {})
+
+
+@pytest.mark.feature_202210
+class TestIsInstitutionUsingNiiStorage(AdminTestCase):
+
+ def setUp(self):
+ super(TestIsInstitutionUsingNiiStorage, self).setUp()
+ self.institution = InstitutionFactory()
+
+ def test_returns_true_when_no_region_configured(self):
+ """Return True when institution has no region (no storage configured yet)."""
+ result = is_institution_using_nii_storage(self.institution)
+ nt.assert_true(result)
+
+ def test_returns_true_when_region_is_nii_storage(self):
+ """Return True when institution's region is NII_STORAGE type."""
+ from addons.osfstorage.models import Region
+ region = RegionFactory(_id=self.institution._id)
+ region.waterbutler_settings['storage']['type'] = Region.NII_STORAGE
+ region.save()
+
+ result = is_institution_using_nii_storage(self.institution)
+ nt.assert_true(result)
+
+ def test_returns_false_when_region_is_custom_storage(self):
+ """Return False when institution's region is custom (non-NII) storage."""
+ from addons.osfstorage.models import Region
+ region = RegionFactory(_id=self.institution._id)
+ region.waterbutler_settings['storage']['type'] = Region.INSTITUTIONS
+ region.save()
+
+ result = is_institution_using_nii_storage(self.institution)
+ nt.assert_false(result)
+
+
+@pytest.mark.feature_202210
+class TestUpsertUserQuotaForInstitution(AdminTestCase):
+
+ def setUp(self):
+ super(TestUpsertUserQuotaForInstitution, self).setUp()
+ self.institution = InstitutionFactory()
+ self.user = UserFactory()
+ self.user.affiliated_institutions.add(self.institution)
+ self.user.save()
+
+ def test_nii_to_custom_creates_custom_storage_quota_with_default_max(self):
+ """NII → custom storage: upsert CUSTOM_STORAGE UserQuota with DEFAULT_MAX_QUOTA."""
+ result = upsert_user_quota_for_institution(
+ self.institution, provider_short_name='s3', old_is_nii=True
+ )
+ nt.assert_is_none(result)
+
+ user_quota = UserQuota.objects.get(
+ user=self.user, storage_type=UserQuota.CUSTOM_STORAGE
+ )
+ nt.assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+
+ def test_nii_to_custom_uses_institution_default_max_quota(self):
+ """NII → custom storage: upsert CUSTOM_STORAGE UserQuota with InstitutionDefaultMaxQuota value."""
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 500}
+ )
+
+ upsert_user_quota_for_institution(
+ self.institution, provider_short_name='s3', old_is_nii=True
+ )
+
+ user_quota = UserQuota.objects.get(
+ user=self.user, storage_type=UserQuota.CUSTOM_STORAGE
+ )
+ nt.assert_equal(user_quota.max_quota, 500)
+
+ def test_nii_to_custom_updates_existing_quota(self):
+ """NII → custom storage: update max_quota when UserQuota already exists."""
+ UserQuota.objects.create(
+ user=self.user,
+ storage_type=UserQuota.CUSTOM_STORAGE,
+ max_quota=100,
+ used=50,
+ )
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 300}
+ )
+
+ upsert_user_quota_for_institution(
+ self.institution, provider_short_name='s3', old_is_nii=True
+ )
+
+ user_quota = UserQuota.objects.get(
+ user=self.user, storage_type=UserQuota.CUSTOM_STORAGE
+ )
+ nt.assert_equal(user_quota.max_quota, 300)
+ nt.assert_equal(user_quota.used, 50)
+
+ def test_custom_to_nii_creates_quota_with_default_max(self):
+ """Custom → NII storage: upsert UserQuota using get_user_quota_type_for_nii_storage."""
+ from addons.osfstorage.models import Region
+ region = RegionFactory(_id=self.institution._id)
+ region.waterbutler_settings['storage']['type'] = Region.NII_STORAGE
+ region.save()
+
+ upsert_user_quota_for_institution(
+ self.institution, provider_short_name='osfstorage', old_is_nii=False
+ )
+
+ user_quota = UserQuota.objects.get(
+ user=self.user, storage_type=UserQuota.CUSTOM_STORAGE
+ )
+ nt.assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+
+ def test_custom_to_nii_resets_existing_quota_to_default(self):
+ """Custom → NII storage: existing CUSTOM_STORAGE quota is overwritten with DEFAULT_MAX_QUOTA."""
+ region = RegionFactory(_id=self.institution._id)
+ region.waterbutler_settings['storage']['type'] = Region.NII_STORAGE
+ region.save()
+
+ UserQuota.objects.create(
+ user=self.user,
+ storage_type=UserQuota.CUSTOM_STORAGE,
+ max_quota=500,
+ used=30,
+ )
+
+ upsert_user_quota_for_institution(
+ self.institution, provider_short_name='osfstorage', old_is_nii=False
+ )
+
+ user_quota = UserQuota.objects.get(user=self.user, storage_type=UserQuota.CUSTOM_STORAGE)
+ nt.assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+ nt.assert_equal(user_quota.used, 30)
+
+ def test_no_change_when_already_nii_to_nii(self):
+ """NII re-save (2nd+ time): existing CUSTOM_STORAGE quota must NOT be overwritten."""
+ region = RegionFactory(_id=self.institution._id)
+ region.waterbutler_settings['storage']['type'] = Region.NII_STORAGE
+ region.save()
+
+ UserQuota.objects.create(
+ user=self.user,
+ storage_type=UserQuota.CUSTOM_STORAGE,
+ max_quota=500,
+ )
+
+ upsert_user_quota_for_institution(
+ self.institution,
+ provider_short_name='osfstorage',
+ old_is_nii=True,
+ old_quota_type=UserQuota.CUSTOM_STORAGE,
+ )
+
+ user_quota = UserQuota.objects.get(user=self.user, storage_type=UserQuota.CUSTOM_STORAGE)
+ nt.assert_equal(user_quota.max_quota, 500)
+
+ def test_no_change_when_custom_to_custom(self):
+ """No quota upsert when switching custom → custom (old_is_nii=False)."""
+ upsert_user_quota_for_institution(
+ self.institution, provider_short_name='s3', old_is_nii=False
+ )
+
+ nt.assert_false(UserQuota.objects.filter(user=self.user).exists())
+
+ def test_nii_quota_type_changed_preserves_existing_user_quota(self):
+ """NII quota type changed (no-region → NII-region): existing quota of each user is preserved."""
+ from addons.osfstorage.models import Region
+ region = RegionFactory(_id=self.institution._id)
+ region.waterbutler_settings['storage']['type'] = Region.NII_STORAGE
+ region.save()
+
+ UserQuota.objects.create(
+ user=self.user,
+ storage_type=UserQuota.NII_STORAGE,
+ max_quota=200,
+ )
+
+ upsert_user_quota_for_institution(
+ self.institution,
+ provider_short_name='osfstorage',
+ old_is_nii=True,
+ old_quota_type=UserQuota.NII_STORAGE,
+ )
+
+ # new_quota_type = CUSTOM_STORAGE (institution now has NII region)
+ user_quota = UserQuota.objects.get(user=self.user, storage_type=UserQuota.CUSTOM_STORAGE)
+ nt.assert_equal(user_quota.max_quota, 200)
+
+ def test_nii_quota_type_changed_uses_default_for_users_without_existing_quota(self):
+ """NII quota type changed: users with no existing quota record receive system default."""
+ from addons.osfstorage.models import Region
+ region = RegionFactory(_id=self.institution._id)
+ region.waterbutler_settings['storage']['type'] = Region.NII_STORAGE
+ region.save()
+
+ # No existing UserQuota for user
+ upsert_user_quota_for_institution(
+ self.institution,
+ provider_short_name='osfstorage',
+ old_is_nii=True,
+ old_quota_type=UserQuota.NII_STORAGE,
+ )
+
+ user_quota = UserQuota.objects.get(user=self.user, storage_type=UserQuota.CUSTOM_STORAGE)
+ nt.assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+
+ def test_no_change_when_old_and_new_quota_type_are_same(self):
+ """No quota upsert when old_quota_type equals new_quota_type (NII quota type unchanged)."""
+ # No region → new_quota_type = NII_STORAGE, old_quota_type = NII_STORAGE → same → return
+ upsert_user_quota_for_institution(
+ self.institution,
+ provider_short_name='osfstorage',
+ old_is_nii=True,
+ old_quota_type=UserQuota.NII_STORAGE,
+ )
+
+ nt.assert_false(UserQuota.objects.filter(user=self.user).exists())
+
+ def test_nii_quota_type_changed_mixed_users(self):
+ """B3: user with existing quota gets preserved value; user without quota gets DEFAULT."""
+ region = RegionFactory(_id=self.institution._id)
+ region.waterbutler_settings['storage']['type'] = Region.NII_STORAGE
+ region.save()
+
+ user2 = UserFactory()
+ user2.affiliated_institutions.add(self.institution)
+ user2.save()
+
+ UserQuota.objects.create(
+ user=self.user,
+ storage_type=UserQuota.NII_STORAGE,
+ max_quota=300,
+ )
+ # user2 has no NII_STORAGE quota
+
+ upsert_user_quota_for_institution(
+ self.institution,
+ provider_short_name='osfstorage',
+ old_is_nii=True,
+ old_quota_type=UserQuota.NII_STORAGE,
+ )
+
+ quota1 = UserQuota.objects.get(user=self.user, storage_type=UserQuota.CUSTOM_STORAGE)
+ quota2 = UserQuota.objects.get(user=user2, storage_type=UserQuota.CUSTOM_STORAGE)
+ nt.assert_equal(quota1.max_quota, 300)
+ nt.assert_equal(quota2.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+
+ def test_non_affiliated_user_quota_not_affected(self):
+ """Users not affiliated with the institution must not have their quota created or modified."""
+ non_affiliated_user = UserFactory()
+ # non_affiliated_user is deliberately NOT added to self.institution
+
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 400}
+ )
+
+ upsert_user_quota_for_institution(
+ self.institution, provider_short_name='s3', old_is_nii=True
+ )
+
+ nt.assert_false(
+ UserQuota.objects.filter(user=non_affiliated_user).exists()
+ )
+
+ @patch('admin.rdm_custom_storage_location.utils.UserQuota.objects.update_or_create')
+ def test_integrity_error_falls_back_to_filter_update(self, mock_uoc):
+ """On IntegrityError, fall back to filter().update() to set max_quota."""
+ from django.db import IntegrityError
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 200}
+ )
+ mock_uoc.side_effect = IntegrityError('duplicate key')
+
+ UserQuota.objects.create(
+ user=self.user,
+ storage_type=UserQuota.CUSTOM_STORAGE,
+ max_quota=50,
+ used=10,
+ )
+
+ upsert_user_quota_for_institution(
+ self.institution, provider_short_name='s3', old_is_nii=True
+ )
+
+ user_quota = UserQuota.objects.get(
+ user=self.user, storage_type=UserQuota.CUSTOM_STORAGE
+ )
+ nt.assert_equal(user_quota.max_quota, 200)
diff --git a/admin_tests/rdm_custom_storage_location/test_views.py b/admin_tests/rdm_custom_storage_location/test_views.py
index fac5695ae67..6385d9cd5bf 100644
--- a/admin_tests/rdm_custom_storage_location/test_views.py
+++ b/admin_tests/rdm_custom_storage_location/test_views.py
@@ -17,6 +17,8 @@
RegionFactory,
InstitutionFactory,
)
+from rest_framework import status as http_status
+from osf.models import UserQuota
class TestInstitutionDefaultStorage(AdminTestCase):
@@ -115,7 +117,19 @@ def test_get_without_custom_storage(self, *args, **kwargs):
nt.assert_equal(res.context_data['selected_provider_short_name'], 'osfstorage')
def test_get_custom_storage(self, *args, **kwargs):
- self.us = RegionFactory()
+ custom_settings = {
+ 'storage': {
+ 'provider': 'glowcloud',
+ 'container': 'osf_storage',
+ 'use_public': True,
+ 'bucket': 'bucket test',
+ 'folder': {'encrypt_uploads': 'encrypt upload test'},
+ },
+ 'extended': {'base_folder': 'base folder'},
+ 'admin_dbmid': 'abc',
+ 'team_folder_id': '1',
+ }
+ self.us = RegionFactory(waterbutler_settings=custom_settings)
self.us._id = self.institution1._id
self.us.save()
res = self.view.get(self.request, *args, **kwargs)
@@ -720,3 +734,103 @@ def test_get_superuser_institution_not_exist(self):
response = self.view_get({'provider': 'test'})
self.assertEquals(response.status_code, 404)
self.assertEquals(response.content, b'{"message": "Institution does not exist"}')
+
+
+class TestSaveCredentialsViewUpsertUserQuota(AdminTestCase):
+ """Test that SaveCredentialsView calls upsert_user_quota_for_institution after a successful save."""
+
+ def setUp(self):
+ super(TestSaveCredentialsViewUpsertUserQuota, self).setUp()
+ self.user = AuthUserFactory()
+ self.user.is_staff = True
+ self.user.is_superuser = True
+ self.user.save()
+ self.institution = InstitutionFactory()
+ self.institution_id = self.institution.id
+
+ def _post(self, params):
+ from django.test import RequestFactory
+ request = RequestFactory().post(
+ 'fake_path',
+ json.dumps(params),
+ content_type='application/json'
+ )
+ request.user = self.user
+ return views.SaveCredentialsView.as_view()(request, institution_id=self.institution_id)
+
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.upsert_user_quota_for_institution')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.add_node_settings_to_projects')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.change_allowed_for_institutions')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.update_nodes_storage')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.save_osfstorage_credentials')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.is_institution_using_nii_storage')
+ def test_upsert_user_quota_called_on_success(
+ self, mock_is_nii, mock_save, mock_update_nodes, mock_change_allowed,
+ mock_add_node, mock_upsert
+ ):
+ """upsert_user_quota_for_institution is called with correct args when save succeeds."""
+ mock_is_nii.return_value = False
+ mock_save.return_value = ({'message': 'NII storage was set successfully'}, http_status.HTTP_200_OK)
+
+ self._post({'provider_short_name': 'osfstorage'})
+
+ # old_quota_type is UserQuota.NII_STORAGE because institution has no region configured
+ mock_upsert.assert_called_once_with(self.institution, 'osfstorage', False, UserQuota.NII_STORAGE)
+
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.upsert_user_quota_for_institution')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.save_osfstorage_credentials')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.is_institution_using_nii_storage')
+ def test_upsert_user_quota_not_called_on_failure(
+ self, mock_is_nii, mock_save, mock_upsert
+ ):
+ """upsert_user_quota_for_institution is NOT called when save returns error."""
+ mock_is_nii.return_value = True
+ mock_save.return_value = ({'message': 'Some error'}, http_status.HTTP_400_BAD_REQUEST)
+
+ self._post({'provider_short_name': 'osfstorage'})
+
+ mock_upsert.assert_not_called()
+
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.upsert_user_quota_for_institution')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.add_node_settings_to_projects')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.change_allowed_for_institutions')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.update_nodes_storage')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.save_osfstorage_credentials')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.is_institution_using_nii_storage')
+ def test_old_is_nii_captured_before_save(
+ self, mock_is_nii, mock_save, mock_update_nodes, mock_change_allowed,
+ mock_add_node, mock_upsert
+ ):
+ """old_is_nii value is captured before credentials are saved and passed to upsert."""
+ mock_is_nii.return_value = True
+ mock_save.return_value = ({'message': 'NII storage was set successfully'}, http_status.HTTP_200_OK)
+
+ self._post({'provider_short_name': 'osfstorage'})
+
+ # old_quota_type is UserQuota.NII_STORAGE because institution has no region configured
+ mock_upsert.assert_called_once_with(self.institution, 'osfstorage', True, UserQuota.NII_STORAGE)
+
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.upsert_user_quota_for_institution')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.add_node_settings_to_projects')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.change_allowed_for_institutions')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.update_nodes_storage')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.save_osfstorage_credentials')
+ @mock.patch('admin.rdm_custom_storage_location.views.utils.is_institution_using_nii_storage')
+ def test_old_quota_type_captured_before_save(
+ self, mock_is_nii, mock_save, mock_update_nodes, mock_change_allowed,
+ mock_add_node, mock_upsert
+ ):
+ """old_quota_type is captured before credentials are saved and passed to upsert."""
+ from osf_tests.factories import RegionFactory
+ from addons.osfstorage.models import Region as OsfRegion
+ region = RegionFactory(_id=self.institution._id)
+ region.waterbutler_settings['storage']['type'] = OsfRegion.NII_STORAGE
+ region.save()
+
+ mock_is_nii.return_value = True
+ mock_save.return_value = ({'message': 'NII storage was set successfully'}, http_status.HTTP_200_OK)
+
+ self._post({'provider_short_name': 'osfstorage'})
+
+ # Institution has a NII region → old_quota_type = UserQuota.CUSTOM_STORAGE
+ mock_upsert.assert_called_once_with(self.institution, 'osfstorage', True, UserQuota.CUSTOM_STORAGE)
diff --git a/admin_tests/rdm_statistics/test_views.py b/admin_tests/rdm_statistics/test_views.py
index 670cfcc31d7..5f60160a58f 100644
--- a/admin_tests/rdm_statistics/test_views.py
+++ b/admin_tests/rdm_statistics/test_views.py
@@ -1,10 +1,13 @@
# -*- coding: utf-8 -*-
from nose import tools as nt
from django.test import RequestFactory
+from django.contrib.auth.models import AnonymousUser
+from django.core.exceptions import PermissionDenied
from tests.base import AdminTestCase
from osf_tests.factories import (
AuthUserFactory,
+ BookmarkCollectionFactory,
InstitutionFactory,
ProjectFactory
)
@@ -14,6 +17,7 @@
from osf.models.user import Institution
from admin.rdm_statistics import views
+from framework.auth import cron_signed_url
from mock import patch
import datetime
@@ -179,7 +183,6 @@ def test_get_context_data(self, **kwargs):
nt.assert_true('current_date' in ctx)
nt.assert_true('user' in ctx)
nt.assert_true('provider_data_array' in ctx)
- nt.assert_true('token' in ctx)
class TestImageView(AdminTestCase):
"""test ImageView"""
@@ -350,9 +353,6 @@ def tearDown(self):
self.user.delete()
self.institution1.delete()
-def test_simple_auth():
- access_key_hexa = '2a85563b2b0f7d3168199f475365f57da1d56e4bb2ce2b7044eb058ae5e287637e7c636a772682d92c8d6b1830b9a97c5a5dc3de7016c60bde4baa7cc3b38aeb'
- nt.assert_true(views.simple_auth(access_key_hexa))
def test_get_start_date():
end_date = datetime.datetime.now()
@@ -408,7 +408,8 @@ def setUp(self):
self.request = RequestFactory().get('/fake_path')
self.view = views.GatherView()
self.view = setup_user_view(self.view, self.request, user=self.user)
- self.view.kwargs = {'institution_id': self.institution1.id, 'access_token': '2A85563B2B0F7D3168199F475365F57DA1D56E4BB2CE2B7044EB058AE5E287637E7C636A772682D92C8D6B1830B9A97C5A5DC3DE7016C60BDE4BAA7CC3B38AEB'.lower()}
+ ts, signature = cron_signed_url.generate_signed_params()
+ self.view.kwargs = {'institution_id': self.institution1.id, 'ts': ts, 'signature': signature}
def tearDown(self):
super(TestGatherView, self).tearDown()
@@ -426,8 +427,14 @@ def test_get(self, *args, **kwargs):
# metadata addon is now enabled by default, so we have 3 providers
nt.assert_equal(len(resp), 3)
+ def test_get_forbidden_with_invalid_signature(self):
+ # GatherView.get() reads ts/signature from self.kwargs, set in setUp()
+ self.view.kwargs['signature'] = 'deadbeef'
+ resp = json.loads(self.view.get(self.request).content)
+ nt.assert_equal(resp['state'], 'fail')
+
def test_send_stat_mail(self, *args, **kwargs):
- nt.assert_equal(views.send_stat_mail(self.request).status_code, 200)
+ nt.assert_equal(views.send_stat_mail_core(self.request).status_code, 200)
def test_send_error_mail(self, *args, **kwargs):
ret = views.send_error_mail(Exception())
@@ -505,3 +512,99 @@ def test_get_graphs(self, mock_sessionget):
provider='osfstorage'
)
nt.assert_equal(result['content-type'], 'image/png')
+
+
+class TestSendStatMailPermission(AdminTestCase):
+ """send_stat_mail (No.86, /statistics/test/mail/) must be restricted to Integrated Admin (is_superuser)."""
+
+ def setUp(self):
+ super(TestSendStatMailPermission, self).setUp()
+ self.general_user = AuthUserFactory()
+ self.superuser = AuthUserFactory()
+ self.superuser.is_superuser = True
+ self.superuser.save()
+
+ def test_send_stat_mail_denied_for_general_user(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = self.general_user
+ with nt.assert_raises(PermissionDenied):
+ views.send_stat_mail(request)
+
+ def test_send_stat_mail_denied_for_anonymous(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = AnonymousUser()
+ with nt.assert_raises(PermissionDenied):
+ views.send_stat_mail(request)
+
+ def test_send_stat_mail_allowed_for_superuser(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = self.superuser
+ response = views.send_stat_mail(request)
+ nt.assert_equal(response.status_code, 200)
+
+ def test_send_stat_mail_denied_for_inactive_superuser(self):
+ # queryset.update() skips OSFUser.save(), which would otherwise recompute is_active
+ OSFUser.objects.filter(pk=self.superuser.pk).update(is_active=False)
+ self.superuser.refresh_from_db()
+ nt.assert_true(self.superuser.is_superuser)
+ request = RequestFactory().get('/fake_path')
+ request.user = self.superuser
+ with nt.assert_raises(PermissionDenied):
+ views.send_stat_mail(request)
+
+ def test_send_stat_mail_denied_for_unregistered_superuser(self):
+ OSFUser.objects.filter(pk=self.superuser.pk).update(is_registered=False)
+ self.superuser.refresh_from_db()
+ nt.assert_true(self.superuser.is_superuser)
+ request = RequestFactory().get('/fake_path')
+ request.user = self.superuser
+ with nt.assert_raises(PermissionDenied):
+ views.send_stat_mail(request)
+
+
+class TestIndexViewPermission(AdminTestCase):
+ """IndexView must be restricted to Integrated Admin or Institutional Admin."""
+
+ def setUp(self):
+ super(TestIndexViewPermission, self).setUp()
+ self.superuser = AuthUserFactory()
+ self.superuser.is_superuser = True
+ self.superuser.save()
+ self.institution_admin = AuthUserFactory()
+ self.institution_admin.is_staff = True
+ self.institution_admin.save()
+ self.general_user = AuthUserFactory()
+
+ def test_denied_for_general_user(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = self.general_user
+ with nt.assert_raises(PermissionDenied):
+ views.IndexView.as_view()(request)
+
+ def test_denied_for_anonymous(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = AnonymousUser()
+ with nt.assert_raises(PermissionDenied):
+ views.IndexView.as_view()(request)
+
+ def test_allowed_for_super_admin(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = self.superuser
+ view = views.IndexView()
+ view.request = request
+ nt.assert_true(view.test_func())
+
+ def test_allowed_for_institution_admin(self):
+ request = RequestFactory().get('/fake_path')
+ request.user = self.institution_admin
+ view = views.IndexView()
+ view.request = request
+ nt.assert_true(view.test_func())
+
+ def test_find_bookmark_collection(self):
+ """Collection has no 'is_deleted' field; filtering on it raised FieldError."""
+ # conftest mocks out new_bookmark_collection for speed, so the bookmark
+ # collection normally created with the user has to be added here.
+ bookmark_collection = BookmarkCollectionFactory(creator=self.superuser)
+ view = views.IndexView()
+ nt.assert_equal(view.find_bookmark_collection(self.superuser), bookmark_collection)
diff --git a/admin_tests/registration_providers/test_views.py b/admin_tests/registration_providers/test_views.py
index 9025c46b68c..e2bc66a9f1d 100644
--- a/admin_tests/registration_providers/test_views.py
+++ b/admin_tests/registration_providers/test_views.py
@@ -2,6 +2,9 @@
import pytest
from django.test import RequestFactory
+from django.contrib.auth.models import AnonymousUser, Permission
+from django.core.exceptions import PermissionDenied
+from django.http import Http404
from osf_tests.factories import (
AuthUserFactory,
@@ -207,3 +210,79 @@ def test_post(self, view, req, schema, provider):
res = view.post(req)
assert res.status_code == 302
assert provider.schemas.get(id=schema_id)
+
+
+@pytest.mark.urls('admin.base.urls')
+class TestChangeSchemaPermission:
+ """ChangeSchema must be restricted to users with 'osf.change_registrationprovider'."""
+
+ @pytest.fixture()
+ def provider(self):
+ return RegistrationProviderFactory()
+
+ def test_get_denied_for_general_user(self, req, provider):
+ with pytest.raises(PermissionDenied):
+ views.ChangeSchema.as_view()(req, registration_provider_id=provider.id)
+
+ def test_get_denied_for_anonymous(self, provider):
+ req = RequestFactory().get('/fake_path')
+ req.user = AnonymousUser()
+ with pytest.raises(PermissionDenied):
+ views.ChangeSchema.as_view()(req, registration_provider_id=provider.id)
+
+ def test_get_allowed_for_user_with_permission(self, req, user, provider):
+ permission = Permission.objects.get(codename='change_registrationprovider')
+ user.user_permissions.add(permission)
+ user.save()
+
+ res = views.ChangeSchema.as_view()(req, registration_provider_id=provider.id)
+ assert res.status_code == 200
+
+ def test_get_allowed_for_superuser(self, req, user, provider):
+ user.is_superuser = True
+ user.save()
+
+ res = views.ChangeSchema.as_view()(req, registration_provider_id=provider.id)
+ assert res.status_code == 200
+
+
+@pytest.mark.urls('admin.base.urls')
+class TestCannotDeleteProviderPermission:
+ """CannotDeleteProvider must be restricted to users with 'osf.delete_registrationprovider'."""
+
+ @pytest.fixture()
+ def provider(self):
+ return RegistrationProviderFactory()
+
+ def test_denied_for_general_user(self, req, provider):
+ with pytest.raises(PermissionDenied):
+ views.CannotDeleteProvider.as_view()(req, registration_provider_id=provider.id)
+
+ def test_denied_for_anonymous(self, provider):
+ req = RequestFactory().get('/fake_path')
+ req.user = AnonymousUser()
+ with pytest.raises(PermissionDenied):
+ views.CannotDeleteProvider.as_view()(req, registration_provider_id=provider.id)
+
+ def test_allowed_for_user_with_permission(self, req, user, provider):
+ permission = Permission.objects.get(codename='delete_registrationprovider')
+ user.user_permissions.add(permission)
+ user.save()
+
+ res = views.CannotDeleteProvider.as_view()(req, registration_provider_id=provider.id)
+ assert res.status_code == 200
+
+ def test_allowed_for_superuser(self, req, user, provider):
+ user.is_superuser = True
+ user.save()
+
+ res = views.CannotDeleteProvider.as_view()(req, registration_provider_id=provider.id)
+ assert res.status_code == 200
+
+ def test_404_for_missing_provider(self, req, user):
+ permission = Permission.objects.get(codename='delete_registrationprovider')
+ user.user_permissions.add(permission)
+ user.save()
+
+ with pytest.raises(Http404):
+ views.CannotDeleteProvider.as_view()(req, registration_provider_id=999999)
diff --git a/api_tests/nodes/views/test_node_addons.py b/api_tests/nodes/views/test_node_addons.py
index 459731af87c..660da131d84 100644
--- a/api_tests/nodes/views/test_node_addons.py
+++ b/api_tests/nodes/views/test_node_addons.py
@@ -1015,8 +1015,8 @@ def _settings_kwargs(self, node, user_settings):
def _mock_folder_result(self):
return {
'name': 'a.bucket',
- 'path': 'a.bucket',
- 'id': 'a.bucket'
+ 'path': '/',
+ 'id': 'a.bucket:/'
}
@mock.patch('addons.s3.models.get_bucket_names')
diff --git a/framework/auth/cron_signed_url.py b/framework/auth/cron_signed_url.py
new file mode 100644
index 00000000000..868e82cf526
--- /dev/null
+++ b/framework/auth/cron_signed_url.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+"""HMAC-signed, time-limited URL auth for cron-triggered admin endpoints.
+
+Used by No.1 (admin/rdm_custom_storage_location/views.py:external_acc_update)
+and No.82 (admin/rdm_statistics/views.py:GatherView), which are called by
+cron.sh scripts with no interactive user/session. A signed URL looks like
+`...///`, where `ts` is a microsecond-resolution Unix
+timestamp and `signature` is HMAC(secret, ts).
+
+Security comes from two independent checks, both required:
+ 1. authenticity - signature must match HMAC(CRON_SIGNED_URL_SECRET, ts)
+ 2. freshness - ts must be within CRON_SIGNED_URL_TTL_SECONDS of "now"
+
+Timestamp precision (microseconds) does not itself add security - it is a
+customer requirement, not a defense mechanism. Both checks above are what
+actually make an intercepted URL become useless after CRON_SIGNED_URL_TTL_SECONDS.
+"""
+import hmac
+import time
+
+from website import settings
+
+
+def _sign(ts):
+ return hmac.new(
+ key=settings.CRON_SIGNED_URL_SECRET.encode('utf-8'),
+ msg=str(ts).encode('utf-8'),
+ digestmod=settings.CRON_SIGNED_URL_HMAC_ALGORITHM,
+ ).hexdigest()
+
+
+def generate_signed_params():
+ """Returns (ts, signature) as strings, ready to drop into a URL path.
+
+ ts is a microsecond-resolution Unix timestamp (str), per customer
+ requirement (see module docstring).
+ """
+ ts = str(int(time.time() * 1000000))
+ return ts, _sign(ts)
+
+
+def verify_signed_params(ts, signature, ttl_seconds=None):
+ """Verifies a (ts, signature) pair produced by generate_signed_params().
+
+ Returns True only if signature matches ts AND ts is within ttl_seconds
+ of now (in either direction). Returns False for any malformed input
+ instead of raising.
+ """
+ if ttl_seconds is None:
+ ttl_seconds = settings.CRON_SIGNED_URL_TTL_SECONDS
+ if not ts or not signature:
+ return False
+ try:
+ ts_us = int(ts)
+ except (TypeError, ValueError):
+ return False
+ expected_signature = _sign(ts)
+ if not hmac.compare_digest(expected_signature, signature.lower()):
+ return False
+ now_us = int(time.time() * 1000000)
+ ttl_us = int(ttl_seconds * 1000000)
+ return abs(now_us - ts_us) <= ttl_us
diff --git a/osf/management/commands/generate_signed_cron_token.py b/osf/management/commands/generate_signed_cron_token.py
new file mode 100644
index 00000000000..98c09cab48f
--- /dev/null
+++ b/osf/management/commands/generate_signed_cron_token.py
@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+from django.core.management.base import BaseCommand
+
+from framework.auth.cron_signed_url import generate_signed_params
+
+
+class Command(BaseCommand):
+ """Prints "/" for building a B(2) signed cron URL.
+
+ Used by admin/rdm_statistics/cron.sh (No.82) and
+ admin/rdm_custom_storage_location/cron.sh (No.1) to mint a fresh
+ signed URL segment immediately before each curl call. See
+ framework.auth.cron_signed_url for the verification side.
+ """
+ help = 'Print "/" for a B(2) signed cron URL.'
+
+ def handle(self, *args, **options):
+ ts, signature = generate_signed_params()
+ self.stdout.write('{}/{}'.format(ts, signature))
diff --git a/osf/migrations/0272_institutiondefaultmaxquota.py b/osf/migrations/0272_institutiondefaultmaxquota.py
new file mode 100644
index 00000000000..92a4cc68639
--- /dev/null
+++ b/osf/migrations/0272_institutiondefaultmaxquota.py
@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.28 on 2026-07-23 16:23
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+import osf.models.base
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('osf', '0271_merge_20260622_1735'),
+ ]
+
+ operations = [
+ migrations.CreateModel(
+ name='InstitutionDefaultMaxQuota',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
+ ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+ ('default_max_quota', models.IntegerField(default=100)),
+ ('institution', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='default_max_quota', to='osf.Institution')),
+ ],
+ options={
+ 'db_table': 'osf_institution_default_max_quota',
+ },
+ bases=(models.Model, osf.models.base.QuerySetExplainMixin),
+ ),
+ ]
diff --git a/osf/migrations/0273_merge_20260922_0854.py b/osf/migrations/0273_merge_20260922_0854.py
new file mode 100644
index 00000000000..bd357db51de
--- /dev/null
+++ b/osf/migrations/0273_merge_20260922_0854.py
@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.28 on 2026-09-22 08:54
+from __future__ import unicode_literals
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('osf', '0272_merge_20260713_1135_'),
+ ('osf', '0272_institutiondefaultmaxquota'),
+ ]
+
+ operations = [
+ ]
diff --git a/osf/models/__init__.py b/osf/models/__init__.py
index 36b22471194..8de4216a335 100644
--- a/osf/models/__init__.py
+++ b/osf/models/__init__.py
@@ -64,6 +64,7 @@
from osf.models.project_storage_type import ProjectStorageType # noqa
from osf.models.region_external_account import RegionExternalAccount # noqa
from osf.models.institution_entitlement import InstitutionEntitlement # noqa
+from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota # noqa
from osf.models.export_data_location import ExportDataLocation # noqa
from osf.models.export_data import ExportData # noqa
from osf.models.export_data_restore import ExportDataRestore # noqa
diff --git a/osf/models/files.py b/osf/models/files.py
index a53cc81f0ef..cdb8d42d838 100644
--- a/osf/models/files.py
+++ b/osf/models/files.py
@@ -557,7 +557,7 @@ def update(self, revision, data, user=None, save=True):
return version
def serialize(self):
- newest_version = self.versions.all().last()
+ newest_version = self.versions.all().first()
if not newest_version:
return dict(self._serialize(), **{
@@ -571,6 +571,8 @@ def serialize(self):
'locked': self.locked._id if self.locked else None,
})
+ oldest_version = self.versions.all().last()
+
return dict(self._serialize(), **{
'size': newest_version.size,
'downloads': self.get_download_count(),
@@ -579,7 +581,7 @@ def serialize(self):
'version': newest_version.identifier if newest_version else None,
'contentType': newest_version.content_type if newest_version else None,
'modified': newest_version.external_modified.isoformat() if newest_version.external_modified else None,
- 'created': self.versions.all().first().external_modified.isoformat() if self.versions.all().first().external_modified else None,
+ 'created': oldest_version.external_modified.isoformat() if oldest_version and oldest_version.external_modified else None,
})
def restore(self, recursive=True, parent=None, save=True, deleted_on=None):
diff --git a/osf/models/institution_default_max_quota.py b/osf/models/institution_default_max_quota.py
new file mode 100644
index 00000000000..8b18e9f7c1c
--- /dev/null
+++ b/osf/models/institution_default_max_quota.py
@@ -0,0 +1,29 @@
+from django.db import models
+
+from osf.models import Institution
+from osf.models.base import BaseModel
+
+
+class InstitutionDefaultMaxQuota(BaseModel):
+ institution = models.OneToOneField(Institution, related_name='default_max_quota', on_delete=models.CASCADE)
+ default_max_quota = models.IntegerField(default=100)
+
+ class Meta:
+ db_table = 'osf_institution_default_max_quota'
+
+ @classmethod
+ def get_quota_by_user(cls, user_id):
+ """
+ Get the default maximum quota for the user's affiliated institution.
+
+ Args:
+ user_id (int): The ID of the user whose affiliated institution's default quota is to be retrieved.
+
+ Returns:
+ int | None: The default maximum quota (in GB) for the user's affiliated institution.
+ Returns None if the user is not affiliated with an institution or if no default quota is configured.
+ """
+ return cls.objects.filter(
+ institution__osfuser__id=user_id,
+ institution__is_deleted=False
+ ).extra(order_by=['osf_osfuser_affiliated_institutions.id']).values_list('default_max_quota', flat=True).first()
diff --git a/osf_tests/management_commands/test_generate_signed_cron_token.py b/osf_tests/management_commands/test_generate_signed_cron_token.py
new file mode 100644
index 00000000000..821d616c75c
--- /dev/null
+++ b/osf_tests/management_commands/test_generate_signed_cron_token.py
@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+import io
+import pytest
+from django.core.management import call_command
+
+from framework.auth import cron_signed_url
+
+
+@pytest.mark.django_db
+class TestGenerateSignedCronToken:
+
+ def test_prints_ts_slash_signature(self):
+ out = io.StringIO()
+ call_command('generate_signed_cron_token', stdout=out)
+ output = out.getvalue().strip()
+ assert output.count('/') == 1
+ ts, signature = output.split('/')
+ assert ts.isdigit()
+ assert len(signature) > 0
+
+ def test_printed_pair_verifies_successfully(self):
+ out = io.StringIO()
+ call_command('generate_signed_cron_token', stdout=out)
+ ts, signature = out.getvalue().strip().split('/')
+ assert cron_signed_url.verify_signed_params(ts, signature)
diff --git a/osf_tests/test_files.py b/osf_tests/test_files.py
index c8adac024ba..e8cdb9caa79 100644
--- a/osf_tests/test_files.py
+++ b/osf_tests/test_files.py
@@ -1,4 +1,7 @@
+import datetime
+
import pytest
+from django.utils import timezone
from django.contrib.contenttypes.models import ContentType
@@ -8,7 +11,8 @@
from osf_tests.factories import (
UserFactory,
ProjectFactory,
- RegionFactory
+ RegionFactory,
+ FileVersionFactory,
)
pytestmark = pytest.mark.django_db
@@ -92,3 +96,51 @@ def test_file_update_respects_region(project, user, create_test_file):
)
assert new_region != original_region
assert new_version.region == new_region
+
+
+def test_file_serialize_no_versions(project):
+ from addons.googledrive.models import GoogleDriveFile
+
+ # GoogleDriveFile uses File.serialize directly (no override)
+ gd_file = GoogleDriveFile(name='empty.txt', target=project)
+ gd_file.save()
+
+ result = gd_file.serialize()
+
+ # newest_version is None → all version-derived fields are None
+ assert result['size'] is None
+ assert result['version'] is None
+ assert result['modified'] is None
+ assert result['created'] is None
+ assert result['contentType'] is None
+
+
+def test_file_serialize_newest_version(project):
+ from addons.googledrive.models import GoogleDriveFile
+
+ # GoogleDriveFile uses File.serialize directly (no override)
+ gd_file = GoogleDriveFile(name='multi_version.txt', target=project)
+ gd_file.save()
+
+ # Create v1 first (older DB timestamp), smaller size
+ older_time = timezone.now() - datetime.timedelta(days=1)
+ v1 = FileVersionFactory(identifier='1', size=1000)
+ v1.external_modified = older_time
+ v1.save()
+ gd_file.add_version(v1)
+
+ # Create v2 second (newer DB timestamp), larger size
+ newer_time = timezone.now()
+ v2 = FileVersionFactory(identifier='2', size=5000)
+ v2.external_modified = newer_time
+ v2.save()
+ gd_file.add_version(v2)
+
+ result = gd_file.serialize()
+
+ # newest_version = versions.all().first() ordered by -created → v2
+ assert result['size'] == 5000
+ assert result['version'] == '2'
+ assert result['modified'] == newer_time.isoformat()
+ # created comes from oldest_version (versions.all().last()) → v1
+ assert result['created'] == older_time.isoformat()
diff --git a/requirements.txt b/requirements.txt
index c9ee19ae23b..f3574223512 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -133,3 +133,5 @@ Babel==2.5.1
Flask-Babel==1.0.0
rsa==3.1.4
flask-compress==1.15
+
+dataclasses==0.8
\ No newline at end of file
diff --git a/scripts/migration/migrate_fix_fileinfo_quota.py b/scripts/migration/migrate_fix_fileinfo_quota.py
new file mode 100644
index 00000000000..73464b870d5
--- /dev/null
+++ b/scripts/migration/migrate_fix_fileinfo_quota.py
@@ -0,0 +1,461 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+migrate_fix_fileinfo_quota.py
+
+Fix missing or zero-size FileInfo records and recalculate UserQuota.
+
+Background
+----------
+During Move/Copy operations, if a system error occurs after the file has been
+written but before the FileInfo hook completes, the resulting OsfStorageFile may
+have no FileInfo row (or a row with file_size=0). This causes UserQuota.used to
+be under-counted for the affected project creator.
+
+This script:
+1. Finds all non-deleted OsfStorageFile nodes that are missing a FileInfo record
+ OR have a FileInfo record with file_size=0.
+2. Populates file_size from the latest FileVersion.size (which is authoritative).
+3. Collects all unique project creators affected by the above.
+4. Calls update_user_used_quota(user) for each affected user to re-sync
+ UserQuota.used from the actual FileInfo totals.
+ Storage type is determined per-project (NII_STORAGE or CUSTOM_STORAGE).
+5. Exports a CSV report of all processed records.
+
+Error Handling (fail-fast):
+ The script fails fast. As soon as it hits a record that cannot be fully fixed —
+ its size cannot be determined, or its project creator / storage_type cannot be
+ resolved — or any unexpected error, it logs THAT single record and raises
+ immediately, so an administrator can fix that record and re-run. It does NOT
+ scan the whole dataset to collect and print a list of every problem. This
+ behaviour is identical in dry-run and in a real run.
+
+ All DB updates run inside a single transaction (transaction.atomic). If an error
+ is raised part-way through a real run, every change is rolled back and the
+ database is left untouched (no partially-applied fixes).
+
+ Note on file size: size is read from the latest FileVersion. A genuine
+ file_size == 0 is a valid size and a FileInfo record with file_size=0 is
+ created/updated; only an indeterminable size (None) is treated as an error.
+
+Usage:
+ # Fix and export results to CSV
+ python -m scripts.migration.migrate_fix_fileinfo_quota
+ python -m scripts.migration.migrate_fix_fileinfo_quota --output result.csv
+
+ # Preview changes and export to CSV without writing to DB (dry run)
+ python -m scripts.migration.migrate_fix_fileinfo_quota --dry
+ python -m scripts.migration.migrate_fix_fileinfo_quota --dry --output preview.csv
+"""
+
+import os
+import csv
+import logging
+import argparse
+from datetime import datetime
+
+import django
+
+# Setup Django before importing models
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'api.base.settings')
+django.setup()
+
+from django.db import transaction
+from django.db.models import Q
+
+from addons.osfstorage.models import OsfStorageFile
+from osf.models import FileInfo, OSFUser
+from website.util.quota import update_user_used_quota, get_project_storage_type
+from scripts import utils as script_utils
+from django.contrib.contenttypes.models import ContentType
+
+logger = logging.getLogger(__name__)
+
+RESULT_FIELDNAMES = [
+ # osf_file_id : _id (hex string) of the OsfStorageFile / BaseFileNode record
+ # fileinfo_id : integer PK of the FileInfo record inserted/updated (empty in dry_run)
+ # fileinfo__id : _id (hex string) of the FileInfo record inserted/updated (empty in dry_run)
+ # fileinfo_file_id: integer PK of BaseFileNode — the value stored as file_id (FK) in the FileInfo table
+ # fileinfo_created: True = new FileInfo row INSERT-ed, False = existing row UPDATE-d (empty in dry_run)
+ # action : would_update (dry-run) | created | updated
+ # quota_recalculated: True if quota was recalculated for this file's creator
+ 'osf_file_id', 'file_name',
+ 'fileinfo_id', 'fileinfo__id', 'fileinfo_file_id',
+ 'file_size_bytes', 'project_guid', 'creator_id', 'storage_type',
+ 'fileinfo_created', 'action', 'quota_recalculated', 'dry_run',
+]
+
+
+# --- Scanning Operations ---
+
+def get_broken_file_nodes():
+ """Return queryset of OsfStorageFile nodes that:
+ - are not deleted (deleted_on is None)
+ - either have no FileInfo, or have FileInfo with file_size == 0
+
+ Optimized using a LEFT OUTER JOIN query to avoid subquery bottlenecks with large datasets.
+ """
+ return (
+ OsfStorageFile.objects
+ .filter(deleted_on=None, deleted_by_id=None)
+ .filter(Q(fileinfo__isnull=True) | Q(fileinfo__file_size=0))
+ # NOTE: Do NOT use prefetch_related('target') here.
+ # BaseFileNode uses TypedModel + GenericForeignKey; Django's GFK prefetch cache
+ # is incompatible with TypedModel proxy resolution and returns None even when
+ # target_content_type_id is valid. Use direct lookup instead (see resolve_target).
+ )
+
+
+def _bulk_get_latest_version_sizes(osf_file_ids):
+ """Fetch the latest FileVersion size for each OsfStorageFile in bulk.
+ Returns a dict mapping osf_file.id -> size (int or None).
+ """
+ if not osf_file_ids:
+ return {}
+
+ # Access the M2M through-model (BaseFileVersionsThrough) without importing it directly.
+ # Fields: basefilenode_id (FK → BaseFileNode), fileversion_id (FK → FileVersion)
+ ThroughModel = OsfStorageFile.versions.through
+
+ # Single JOIN query: fetches all through-entries + related FileVersion data at once.
+ entries = (
+ ThroughModel.objects
+ .filter(basefilenode_id__in=osf_file_ids)
+ .select_related('fileversion')
+ )
+
+ # Group by file and pick the version with the highest numeric identifier.
+ # result maps: file_node_id → (version_number_int, size)
+ best = {} # file_node_id -> (ver_num, size)
+ for entry in entries:
+ fv = entry.fileversion
+ file_node_id = entry.basefilenode_id
+
+ # identifier is a text field ('1', '2', ...) — parse to int for correct ordering
+ try:
+ ver_num = int(fv.identifier)
+ except (TypeError, ValueError):
+ ver_num = -1 # non-numeric identifiers rank last
+
+ if file_node_id not in best or ver_num > best[file_node_id][0]:
+ # Resolve size: prefer fv.size, fall back to metadata dict
+ size = fv.size
+ if size is None and isinstance(fv.metadata, dict):
+ try:
+ size = int(fv.metadata.get('size'))
+ except (TypeError, ValueError):
+ size = None
+ best[file_node_id] = (ver_num, size)
+
+ return {file_id: info[1] for file_id, info in best.items()}
+
+
+def scan_broken_files():
+ """Scan for OsfStorageFile nodes with missing or zero-size FileInfo.
+
+ Fails fast: the moment a record is found that cannot be fully fixed (size cannot
+ be determined, or its project creator / storage_type cannot be resolved), this
+ raises so the administrator can fix that one record and re-run. No DB writes.
+ Returns a list of fully-fixable row dicts when every record is OK.
+ """
+ broken_files = list(get_broken_file_nodes())
+ total = len(broken_files)
+ logger.info('Scanning OsfStorageFile records...')
+ logger.info('Found {} file node(s) with missing or zero-size FileInfo.'.format(total))
+
+ # Bulk-fetch latest version sizes to avoid N+1 queries
+ file_ids = [f.id for f in broken_files]
+ size_map = _bulk_get_latest_version_sizes(file_ids)
+
+ rows = []
+ for osf_file in broken_files:
+ size = size_map.get(osf_file.id)
+
+ # Fail fast: the size must be determinable. (A genuine file_size == 0 is a
+ # valid size and is NOT an error — only an indeterminable size is.)
+ if size is None:
+ raise RuntimeError(
+ 'File {} (_id={}) — could not determine size from any FileVersion. '
+ 'Fix this record and re-run.'.format(osf_file.name, osf_file._id)
+ )
+
+ # Resolve the project target, then its creator / guid / storage_type.
+ project_guid = ''
+ creator_id = ''
+ storage_type_value = None
+ try:
+ # Direct lookup via ContentType to avoid GFK+TypedModel prefetch incompatibility.
+ # osf_file.target (GFK accessor) returns None when used after prefetch_related
+ # because TypedModel proxy resolution breaks the GFK prefetch cache.
+ target = None
+ if osf_file.target_content_type_id and osf_file.target_object_id:
+ ct = ContentType.objects.get_for_id(osf_file.target_content_type_id)
+ model_class = ct.model_class()
+ if model_class:
+ target = model_class.objects.filter(id=osf_file.target_object_id).first()
+
+ if target is not None:
+ first_guid = target.guids.values_list('_id', flat=True).first()
+ project_guid = first_guid or ''
+ creator = getattr(target, 'creator', None)
+ if creator:
+ creator_id = creator._id
+ storage_type_value = get_project_storage_type(target)
+ except Exception as e:
+ # Fail fast: do not swallow unexpected errors.
+ logger.error('Could not resolve target/creator for file {}: {} — aborting (fail fast).'.format(
+ osf_file._id, e
+ ))
+ raise
+
+ # Fail fast: creator and storage_type must be resolvable to recalculate quota.
+ if not creator_id:
+ raise RuntimeError(
+ 'File {} (_id={}, project_guid={}) — creator could not be resolved '
+ '(target missing or has no creator). Fix this record and re-run.'.format(
+ osf_file.name, osf_file._id, project_guid
+ )
+ )
+ if storage_type_value is None:
+ raise RuntimeError(
+ 'File {} (_id={}, project_guid={}) — project storage_type could not be '
+ 'determined. Fix this record and re-run.'.format(
+ osf_file.name, osf_file._id, project_guid
+ )
+ )
+
+ rows.append({
+ 'osf_file_id': osf_file._id, # hex _id string of OsfStorageFile
+ 'fileinfo_file_id': osf_file.id, # integer PK stored as file_id FK in FileInfo table
+ 'file_name': osf_file.name,
+ 'file_size_bytes': size,
+ 'project_guid': project_guid,
+ 'creator_id': creator_id,
+ 'storage_type': storage_type_value,
+ 'osf_file': osf_file,
+ })
+
+ logger.info('Scan complete. {} record(s) ready to fix.'.format(len(rows)))
+ return rows
+
+
+def print_results(rows):
+ """Display scanned records in a formatted summary table."""
+ if not rows:
+ print(' No broken FileInfo records found.')
+ return
+
+ def _disp(value):
+ # Display helper: never let None (or any value) break str formatting.
+ return '' if value is None else str(value)
+
+ print('\n' + '-' * 160)
+ print(' {:<26} {:<16} {:<22} {:<18} {:<16} {:<14} {:<18}'.format(
+ 'OSF_FILE_ID (_id)', 'FILEINFO_FILE_ID', 'FILE_NAME', 'PROJECT_GUID', 'CREATOR_ID', 'STORAGE_TYPE', 'SIZE (bytes)'
+ ))
+ print('-' * 160)
+ for r in rows:
+ print(' {:<26} {:<16} {:<22} {:<18} {:<16} {:<14} {:<18}'.format(
+ _disp(r.get('osf_file_id')),
+ _disp(r.get('fileinfo_file_id')),
+ _disp(r.get('file_name'))[:20],
+ _disp(r.get('project_guid')),
+ _disp(r.get('creator_id')),
+ _disp(r.get('storage_type')),
+ _disp(r.get('file_size_bytes')),
+ ))
+ print('-' * 160)
+ print(' Total: {} record(s) would be updated.\n'.format(len(rows)))
+
+
+# --- Updating Operations ---
+
+def apply_fixes(rows, dry_run=False):
+ """Apply FileInfo fixes and recalculate UserQuota for affected users.
+
+ This function wraps all database modifications in a single database transaction
+ when dry_run is False. If any unhandled exception occurs, the transaction is
+ rolled back, ensuring database consistency.
+
+ If dry_run is True, no database writes are performed.
+ Returns the list of result rows with 'action', 'quota_recalculated', and 'dry_run' fields added.
+ """
+ if dry_run:
+ logger.info('[DRY RUN] Simulating FileInfo updates (no DB writes)...')
+ return _apply_fixes_internal(rows, dry_run=True)
+ else:
+ logger.info('Applying FileInfo updates (wrapped in database transaction)...')
+ with transaction.atomic():
+ return _apply_fixes_internal(rows, dry_run=False)
+
+
+def _apply_fixes_internal(rows, dry_run=False):
+ """Internal helper to apply fixes, called inside/outside transaction.
+
+ Every row here is already fully fixable (scan_broken_files fails fast otherwise),
+ so each row gets a FileInfo update and contributes to a UserQuota recalculation.
+ """
+ result_rows = []
+ affected_user_storage = set() # set of (creator_id, storage_type)
+ fixed = 0
+
+ for row in rows:
+ # scan stores the osf_file object on the row; fall back to a lookup just in case.
+ osf_file = row.get('osf_file')
+ if osf_file is None:
+ osf_file = OsfStorageFile.objects.filter(_id=row['osf_file_id']).first()
+ if osf_file is None:
+ # Fail fast: a scanned row must resolve to a file.
+ raise RuntimeError('File {} not found when applying fix.'.format(row['osf_file_id']))
+
+ fileinfo_id = ''
+ fileinfo__id = ''
+ fileinfo_created = ''
+ if not dry_run:
+ fileinfo_obj, created = FileInfo.objects.update_or_create(
+ file=osf_file,
+ defaults={'file_size': row['file_size_bytes']},
+ )
+ fileinfo_id = fileinfo_obj.id # integer PK of FileInfo
+ fileinfo__id = fileinfo_obj._id # hex _id string of FileInfo
+ fileinfo_created = created # True = INSERT, False = UPDATE
+
+ creator_id = row['creator_id']
+ storage_type = row['storage_type']
+ affected_user_storage.add((creator_id, storage_type))
+
+ action = 'would_update' if dry_run else ('created' if fileinfo_created else 'updated')
+
+ logger.info('{} {} (osf_file_id={}, fileinfo_id={}, fileinfo__id={}, fileinfo_file_id={}, created={}) file_size={} bytes storage_type={}'.format(
+ '[DRY RUN]' if dry_run else 'Fixed:',
+ row['file_name'], row['osf_file_id'],
+ fileinfo_id, fileinfo__id, row['fileinfo_file_id'],
+ fileinfo_created, row['file_size_bytes'], storage_type,
+ ))
+
+ result_rows.append({
+ 'osf_file_id': row['osf_file_id'],
+ 'fileinfo_file_id': row['fileinfo_file_id'],
+ 'file_name': row['file_name'],
+ 'file_size_bytes': row['file_size_bytes'],
+ 'project_guid': row['project_guid'],
+ 'creator_id': row['creator_id'],
+ 'storage_type': row['storage_type'],
+ 'fileinfo_id': fileinfo_id,
+ 'fileinfo__id': fileinfo__id,
+ 'fileinfo_created': fileinfo_created,
+ 'action': action,
+ 'quota_recalculated': False, # will be updated below after quota recalc
+ 'dry_run': dry_run,
+ })
+ fixed += 1
+
+ logger.info('FileInfo backfill complete. fixed={} (dry={})'.format(fixed, dry_run))
+
+ # --- Recalculate UserQuota.used for all affected (user, storage_type) pairs ---
+ logger.info('Recalculating UserQuota.used for {} unique (user, storage_type) pair(s).'.format(
+ len(affected_user_storage)
+ ))
+
+ quota_updated = 0
+ recalculated_user_storage = set()
+
+ for user_id, storage_type in sorted(affected_user_storage):
+ try:
+ user = OSFUser.load(user_id)
+ if user is None:
+ # Fail fast: a creator_id resolved during scan must load here.
+ raise RuntimeError(
+ 'User {} (storage_type={}) could not be loaded for quota recalc.'.format(
+ user_id, storage_type
+ )
+ )
+
+ if not dry_run:
+ update_user_used_quota(user, storage_type=storage_type, is_recalculating_quota=True)
+
+ logger.info('{} Recalculated quota for user {} (storage_type={})'.format(
+ '[DRY RUN]' if dry_run else '', user_id, storage_type
+ ))
+ quota_updated += 1
+ recalculated_user_storage.add((user_id, storage_type))
+ except Exception as e:
+ # Fail fast: re-raise so transaction.atomic() rolls back ALL FileInfo updates.
+ logger.error('Failed to recalculate quota for user {} (storage_type={}): {} — '
+ 'rolling back transaction (fail fast).'.format(user_id, storage_type, e))
+ raise
+
+ logger.info('Quota recalculation complete. updated={} (dry={})'.format(quota_updated, dry_run))
+
+ # Update quota_recalculated flag in result rows
+ for row in result_rows:
+ if dry_run:
+ row['quota_recalculated'] = 'would_recalculate'
+ elif (row['creator_id'], row['storage_type']) in recalculated_user_storage:
+ row['quota_recalculated'] = True
+
+ return result_rows
+
+
+# --- CSV Export ---
+
+def save_result_csv(result_rows, output_path):
+ """Save full results to CSV file."""
+ with open(output_path, 'w', newline='', encoding='utf-8-sig') as f:
+ writer = csv.DictWriter(f, fieldnames=RESULT_FIELDNAMES)
+ writer.writeheader()
+ writer.writerows(result_rows)
+ total = len(result_rows)
+ print('\n Exported {} result(s) to: {}'.format(total, output_path))
+
+
+# --- Main ---
+
+def main():
+ ts = datetime.now().strftime('%Y%m%d_%H%M%S')
+ default_output = f'migrate_fix_fileinfo_quota_{ts}.csv'
+
+ parser = argparse.ArgumentParser(
+ description='Fix missing/zero-size FileInfo records and recalculate UserQuota.',
+ )
+ parser.add_argument(
+ '--output', '-o',
+ default=default_output,
+ help=f'Output CSV file path (default: {default_output})',
+ )
+ parser.add_argument(
+ '--dry', action='store_true', default=False,
+ help='Run in dry-run mode (do not commit updates to the database).',
+ )
+
+ args = parser.parse_args()
+
+ if not args.dry:
+ script_utils.add_file_logger(logger, __file__)
+
+ # Configure console logging format
+ logging.basicConfig(
+ level=logging.INFO,
+ format='%(asctime)s %(levelname)s %(message)s',
+ )
+
+ # 1. Scan for broken records
+ rows = scan_broken_files()
+
+ if not rows:
+ print('No broken FileInfo records found.')
+ return
+
+ # 2. Print summary table
+ print_results(rows)
+
+ # 3. Apply fixes (or simulate in dry-run mode). scan_broken_files() has already
+ # failed fast on any unfixable record, so every row here is fully fixable.
+ result_rows = apply_fixes(rows, dry_run=args.dry)
+
+ # 4. Export results to CSV
+ save_result_csv(result_rows, args.output)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/tests/test_addons.py b/tests/test_addons.py
index e6f18031b56..2254a03fa75 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -25,7 +25,7 @@
DraftRegistrationFactory,
ExportDataLocationFactory,
)
-from website import settings
+from website import settings, mails
from api.base import settings as api_settings
from addons.base import views
from addons.github.exceptions import ApiError
@@ -113,12 +113,33 @@ def test_auth_download(self):
data = jwt.decode(jwe.decrypt(res.json['payload'].encode('utf-8'), self.JWE_KEY), settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data']
assert_equal(data['auth'], views.make_auth(self.user))
assert_equal(data['credentials'], self.node_addon.serialize_waterbutler_credentials())
- assert_equal(data['settings'], self.node_addon.serialize_waterbutler_settings())
+ expected_settings = self.node_addon.serialize_waterbutler_settings().copy()
+ expected_settings['max_file_size'] = 100
+ assert_equal(data['settings'], expected_settings)
expected_url = furl.furl(self.node.api_url_for('create_waterbutler_log', _absolute=True, _internal=True))
observed_url = furl.furl(data['callback_url'])
observed_url.port = expected_url.port
assert_equal(expected_url, observed_url)
+ def test_auth_max_file_size_honors_high_upload_limit_tag(self):
+ """A user with the `high_upload_limit` system tag gets the high max_file_size
+ sent to WaterButler."""
+ self.user.add_system_tag('high_upload_limit')
+ self.user.save()
+ url = self.build_url()
+ with mock.patch.object(self.node_addon.config, 'high_max_file_size', 500, create=True):
+ res = self.app.get(url, auth=self.user.auth)
+ data = jwt.decode(jwe.decrypt(res.json['payload'].encode('utf-8'), self.JWE_KEY), settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data']
+ assert_equal(data['settings']['max_file_size'], 500)
+
+ def test_auth_max_file_size_without_high_upload_limit_tag(self):
+ """An untagged user keeps the addon's ordinary limit even when a high limit exists."""
+ url = self.build_url()
+ with mock.patch.object(self.node_addon.config, 'high_max_file_size', 500, create=True):
+ res = self.app.get(url, auth=self.user.auth)
+ data = jwt.decode(jwe.decrypt(res.json['payload'].encode('utf-8'), self.JWE_KEY), settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data']
+ assert_equal(data['settings']['max_file_size'], 100)
+
def test_auth_render_action_returns_200(self):
url = self.build_url(action='render')
res = self.app.get(url, auth=self.user.auth)
@@ -153,7 +174,9 @@ def test_auth_bad_cookie(self):
data = jwt.decode(jwe.decrypt(res.json['payload'].encode('utf-8'), self.JWE_KEY), settings.WATERBUTLER_JWT_SECRET, algorithm=settings.WATERBUTLER_JWT_ALGORITHM)['data']
assert_equal(data['auth'], views.make_auth(self.user))
assert_equal(data['credentials'], self.node_addon.serialize_waterbutler_credentials())
- assert_equal(data['settings'], self.node_addon.serialize_waterbutler_settings())
+ expected_settings = self.node_addon.serialize_waterbutler_settings().copy()
+ expected_settings['max_file_size'] = 100
+ assert_equal(data['settings'], expected_settings)
expected_url = furl.furl(self.node.api_url_for('create_waterbutler_log', _absolute=True, _internal=True))
observed_url = furl.furl(data['callback_url'])
observed_url.port = expected_url.port
@@ -1019,6 +1042,99 @@ def test_add_folder_osfstorage_log(self, mock_basefilenode):
assert_equal(self.node.logs.count(), nlogs + 1)
assert('urls' not in self.node.logs.filter(action='osf_storage_file_added')[0].params)
+ @mock.patch('website.mails.send_mail')
+ def test_create_waterbutler_log_quota_exceeded_error(self, mock_send_mail):
+ url = self.node.api_url_for('create_waterbutler_log')
+ payload = self.build_payload(
+ action='move',
+ metadata={'path': 'pizza'},
+ source={
+ 'provider': 'github',
+ 'materialized': '/src.txt',
+ 'path': '/src.txt',
+ 'name': 'src.txt',
+ 'nid': self.node._id,
+ },
+ destination={
+ 'provider': 'github',
+ 'materialized': '/dest.txt',
+ 'path': '/dest.txt',
+ 'name': 'dest.txt',
+ 'nid': self.node._id,
+ },
+ errors=['']
+ )
+ res = self.app.put_json(url, payload, headers={'Content-Type': 'application/json'})
+ assert_equal(res.status_code, 200)
+ assert_true(mock_send_mail.called)
+ call_args = mock_send_mail.call_args[1]
+ assert_equal(call_args['error_info'], {'type': 'quota_exceeded'})
+ assert_equal(mock_send_mail.call_args[0][1], mails.FILE_OPERATION_FAILED)
+
+ @mock.patch('website.mails.send_mail')
+ def test_create_waterbutler_log_oversized_error(self, mock_send_mail):
+ url = self.node.api_url_for('create_waterbutler_log')
+ payload = self.build_payload(
+ action='move',
+ metadata={'path': 'pizza'},
+ source={
+ 'provider': 'github',
+ 'materialized': '/src.txt',
+ 'path': '/src.txt',
+ 'name': 'src.txt',
+ 'nid': self.node._id,
+ },
+ destination={
+ 'provider': 'github',
+ 'materialized': '/dest.txt',
+ 'path': '/dest.txt',
+ 'name': 'dest.txt',
+ 'nid': self.node._id,
+ },
+ errors=['']
+ )
+ res = self.app.put_json(url, payload, headers={'Content-Type': 'application/json'})
+ assert_equal(res.status_code, 200)
+ assert_true(mock_send_mail.called)
+ call_args = mock_send_mail.call_args[1]
+ assert_equal(call_args['error_info'], {
+ 'type': 'oversized',
+ 'oversized_files': [{'name': 'large.zip', 'size': 2000000000}],
+ 'max_size': 1000000000
+ })
+ assert_equal(mock_send_mail.call_args[0][1], mails.FILE_OPERATION_FAILED)
+
+ @mock.patch('addons.base.views.logger.warning')
+ @mock.patch('website.mails.send_mail')
+ def test_create_waterbutler_log_unparseable_error(self, mock_send_mail, mock_logger_warning):
+ url = self.node.api_url_for('create_waterbutler_log')
+ payload = self.build_payload(
+ action='move',
+ metadata={'path': 'pizza'},
+ source={
+ 'provider': 'github',
+ 'materialized': '/src.txt',
+ 'path': '/src.txt',
+ 'name': 'src.txt',
+ 'nid': self.node._id,
+ },
+ destination={
+ 'provider': 'github',
+ 'materialized': '/dest.txt',
+ 'path': '/dest.txt',
+ 'name': 'dest.txt',
+ 'nid': self.node._id,
+ },
+ errors=['']
+ )
+ res = self.app.put_json(url, payload, headers={'Content-Type': 'application/json'})
+ assert_equal(res.status_code, 200)
+ assert_true(mock_send_mail.called)
+ assert_true(mock_logger_warning.called)
+ call_args = mock_send_mail.call_args[1]
+ assert_equal(call_args['error_info'], None)
+ assert_equal(mock_send_mail.call_args[0][1], mails.FILE_OPERATION_FAILED)
+
class TestAddonLogsDifferentProvider(OsfTestCase):
def setUp(self):
diff --git a/tests/test_cron_signed_url.py b/tests/test_cron_signed_url.py
new file mode 100644
index 00000000000..d77169a13ce
--- /dev/null
+++ b/tests/test_cron_signed_url.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+import time
+import mock
+from nose.tools import * # noqa (PEP8 asserts)
+
+from framework.auth import cron_signed_url
+
+
+class TestCronSignedUrl:
+
+ def setup_method(self, method):
+ self.secret_patch = mock.patch('website.settings.CRON_SIGNED_URL_SECRET', 'test-secret')
+ self.ttl_patch = mock.patch('website.settings.CRON_SIGNED_URL_TTL_SECONDS', 100)
+ self.secret_patch.start()
+ self.ttl_patch.start()
+
+ def teardown_method(self, method):
+ self.secret_patch.stop()
+ self.ttl_patch.stop()
+
+ def test_generate_then_verify_succeeds(self):
+ ts, signature = cron_signed_url.generate_signed_params()
+ assert_true(cron_signed_url.verify_signed_params(ts, signature))
+
+ def test_ts_is_microsecond_precision(self):
+ before_us = int(time.time() * 1000000)
+ ts, _ = cron_signed_url.generate_signed_params()
+ after_us = int(time.time() * 1000000)
+ assert_true(before_us <= int(ts) <= after_us)
+
+ def test_tampered_signature_is_rejected(self):
+ ts, signature = cron_signed_url.generate_signed_params()
+ flipped_char = '0' if signature[0] != '0' else '1'
+ tampered = flipped_char + signature[1:]
+ assert_false(cron_signed_url.verify_signed_params(ts, tampered))
+
+ def test_tampered_timestamp_is_rejected(self):
+ ts, signature = cron_signed_url.generate_signed_params()
+ tampered_ts = str(int(ts) + 1)
+ assert_false(cron_signed_url.verify_signed_params(tampered_ts, signature))
+
+ def test_expired_timestamp_is_rejected(self):
+ with mock.patch('time.time', return_value=time.time() - 200):
+ ts, signature = cron_signed_url.generate_signed_params()
+ assert_false(cron_signed_url.verify_signed_params(ts, signature))
+
+ def test_future_timestamp_beyond_ttl_is_rejected(self):
+ with mock.patch('time.time', return_value=time.time() + 200):
+ ts, signature = cron_signed_url.generate_signed_params()
+ assert_false(cron_signed_url.verify_signed_params(ts, signature))
+
+ def test_non_numeric_timestamp_is_rejected(self):
+ assert_false(cron_signed_url.verify_signed_params('not-a-number', 'deadbeef'))
+
+ def test_missing_signature_is_rejected(self):
+ ts, _ = cron_signed_url.generate_signed_params()
+ assert_false(cron_signed_url.verify_signed_params(ts, ''))
+
+ def test_wrong_secret_is_rejected(self):
+ ts, signature = cron_signed_url.generate_signed_params()
+ with mock.patch('website.settings.CRON_SIGNED_URL_SECRET', 'a-different-secret'):
+ assert_false(cron_signed_url.verify_signed_params(ts, signature))
diff --git a/tests/test_institution_default_max_quota.py b/tests/test_institution_default_max_quota.py
new file mode 100644
index 00000000000..af02b5a0d63
--- /dev/null
+++ b/tests/test_institution_default_max_quota.py
@@ -0,0 +1,54 @@
+# -*- coding: utf-8 -*-
+from nose import tools as nt
+
+from osf.models import InstitutionDefaultMaxQuota
+from osf_tests.factories import AuthUserFactory, InstitutionFactory
+from tests.base import OsfTestCase
+
+
+class TestInstitutionDefaultMaxQuota(OsfTestCase):
+
+ def test_get_quota_by_user_has_institution(self):
+ user = AuthUserFactory()
+ institution = InstitutionFactory()
+
+ user.affiliated_institutions.add(institution)
+
+ InstitutionDefaultMaxQuota.objects.create(
+ institution=institution,
+ default_max_quota=500
+ )
+
+ result = InstitutionDefaultMaxQuota.get_quota_by_user(user.id)
+
+ nt.assert_equal(result, 500)
+
+ def test_get_quota_by_user_no_institution(self):
+ user = AuthUserFactory()
+
+ result = InstitutionDefaultMaxQuota.get_quota_by_user(user.id)
+
+ nt.assert_equal(result, None)
+
+ def test_get_quota_by_user_multiple_institutions(self):
+ user = AuthUserFactory()
+
+ inst1 = InstitutionFactory()
+ inst2 = InstitutionFactory()
+
+ user.affiliated_institutions.add(inst1)
+ user.affiliated_institutions.add(inst2)
+
+ InstitutionDefaultMaxQuota.objects.create(
+ institution=inst1,
+ default_max_quota=100
+ )
+
+ InstitutionDefaultMaxQuota.objects.create(
+ institution=inst2,
+ default_max_quota=200
+ )
+
+ result = InstitutionDefaultMaxQuota.get_quota_by_user(user.id)
+
+ nt.assert_equal(result, 100)
diff --git a/tests/test_oauth.py b/tests/test_oauth.py
index 74ce6195df1..af4002d5bf7 100644
--- a/tests/test_oauth.py
+++ b/tests/test_oauth.py
@@ -38,6 +38,7 @@
)
from django.test import RequestFactory
from admin.rdm_custom_storage_location import views as customstoragelocation_update
+from framework.auth import cron_signed_url
SILENT_LOGGERS = ['oauthlib', 'requests_oauthlib']
@@ -1050,11 +1051,12 @@ def test_callback_with_institution(self, mock_drive_refresh):
assert_equal(account.oauth_key, 'mock_access_token')
assert_equal(account.provider_id, 'mock_provider_id')
req = RequestFactory().get('http://localhost:8001/customstoragelocation/external_acc_update/')
- res = customstoragelocation_update.external_acc_update(req,access_token='d610ef95f0b0f5868f13919b8ed64070b9acb9c19b8da9f2c514ed938203ec3e236c9cad4f4146bdf22b4e79cf0d92f6d4f4c996d236c6b0ee79a1336b26afb7')
- assert_equal(res.status_code,200)
+ ts, signature = cron_signed_url.generate_signed_params()
+ res = customstoragelocation_update.external_acc_update(req, ts=ts, signature=signature)
+ assert_equal(res.status_code, 200)
assert_equal(res.content.decode(), 'Done')
- res = customstoragelocation_update.external_acc_update(req,access_token='b610ef95f0b0f5868f13919b8ed64070b9acb9c19b8da9f2c514ed938203ec3e236c9cad4f4146bdf22b4e79cf0d92f6d4f4c996d236c6b0ee79a1336b26afb7')
- assert_equal(res.status_code,200)
+ res = customstoragelocation_update.external_acc_update(req, ts=ts, signature='deadbeef')
+ assert_equal(res.status_code, 200)
assert_not_equal(res.content.decode(), 'Done')
@responses.activate
diff --git a/tests/test_quota.py b/tests/test_quota.py
index 2fc183213d0..d5b1efc339a 100644
--- a/tests/test_quota.py
+++ b/tests/test_quota.py
@@ -4,7 +4,7 @@
from nose.tools import * # noqa (PEP8 asserts)
import pytest
-from addons.osfstorage.models import OsfStorageFileNode
+from addons.osfstorage.models import OsfStorageFileNode, Region
from api.base import settings as api_settings
from framework.auth import signing
from tests.base import OsfTestCase
@@ -16,6 +16,7 @@
)
from osf.utils.requests import check_select_for_update
from website.util import web_url_for, quota
+from django.db import IntegrityError
@pytest.mark.enable_implicit_clean
@@ -268,7 +269,7 @@ def setUp(self):
)
self.file.save()
- def test_add_file_info(self):
+ def test_file_info_skipped_for_osfstorage_signal(self):
file_info_query = FileInfo.objects.filter(file=self.file)
assert_false(file_info_query.exists())
@@ -294,9 +295,8 @@ def test_add_file_info(self):
)
file_info_list = FileInfo.objects.filter(file=self.file).all()
- assert_equal(file_info_list.count(), 1)
- file_info = file_info_list.first()
- assert_equal(file_info.file_size, 1000)
+ # FileInfo was not created
+ assert_equal(file_info_list.count(), 0)
def test_update_file_info(self):
file_info = FileInfo(file=self.file, file_size=1000)
@@ -386,7 +386,7 @@ def setUp(self):
target_content_type_id=2
)
- def test_add_first_file(self):
+ def test_first_file_userquota_not_created_for_osfstorage_signal(self):
assert_false(UserQuota.objects.filter(user=self.project_creator).exists())
quota.update_used_quota(
@@ -414,9 +414,7 @@ def test_add_first_file(self):
storage_type=UserQuota.NII_STORAGE,
user=self.project_creator
).all()
- assert_equal(len(user_quota), 1)
- user_quota = user_quota[0]
- assert_equal(user_quota.used, 1000)
+ assert_equal(len(user_quota), 0)
def test_add_first_file_custom_storage(self):
assert_false(UserQuota.objects.filter(user=self.project_creator).exists())
@@ -453,11 +451,9 @@ def test_add_first_file_custom_storage(self):
storage_type=UserQuota.CUSTOM_STORAGE,
user=self.project_creator
).all()
- assert_equal(len(user_quota), 1)
- user_quota = user_quota[0]
- assert_equal(user_quota.used, 1200)
+ assert_equal(len(user_quota), 0)
- def test_add_file(self):
+ def test_userquota_unchanged_for_osfstorage_signal(self):
UserQuota.objects.create(
user=self.project_creator,
storage_type=UserQuota.NII_STORAGE,
@@ -492,7 +488,7 @@ def test_add_file(self):
).all()
assert_equal(len(user_quota), 1)
user_quota = user_quota[0]
- assert_equal(user_quota.used, 6500)
+ assert_equal(user_quota.used, 5500)
def test_add_file_custom_storage(self):
UserQuota.objects.create(
@@ -536,7 +532,7 @@ def test_add_file_custom_storage(self):
).all()
assert_equal(len(user_quota), 1)
user_quota = user_quota[0]
- assert_equal(user_quota.used, 6700)
+ assert_equal(user_quota.used, 5500)
def test_add_file_negative_size(self):
quota.update_used_quota(
@@ -545,9 +541,9 @@ def test_add_file_negative_size(self):
user=self.user,
event_type=FileLog.FILE_ADDED,
payload={
- 'provider': 'osfstorage',
+ 'provider': 's3compatinstitutions',
'metadata': {
- 'provider': 'osfstorage',
+ 'provider': 's3compatinstitutions',
'name': 'testfile',
'materialized': '/filename',
'path': self.file._id,
@@ -599,6 +595,51 @@ def test_delete_file(self):
)
assert_equal(user_quota.used, 4500)
+ @mock.patch('website.util.quota.check_select_for_update')
+ def test_delete_file_without_select_for_update(self, mock_check_select_for_update):
+ # Forces node_removed() into the `else` branch (FileInfo.objects.get
+ # without select_for_update), instead of relying on whether the test
+ # happens to run inside an atomic transaction.
+ mock_check_select_for_update.return_value = False
+
+ UserQuota.objects.create(
+ user=self.project_creator,
+ storage_type=UserQuota.NII_STORAGE,
+ max_quota=api_settings.DEFAULT_MAX_QUOTA,
+ used=5500
+ )
+ FileInfo.objects.create(file=self.file, file_size=1000)
+
+ self.file.deleted_on = datetime.datetime.now()
+ self.file.deleted_by = self.user
+ self.file.type = 'osf.trashedfile'
+ self.file.save()
+
+ quota.update_used_quota(
+ self=None,
+ target=self.node,
+ user=self.user,
+ event_type=FileLog.FILE_REMOVED,
+ payload={
+ 'provider': 'osfstorage',
+ 'metadata': {
+ 'provider': 'osfstorage',
+ 'name': 'testfile',
+ 'materialized': '/filename',
+ 'path': self.file._id,
+ 'kind': 'file',
+ 'extra': {}
+ }
+ }
+ )
+
+ mock_check_select_for_update.assert_called()
+ user_quota = UserQuota.objects.get(
+ storage_type=UserQuota.NII_STORAGE,
+ user=self.project_creator
+ )
+ assert_equal(user_quota.used, 4500)
+
def test_delete_file_custom_storage(self):
UserQuota.objects.create(
user=self.project_creator,
@@ -987,9 +1028,9 @@ def test_edit_file_negative_size(self):
user=self.user,
event_type=FileLog.FILE_UPDATED,
payload={
- 'provider': 'osfstorage',
+ 'provider': 's3compatinstitutions',
'metadata': {
- 'provider': 'osfstorage',
+ 'provider': 's3compatinstitutions',
'name': 'testfile',
'materialized': '/filename',
'path': self.file._id,
@@ -1358,7 +1399,7 @@ def test_delete_file_with_Amazon_S3_Compatible_Storage_for_Institution(self):
if check_select_for_update():
mock_file_info.objects.filter.return_value.select_for_update.return_value.get.return_value = FileInfo(
file=self.base_file_node, file_size=1000)
- mock_user_quota.objects.filter.return_value.select_for_update.return_value.first.return_value = UserQuota(
+ mock_user_quota.objects.filter.return_value.select_for_update.return_value.get.return_value = UserQuota(
user=self.project_creator,
storage_type=UserQuota.CUSTOM_STORAGE,
max_quota=api_settings.DEFAULT_MAX_QUOTA,
@@ -1366,12 +1407,14 @@ def test_delete_file_with_Amazon_S3_Compatible_Storage_for_Institution(self):
)
else:
mock_file_info.objects.get.return_value = FileInfo(file=self.base_file_node, file_size=1000)
- mock_user_quota.objects.filter.return_value.first.return_value = UserQuota(
+ _real_user_quota = UserQuota(
user=self.project_creator,
storage_type=UserQuota.CUSTOM_STORAGE,
max_quota=api_settings.DEFAULT_MAX_QUOTA,
used=5500
)
+ mock_user_quota.objects.get.return_value = _real_user_quota
+ mock_user_quota.objects.filter.return_value.first.return_value = _real_user_quota
with mock.patch('website.util.quota.BaseFileNode', mock_base_file_node):
with mock.patch('website.util.quota.FileInfo', mock_file_info):
with mock.patch('website.util.quota.UserQuota', mock_user_quota):
@@ -1414,7 +1457,7 @@ def test_delete_folder_with_Amazon_S3_Compatible_Storage_for_Institution(self):
if check_select_for_update():
mock_file_info.objects.filter.return_value.select_for_update.return_value.get.return_value = FileInfo(
file=self.base_file_node, file_size=1500)
- mock_user_quota.objects.filter.return_value.select_for_update.return_value.first.return_value = UserQuota(
+ mock_user_quota.objects.filter.return_value.select_for_update.return_value.get.return_value = UserQuota(
user=self.project_creator,
storage_type=UserQuota.CUSTOM_STORAGE,
max_quota=api_settings.DEFAULT_MAX_QUOTA,
@@ -1422,12 +1465,14 @@ def test_delete_folder_with_Amazon_S3_Compatible_Storage_for_Institution(self):
)
else:
mock_file_info.objects.get.return_value = FileInfo(file=self.base_file_node, file_size=1500)
- mock_user_quota.objects.filter.return_value.first.return_value = UserQuota(
+ _real_user_quota = UserQuota(
user=self.project_creator,
storage_type=UserQuota.CUSTOM_STORAGE,
max_quota=api_settings.DEFAULT_MAX_QUOTA,
used=5500
)
+ mock_user_quota.objects.get.return_value = _real_user_quota
+ mock_user_quota.objects.filter.return_value.first.return_value = _real_user_quota
with mock.patch('website.util.quota.BaseFileNode', mock_base_file_node):
with mock.patch('website.util.quota.FileInfo', mock_file_info):
with mock.patch('website.util.quota.UserQuota', mock_user_quota):
@@ -1532,6 +1577,36 @@ def test_update_user_used_quota_method_for_recalculate_quota_process__nii_custom
user_quota = user_quota.first()
assert_equal(user_quota.used, 1000)
+ @mock.patch('website.util.quota.check_select_for_update', return_value=False)
+ @mock.patch('website.util.quota.used_quota')
+ def test_update_user_used_quota_without_select_for_update_existing(self, mock_used, _mock_check):
+ """update_user_used_quota should work without select_for_update when UserQuota exists"""
+ mock_used.return_value = 750
+
+ quota.update_user_used_quota(
+ user=self.user,
+ storage_type=UserQuota.NII_STORAGE
+ )
+
+ user_quota = UserQuota.objects.get(user=self.user, storage_type=UserQuota.NII_STORAGE)
+ assert_equal(user_quota.used, 750)
+
+ @mock.patch('website.util.quota.check_select_for_update', return_value=False)
+ @mock.patch('website.util.quota.used_quota')
+ def test_update_user_used_quota_without_select_for_update_create(self, mock_used, _mock_check):
+ """update_user_used_quota should work without select_for_update when creating new UserQuota"""
+ another_user = UserFactory()
+ mock_used.return_value = 600
+
+ quota.update_user_used_quota(
+ user=another_user,
+ storage_type=UserQuota.NII_STORAGE
+ )
+
+ user_quota = UserQuota.objects.get(user=another_user, storage_type=UserQuota.NII_STORAGE)
+ assert_equal(user_quota.used, 600)
+ assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+
class TestQuotaApiWaterbutler(OsfTestCase):
def setUp(self):
@@ -1551,6 +1626,20 @@ def test_default_values(self):
assert_equal(response.json['max'], api_settings.DEFAULT_MAX_QUOTA * api_settings.SIZE_UNIT_GB)
assert_equal(response.json['used'], 0)
+ def test_user_guid_and_storage_type_present_and_consistent_with_creator(self):
+ response = self.app.get(
+ '{}?payload={payload}&signature={signature}'.format(
+ self.node.api_url_for('waterbutler_creator_quota'),
+ **signing.sign_data(signing.default_signer, {})
+ )
+ )
+ assert_equal(response.status_code, 200)
+ assert_in('user_guid', response.json)
+ assert_in('storage_type', response.json)
+ from website.util import quota as quota_util
+ assert_equal(response.json['user_guid'], self.node.creator._id)
+ assert_equal(response.json['storage_type'], quota_util.get_project_storage_type(self.node))
+
def test_used_half_custom_quota(self):
UserQuota.objects.create(
storage_type=UserQuota.NII_STORAGE,
@@ -1617,6 +1706,15 @@ def test_default_values(self):
assert_equal(response.json['max'], api_settings.DEFAULT_MAX_QUOTA * api_settings.SIZE_UNIT_GB)
assert_equal(response.json['used'], 0)
+ def test_user_guid_and_storage_type_present_and_consistent_with_creator(self):
+ response = self.app.get(
+ self.node.api_url_for('get_creator_quota'),
+ auth=self.user.auth
+ )
+ assert_equal(response.status_code, 200)
+ assert_in('user_guid', response.json)
+ assert_in('storage_type', response.json)
+
def test_used_half_custom_quota(self):
UserQuota.objects.create(
storage_type=UserQuota.NII_STORAGE,
@@ -1661,3 +1759,755 @@ def test_used_half_custom_institution_quota(self):
assert_equal(response.status_code, 200)
assert_equal(response.json['max'], 200 * api_settings.SIZE_UNIT_GB)
assert_equal(response.json['used'], 100 * api_settings.SIZE_UNIT_GB)
+
+
+@pytest.mark.enable_implicit_clean
+class TestGetQuotaInfoWithInstitutionDefaultMaxQuota(OsfTestCase):
+ """Test get_quota_info() with InstitutionDefaultMaxQuota logic"""
+
+ def setUp(self):
+ super(TestGetQuotaInfoWithInstitutionDefaultMaxQuota, self).setUp()
+ self.user = AuthUserFactory()
+ self.institution = InstitutionFactory(name='Test Institution')
+ self.user.affiliated_institutions.add(self.institution)
+
+ @mock.patch('website.util.quota.used_quota')
+ def test_get_quota_info_nii_storage_uses_default(self, mock_used_quota):
+ """NII_STORAGE should always use DEFAULT_MAX_QUOTA, ignoring InstitutionDefaultMaxQuota"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+ mock_used_quota.return_value = 50
+
+ # Set institution quota
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 500}
+ )
+
+ # NII_STORAGE should ignore institution quota
+ max_quota, used = quota.get_quota_info(self.user, UserQuota.NII_STORAGE)
+
+ assert_equal(max_quota, api_settings.DEFAULT_MAX_QUOTA)
+ assert_equal(used, 50)
+
+ @mock.patch('website.util.quota.used_quota')
+ def test_get_quota_info_custom_storage_with_institution_quota(self, mock_used_quota):
+ """CUSTOM_STORAGE should use InstitutionDefaultMaxQuota when available"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+ mock_used_quota.return_value = 100
+
+ # Set institution quota
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 350}
+ )
+
+ # CUSTOM_STORAGE should use institution quota
+ max_quota, used = quota.get_quota_info(self.user, UserQuota.CUSTOM_STORAGE)
+
+ assert_equal(max_quota, 350)
+ assert_equal(used, 100)
+
+ @mock.patch('website.util.quota.used_quota')
+ def test_get_quota_info_custom_storage_fallback_to_default(self, mock_used_quota):
+ """CUSTOM_STORAGE should fallback to DEFAULT_MAX_QUOTA when no InstitutionDefaultMaxQuota"""
+ mock_used_quota.return_value = 75
+
+ # No InstitutionDefaultMaxQuota set
+ max_quota, used = quota.get_quota_info(self.user, UserQuota.CUSTOM_STORAGE)
+
+ assert_equal(max_quota, api_settings.DEFAULT_MAX_QUOTA)
+ assert_equal(used, 75)
+
+ @mock.patch('website.util.quota.used_quota')
+ def test_get_quota_info_with_existing_userquota(self, mock_used_quota):
+ """get_quota_info should return existing UserQuota, not use InstitutionDefaultMaxQuota"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+ mock_used_quota.return_value = 0
+
+ # Create UserQuota with custom max_quota
+ UserQuota.objects.create(
+ user=self.user,
+ storage_type=UserQuota.CUSTOM_STORAGE,
+ max_quota=600,
+ used=200
+ )
+
+ # Set institution quota (should be ignored)
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 350}
+ )
+
+ # Should return existing UserQuota, not institution quota
+ max_quota, used = quota.get_quota_info(self.user, UserQuota.CUSTOM_STORAGE)
+
+ assert_equal(max_quota, 600)
+ assert_equal(used, 200)
+
+
+@pytest.mark.enable_implicit_clean
+class TestFileAddedWithInstitutionDefaultMaxQuota(OsfTestCase):
+ """Test file_added() with InstitutionDefaultMaxQuota logic"""
+
+ def setUp(self):
+ super(TestFileAddedWithInstitutionDefaultMaxQuota, self).setUp()
+ self.user = AuthUserFactory()
+ self.institution = InstitutionFactory(name='Test Institution')
+ self.region = RegionFactory(_id=self.institution._id)
+ self.region.waterbutler_settings['storage']['type'] = Region.INSTITUTIONS
+ self.region.save()
+ self.user.affiliated_institutions.add(self.institution)
+ self.project = ProjectFactory(creator=self.user, is_deleted=False)
+ self.file_node = mock.MagicMock()
+
+ def tearDown(self):
+ super(TestFileAddedWithInstitutionDefaultMaxQuota, self).tearDown()
+ if hasattr(self, 'project') and self.project:
+ self.project.delete()
+
+ @mock.patch('osf.models.FileInfo.objects.create')
+ @mock.patch('website.util.quota.used_quota')
+ def test_file_added_custom_storage_with_institution_quota(self, mock_used_quota, mock_create):
+ """file_added should create UserQuota with InstitutionDefaultMaxQuota value"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+ mock_used_quota.return_value = 0
+
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 400}
+ )
+
+ payload = {'metadata': {'size': '100'}}
+ quota.file_added(self.project, payload, self.file_node, UserQuota.CUSTOM_STORAGE)
+
+ user_quota = UserQuota.objects.get(
+ user=self.project.creator,
+ storage_type=UserQuota.CUSTOM_STORAGE
+ )
+ assert_equal(user_quota.max_quota, 400)
+ assert_equal(user_quota.used, 100)
+
+ @mock.patch('osf.models.FileInfo.objects.create')
+ @mock.patch('website.util.quota.used_quota')
+ def test_file_added_custom_storage_without_institution_quota(self, mock_used_quota, mock_create):
+ """file_added should fallback to DEFAULT_MAX_QUOTA when no InstitutionDefaultMaxQuota"""
+ mock_used_quota.return_value = 0
+
+ # No InstitutionDefaultMaxQuota
+ payload = {'metadata': {'size': '150'}}
+ quota.file_added(self.project, payload, self.file_node, UserQuota.CUSTOM_STORAGE)
+
+ user_quota = UserQuota.objects.get(
+ user=self.project.creator,
+ storage_type=UserQuota.CUSTOM_STORAGE
+ )
+ assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+ assert_equal(user_quota.used, 150)
+
+ @mock.patch('osf.models.FileInfo.objects.create')
+ @mock.patch('website.util.quota.used_quota')
+ def test_file_added_nii_storage_ignores_institution_quota(self, mock_used_quota, mock_create):
+ """file_added should use DEFAULT_MAX_QUOTA for NII_STORAGE, ignoring InstitutionDefaultMaxQuota"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+ mock_used_quota.return_value = 0
+
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 400}
+ )
+
+ payload = {'metadata': {'size': '75'}}
+ quota.file_added(self.project, payload, self.file_node, UserQuota.NII_STORAGE)
+
+ user_quota = UserQuota.objects.get(
+ user=self.project.creator,
+ storage_type=UserQuota.NII_STORAGE
+ )
+ assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+ assert_equal(user_quota.used, 75)
+
+
+@pytest.mark.enable_implicit_clean
+class TestUpdateUserUsedQuotaWithInstitutionDefaultMaxQuota(OsfTestCase):
+ """Test update_user_used_quota() with InstitutionDefaultMaxQuota logic"""
+
+ def setUp(self):
+ super(TestUpdateUserUsedQuotaWithInstitutionDefaultMaxQuota, self).setUp()
+ self.user = UserFactory()
+ self.user.save()
+ self.institution = InstitutionFactory(name='Test Institution')
+ self.user.affiliated_institutions.add(self.institution)
+
+ @mock.patch('website.util.quota.used_quota')
+ def test_update_user_used_quota_create_custom_storage_with_institution_quota(self, mock_used_quota):
+ """update_user_used_quota should create UserQuota with InstitutionDefaultMaxQuota value"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+ mock_used_quota.return_value = 300
+
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 500}
+ )
+
+ quota.update_user_used_quota(self.user, UserQuota.CUSTOM_STORAGE)
+
+ user_quota = UserQuota.objects.get(
+ user=self.user,
+ storage_type=UserQuota.CUSTOM_STORAGE
+ )
+ assert_equal(user_quota.max_quota, 500)
+ assert_equal(user_quota.used, 300)
+
+ @mock.patch('website.util.quota.used_quota')
+ def test_update_user_used_quota_create_custom_storage_without_institution_quota(self, mock_used_quota):
+ """update_user_used_quota should fallback to DEFAULT_MAX_QUOTA when no InstitutionDefaultMaxQuota"""
+ mock_used_quota.return_value = 250
+
+ # No InstitutionDefaultMaxQuota
+ quota.update_user_used_quota(self.user, UserQuota.CUSTOM_STORAGE)
+
+ user_quota = UserQuota.objects.get(
+ user=self.user,
+ storage_type=UserQuota.CUSTOM_STORAGE
+ )
+ assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+ assert_equal(user_quota.used, 250)
+
+ @mock.patch('website.util.quota.used_quota')
+ def test_update_user_used_quota_nii_storage_ignores_institution_quota(self, mock_used_quota):
+ """update_user_used_quota should use DEFAULT_MAX_QUOTA for NII_STORAGE"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+ mock_used_quota.return_value = 200
+
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 500}
+ )
+
+ quota.update_user_used_quota(self.user, UserQuota.NII_STORAGE)
+
+ user_quota = UserQuota.objects.get(
+ user=self.user,
+ storage_type=UserQuota.NII_STORAGE
+ )
+ assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+ assert_equal(user_quota.used, 200)
+
+
+class TestGetDefaultMaxQuota(OsfTestCase):
+ """Unit tests for get_default_max_quota()"""
+
+ def setUp(self):
+ super(TestGetDefaultMaxQuota, self).setUp()
+ self.user = AuthUserFactory()
+ self.institution = InstitutionFactory(name='Test Institution')
+ self.user.affiliated_institutions.add(self.institution)
+
+ def _make_non_nii_region(self):
+ region = RegionFactory(_id=self.institution._id)
+ region.waterbutler_settings['storage']['type'] = Region.INSTITUTIONS
+ region.save()
+ return region
+
+ def _make_nii_region(self):
+ region = RegionFactory(_id=self.institution._id)
+ region.waterbutler_settings['storage']['type'] = Region.NII_STORAGE
+ region.save()
+ return region
+
+ def test_nii_storage_returns_default(self):
+ """NII_STORAGE always returns DEFAULT_MAX_QUOTA regardless of institution quota"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+ self._make_non_nii_region()
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 500}
+ )
+
+ result = quota.get_default_max_quota(self.user, UserQuota.NII_STORAGE)
+
+ assert_equal(result, api_settings.DEFAULT_MAX_QUOTA)
+
+ def test_custom_storage_no_institution_returns_default(self):
+ """CUSTOM_STORAGE returns DEFAULT_MAX_QUOTA when user has no affiliated institution"""
+ user_no_inst = AuthUserFactory()
+
+ result = quota.get_default_max_quota(user_no_inst, UserQuota.CUSTOM_STORAGE)
+
+ assert_equal(result, api_settings.DEFAULT_MAX_QUOTA)
+
+ def test_custom_storage_institution_uses_nii_storage_returns_default(self):
+ """CUSTOM_STORAGE returns DEFAULT_MAX_QUOTA when institution's region is NII_STORAGE"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+ self._make_nii_region()
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 500}
+ )
+
+ result = quota.get_default_max_quota(self.user, UserQuota.CUSTOM_STORAGE)
+
+ assert_equal(result, api_settings.DEFAULT_MAX_QUOTA)
+
+ def test_custom_storage_no_institution_quota_returns_default(self):
+ """CUSTOM_STORAGE returns DEFAULT_MAX_QUOTA when no InstitutionDefaultMaxQuota is set"""
+ self._make_non_nii_region()
+ # No InstitutionDefaultMaxQuota created
+
+ result = quota.get_default_max_quota(self.user, UserQuota.CUSTOM_STORAGE)
+
+ assert_equal(result, api_settings.DEFAULT_MAX_QUOTA)
+
+ def test_custom_storage_with_institution_quota_returns_institution_value(self):
+ """CUSTOM_STORAGE returns institution quota when region is non-NII and quota is set"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+ self._make_non_nii_region()
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=self.institution.id,
+ defaults={'default_max_quota': 350}
+ )
+
+ result = quota.get_default_max_quota(self.user, UserQuota.CUSTOM_STORAGE)
+
+ assert_equal(result, 350)
+
+
+@pytest.mark.enable_implicit_clean
+class TestFileAddedWithoutSelectForUpdate(OsfTestCase):
+ """Test file_added() without select_for_update branch"""
+
+ def setUp(self):
+ super(TestFileAddedWithoutSelectForUpdate, self).setUp()
+ self.user = UserFactory()
+ self.project_creator = UserFactory()
+ self.node = ProjectFactory(creator=self.project_creator)
+ self.file = OsfStorageFileNode.create(
+ target=self.node,
+ path='/testfile',
+ _id='testfile',
+ name='testfile',
+ materialized_path='/testfile'
+ )
+ self.file.save()
+
+ @mock.patch('website.util.quota.check_select_for_update', return_value=False)
+ def test_file_added_without_select_for_update_new_userquota(self, _mock_check):
+ """file_added should work without select_for_update when creating new UserQuota"""
+ assert_false(UserQuota.objects.filter(user=self.project_creator).exists())
+
+ payload = {'metadata': {'size': 1000}}
+ quota.file_added(self.node, payload, self.file, UserQuota.NII_STORAGE)
+
+ user_quota = UserQuota.objects.get(user=self.project_creator, storage_type=UserQuota.NII_STORAGE)
+ assert_equal(user_quota.used, 1000)
+ assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+
+ @mock.patch('website.util.quota.check_select_for_update', return_value=False)
+ def test_file_added_without_select_for_update_existing_userquota(self, _mock_check):
+ """file_added should work without select_for_update when updating existing UserQuota"""
+ UserQuota.objects.create(
+ user=self.project_creator,
+ storage_type=UserQuota.NII_STORAGE,
+ max_quota=500,
+ used=2000
+ )
+
+ payload = {'metadata': {'size': 500}}
+ quota.file_added(self.node, payload, self.file, UserQuota.NII_STORAGE)
+
+ user_quota = UserQuota.objects.get(user=self.project_creator, storage_type=UserQuota.NII_STORAGE)
+ assert_equal(user_quota.used, 2500)
+ assert_equal(user_quota.max_quota, 500)
+
+
+@pytest.mark.enable_implicit_clean
+class TestNodeRemovedWithoutSelectForUpdate(OsfTestCase):
+ """Test node_removed() without select_for_update branch"""
+
+ def setUp(self):
+ super(TestNodeRemovedWithoutSelectForUpdate, self).setUp()
+ self.user = UserFactory()
+ self.project_creator = UserFactory()
+ self.node = ProjectFactory(creator=self.project_creator)
+ self.file = OsfStorageFileNode.create(
+ target=self.node,
+ path='/testfile',
+ _id='testfile',
+ name='testfile',
+ materialized_path='/testfile'
+ )
+ self.file.save()
+
+ @mock.patch('website.util.quota.check_select_for_update', return_value=False)
+ def test_node_removed_without_select_for_update(self, _mock_check):
+ """node_removed should work without select_for_update when deleting file"""
+ UserQuota.objects.create(
+ user=self.project_creator,
+ storage_type=UserQuota.NII_STORAGE,
+ max_quota=api_settings.DEFAULT_MAX_QUOTA,
+ used=1500
+ )
+ FileInfo.objects.create(file=self.file, file_size=1000)
+
+ self.file.deleted_on = datetime.datetime.now()
+ self.file.deleted_by = self.user
+ self.file.type = 'osf.trashedfile'
+ self.file.save()
+
+ quota.node_removed(self.node, self.user, {}, self.file, UserQuota.NII_STORAGE)
+
+ user_quota = UserQuota.objects.get(user=self.project_creator, storage_type=UserQuota.NII_STORAGE)
+ assert_equal(user_quota.used, 500)
+
+
+@pytest.mark.enable_implicit_clean
+class TestUserQuotaGetOrCreate(OsfTestCase):
+ """Test get_or_create logic in file_modified() function"""
+
+ def setUp(self):
+ super(TestUserQuotaGetOrCreate, self).setUp()
+ self.user = UserFactory()
+ self.project_creator = UserFactory()
+ self.node = ProjectFactory(creator=self.project_creator)
+ self.file = OsfStorageFileNode.create(
+ target=self.node,
+ path='/testfile',
+ _id='testfile',
+ name='testfile',
+ materialized_path='/testfile'
+ )
+ self.file.save()
+
+ def test_get_or_create_creates_new_userquota(self):
+ """get_or_create should create new UserQuota if it doesn't exist"""
+ assert_false(UserQuota.objects.filter(user=self.project_creator).exists())
+
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 5000
+ }
+ },
+ file_node=self.file,
+ storage_type=UserQuota.NII_STORAGE
+ )
+
+ user_quota = UserQuota.objects.filter(user=self.project_creator, storage_type=UserQuota.NII_STORAGE)
+ assert_equal(len(user_quota), 1)
+ user_quota = user_quota[0]
+ assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+ assert_equal(user_quota.used, 5000)
+
+ def test_get_or_create_retrieves_existing_userquota(self):
+ """get_or_create should retrieve existing UserQuota without creating a new one"""
+ UserQuota.objects.create(
+ user=self.project_creator,
+ storage_type=UserQuota.NII_STORAGE,
+ max_quota=200,
+ used=100
+ )
+
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 200
+ }
+ },
+ file_node=self.file,
+ storage_type=UserQuota.NII_STORAGE
+ )
+
+ user_quota_list = UserQuota.objects.filter(user=self.project_creator, storage_type=UserQuota.NII_STORAGE)
+ assert_equal(len(user_quota_list), 1)
+ user_quota = user_quota_list[0]
+ assert_equal(user_quota.max_quota, 200)
+ assert_equal(user_quota.used, 300)
+
+ def test_get_or_create_with_custom_storage_new_userquota(self):
+ """get_or_create should work correctly with CUSTOM_STORAGE"""
+ institution = InstitutionFactory()
+ self.project_creator.affiliated_institutions.add(institution)
+ RegionFactory(_id=institution._id)
+ ProjectStorageType.objects.filter(node=self.node).update(
+ storage_type=ProjectStorageType.CUSTOM_STORAGE
+ )
+
+ assert_false(UserQuota.objects.filter(user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE).exists())
+
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 3000
+ }
+ },
+ file_node=self.file,
+ storage_type=UserQuota.CUSTOM_STORAGE
+ )
+
+ user_quota = UserQuota.objects.get(user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE)
+ assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+ assert_equal(user_quota.used, 3000)
+
+ def test_get_or_create_with_default_max_quota_logic(self):
+ """get_or_create should use get_default_max_quota() for max_quota value"""
+ from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
+
+ institution = InstitutionFactory()
+ self.project_creator.affiliated_institutions.add(institution)
+ RegionFactory(_id=institution._id)
+ ProjectStorageType.objects.filter(node=self.node).update(
+ storage_type=ProjectStorageType.CUSTOM_STORAGE
+ )
+
+ custom_max = 450
+ InstitutionDefaultMaxQuota.objects.update_or_create(
+ institution_id=institution.id,
+ defaults={'default_max_quota': custom_max}
+ )
+
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 2000
+ }
+ },
+ file_node=self.file,
+ storage_type=UserQuota.CUSTOM_STORAGE
+ )
+
+ user_quota = UserQuota.objects.get(user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE)
+ assert_equal(user_quota.max_quota, custom_max)
+ assert_equal(user_quota.used, 2000)
+
+ def test_get_or_create_multiple_storage_types(self):
+ """get_or_create should handle multiple storage types for same user"""
+ institution = InstitutionFactory()
+ self.project_creator.affiliated_institutions.add(institution)
+ RegionFactory(_id=institution._id)
+
+ # Create separate files for each storage type
+ file_nii = OsfStorageFileNode.create(
+ target=self.node,
+ path='/testfile_nii',
+ _id='testfile_nii',
+ name='testfile_nii',
+ materialized_path='/testfile_nii'
+ )
+ file_nii.save()
+
+ file_custom = OsfStorageFileNode.create(
+ target=self.node,
+ path='/testfile_custom',
+ _id='testfile_custom',
+ name='testfile_custom',
+ materialized_path='/testfile_custom'
+ )
+ file_custom.save()
+
+ # Create NII_STORAGE first
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 1000
+ }
+ },
+ file_node=file_nii,
+ storage_type=UserQuota.NII_STORAGE
+ )
+
+ # Create CUSTOM_STORAGE
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 2000
+ }
+ },
+ file_node=file_custom,
+ storage_type=UserQuota.CUSTOM_STORAGE
+ )
+
+ nii_quota = UserQuota.objects.get(user=self.project_creator, storage_type=UserQuota.NII_STORAGE)
+ custom_quota = UserQuota.objects.get(user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE)
+
+ assert_equal(nii_quota.used, 1000)
+ assert_equal(custom_quota.used, 2000)
+
+ def test_get_or_create_preserves_existing_max_quota(self):
+ """get_or_create should not overwrite existing max_quota when record exists"""
+ custom_max = 300
+ UserQuota.objects.create(
+ user=self.project_creator,
+ storage_type=UserQuota.NII_STORAGE,
+ max_quota=custom_max,
+ used=0
+ )
+
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 1500
+ }
+ },
+ file_node=self.file,
+ storage_type=UserQuota.NII_STORAGE
+ )
+
+ user_quota = UserQuota.objects.get(user=self.project_creator, storage_type=UserQuota.NII_STORAGE)
+ assert_equal(user_quota.max_quota, custom_max)
+ assert_equal(user_quota.used, 1500)
+
+ @mock.patch('website.util.quota.check_select_for_update', return_value=False)
+ def test_get_or_create_without_select_for_update_new_userquota(self, _mock_check_select):
+ """get_or_create without select_for_update should create new UserQuota"""
+ assert_false(UserQuota.objects.filter(user=self.project_creator).exists())
+
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 5000
+ }
+ },
+ file_node=self.file,
+ storage_type=UserQuota.NII_STORAGE
+ )
+
+ user_quota = UserQuota.objects.filter(user=self.project_creator, storage_type=UserQuota.NII_STORAGE)
+ assert_equal(len(user_quota), 1)
+ user_quota = user_quota[0]
+ assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+ assert_equal(user_quota.used, 5000)
+
+ @mock.patch('website.util.quota.check_select_for_update', return_value=False)
+ def test_get_or_create_without_select_for_update_existing_userquota(self, _mock_check_select):
+ """get_or_create without select_for_update should retrieve existing UserQuota"""
+ UserQuota.objects.create(
+ user=self.project_creator,
+ storage_type=UserQuota.NII_STORAGE,
+ max_quota=200,
+ used=100
+ )
+
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 200
+ }
+ },
+ file_node=self.file,
+ storage_type=UserQuota.NII_STORAGE
+ )
+
+ user_quota_list = UserQuota.objects.filter(user=self.project_creator, storage_type=UserQuota.NII_STORAGE)
+ assert_equal(len(user_quota_list), 1)
+ user_quota = user_quota_list[0]
+ assert_equal(user_quota.max_quota, 200)
+ assert_equal(user_quota.used, 300)
+
+ @mock.patch('website.util.quota.check_select_for_update', return_value=False)
+ def test_get_or_create_without_select_for_update_custom_storage(self, mock_check_select):
+ """get_or_create without select_for_update should work with CUSTOM_STORAGE"""
+ institution = InstitutionFactory()
+ self.project_creator.affiliated_institutions.add(institution)
+ RegionFactory(_id=institution._id)
+ ProjectStorageType.objects.filter(node=self.node).update(
+ storage_type=ProjectStorageType.CUSTOM_STORAGE
+ )
+
+ assert_false(UserQuota.objects.filter(user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE).exists())
+
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 3000
+ }
+ },
+ file_node=self.file,
+ storage_type=UserQuota.CUSTOM_STORAGE
+ )
+
+ user_quota = UserQuota.objects.get(user=self.project_creator, storage_type=UserQuota.CUSTOM_STORAGE)
+ assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+ assert_equal(user_quota.used, 3000)
+
+ @mock.patch('website.util.quota.check_select_for_update')
+ def test_file_modified_else_branch_fileinfo_get(self, mock_check):
+ """Test the else branch at line 330 for FileInfo.get without select_for_update"""
+ mock_check.return_value = False
+
+ # Create initial FileInfo
+ FileInfo.objects.create(file=self.file, file_size=500)
+
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 1200
+ }
+ },
+ file_node=self.file,
+ storage_type=UserQuota.NII_STORAGE
+ )
+
+ user_quota = UserQuota.objects.get(user=self.project_creator, storage_type=UserQuota.NII_STORAGE)
+ # used = 1200 - 500 = 700 (retrieved existing FileInfo via else branch)
+ assert_equal(user_quota.used, 700)
+
+ # Verify check_select_for_update was called multiple times (for UserQuota and FileInfo)
+ assert_true(mock_check.called)
+
+ @mock.patch('website.util.quota.check_select_for_update', return_value=False)
+ def test_file_modified_without_select_for_update_fileinfo_not_exists(self, mock_check_select):
+ """file_modified should handle FileInfo.DoesNotExist without select_for_update"""
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 2000
+ }
+ },
+ file_node=self.file,
+ storage_type=UserQuota.NII_STORAGE
+ )
+
+ user_quota = UserQuota.objects.get(user=self.project_creator, storage_type=UserQuota.NII_STORAGE)
+ # used = 2000 - 0 = 2000 (no FileInfo exists, so old size is 0)
+ assert_equal(user_quota.used, 2000)
+
+ file_info = FileInfo.objects.get(file=self.file)
+ assert_equal(file_info.file_size, 2000)
+
+ @mock.patch('website.util.quota.check_select_for_update')
+ def test_file_modified_else_branch_get_or_create(self, mock_check):
+ """Test the else branch at line 315 for get_or_create without select_for_update"""
+ mock_check.return_value = False
+
+ assert_false(UserQuota.objects.filter(user=self.project_creator, storage_type=UserQuota.NII_STORAGE).exists())
+
+ quota.file_modified(
+ target=self.node,
+ payload={
+ 'metadata': {
+ 'size': 1500
+ }
+ },
+ file_node=self.file,
+ storage_type=UserQuota.NII_STORAGE
+ )
+
+ # Verify that the else branch (line 315) was executed successfully
+ user_quota = UserQuota.objects.get(user=self.project_creator, storage_type=UserQuota.NII_STORAGE)
+ assert_equal(user_quota.used, 1500)
+ assert_equal(user_quota.max_quota, api_settings.DEFAULT_MAX_QUOTA)
+
+ # Verify check_select_for_update was called
+ assert_true(mock_check.called)
diff --git a/tests/test_quota_move_copy.py b/tests/test_quota_move_copy.py
new file mode 100644
index 00000000000..49081f7ad72
--- /dev/null
+++ b/tests/test_quota_move_copy.py
@@ -0,0 +1,287 @@
+import pytest
+from unittest import mock
+from website.util import quota
+from django.db import IntegrityError
+
+@pytest.fixture
+def mock_user():
+ user = mock.Mock()
+ user.id = 1
+ user._id = 'abc123'
+ user.userquota_set.get.side_effect = quota.UserQuota.DoesNotExist
+ return user
+
+@pytest.fixture
+def mock_node():
+ node = mock.Mock(spec=quota.AbstractNode)
+ node.id = 42
+ node.creator = mock.Mock()
+ node.type = 'osf.node'
+ return node
+
+def test_get_file_size_file():
+ children = {'children': [{'kind': 'file', 'size': 100}, {'kind': 'file', 'size': 200}]}
+ assert quota.get_file_size(children) == 300
+
+def test_get_file_size_nested():
+ children = {
+ 'children': [
+ {'kind': 'file', 'size': 100},
+ {'kind': 'folder', 'children': [{'kind': 'file', 'size': 50}]}
+ ]
+ }
+ assert quota.get_file_size(children) == 150
+
+@mock.patch('website.util.quota.logger')
+def test_get_file_size_none_size_treated_as_zero_and_warns(mock_logger):
+ """A child with size=None must not crash get_file_size."""
+ children = {'children': [{'kind': 'file', 'size': None, 'name': 'unknown.gdoc'}]}
+ assert quota.get_file_size(children) == 0
+ assert mock_logger.warning.called
+
+@mock.patch('website.util.quota.logger')
+def test_get_file_size_negative_size_treated_as_zero_and_warns(mock_logger):
+ """A negative child size must be treated as 0."""
+ children = {'children': [{'kind': 'file', 'size': -1, 'name': 'bad.txt'}]}
+ assert quota.get_file_size(children) == 0
+ assert mock_logger.warning.called
+
+def test_get_file_size_string_size():
+ """Providers that report size as a str (e.g. s3compat HEAD Content-Length header)
+ must not raise TypeError when compared against 0."""
+ children = {'children': [{'kind': 'file', 'size': '100'}, {'kind': 'file', 'size': '200'}]}
+ assert quota.get_file_size(children) == 300
+
+@mock.patch('website.util.quota.logger')
+def test_get_file_size_negative_string_size_treated_as_zero_and_warns(mock_logger):
+ """A negative child size reported as a str must also be treated as 0."""
+ children = {'children': [{'kind': 'file', 'size': '-1', 'name': 'bad.txt'}]}
+ assert quota.get_file_size(children) == 0
+ assert mock_logger.warning.called
+
+@mock.patch('website.util.quota.logger')
+def test_get_file_size_mixed_children_none_and_valid(mock_logger):
+ """A None-size sibling must not swallow the total contributed by valid siblings."""
+ children = {
+ 'children': [
+ {'kind': 'file', 'size': 100, 'name': 'ok.txt'},
+ {'kind': 'file', 'size': None, 'name': 'unknown.gdoc'},
+ ]
+ }
+ assert quota.get_file_size(children) == 100
+ assert mock_logger.warning.called
+
+@mock.patch('website.util.quota.ProjectStorageType.objects.get')
+@mock.patch('website.util.quota.AbstractNode.objects.get')
+@mock.patch('website.util.quota.update_quota')
+@pytest.mark.django_db
+def test_handle_move_copy_source_extend_dest_osfstorage(mock_update_quota, mock_node_get, mock_pst_get, mock_node):
+ """source=s3compatinstitutions, dest=osfstorage"""
+ mock_pst_get.return_value = mock.Mock(storage_type=1)
+ mock_node_get.return_value = mock_node
+
+ payload = {
+ 'destination': {'provider': 'osfstorage', 'size': 123, 'children': None},
+ 'source': {'nid': 'nid1', 'provider': 's3compatinstitutions'}
+ }
+ quota._handle_move_copy(quota.FileLog.FILE_MOVED, mock_node, mock.Mock(), payload)
+ mock_update_quota.assert_not_called()
+
+@mock.patch('website.util.quota.ProjectStorageType.objects.get')
+@mock.patch('website.util.quota.AbstractNode.objects.get')
+@mock.patch('website.util.quota.update_quota')
+@pytest.mark.django_db
+def test_handle_move_copy_source_osfstorage(mock_update_quota, mock_node_get, mock_pst_get, mock_user):
+ mock_node = mock.create_autospec(quota.AbstractNode, instance=True)
+ mock_node.type = 'osf.node'
+ mock_pst_get.return_value = mock.Mock(storage_type=1)
+ mock_node_get.return_value = mock_node
+
+ payload = {
+ 'destination': {'provider': 's3compatinstitutions', 'size': 123},
+ 'source': {'provider': 'osfstorage', 'size': 123, 'nid': 'nid1'}
+ }
+ quota._handle_move_copy(quota.FileLog.FILE_MOVED, mock_node, mock_user, payload)
+ mock_update_quota.assert_called_once()
+
+@mock.patch('website.util.quota.update_quota')
+@pytest.mark.django_db
+def test_handle_move_copy_no_osfstorage(mock_update_quota, mock_node, mock_user):
+ payload = {
+ 'destination': {'provider': 's3compatinstitutions'},
+ 'source': {'provider': 's3compatinstitutions'}
+ }
+ quota._handle_move_copy(quota.FileLog.FILE_MOVED, mock_node, mock_user, payload)
+ mock_update_quota.assert_not_called()
+
+@mock.patch('website.util.quota.ProjectStorageType.objects.get')
+@mock.patch('website.util.quota.AbstractNode.objects.get')
+@mock.patch('website.util.quota.update_quota')
+@pytest.mark.django_db
+def test_file_moved_file(mock_update_quota, mock_node_get, mock_pst_get, mock_node):
+ mock_pst_get.return_value = mock.Mock(storage_type=1)
+ mock_node_get.return_value = mock_node
+ payload = {
+ 'destination': {'size': 123, 'children': None},
+ 'source': {'nid': 'nid1', 'provider': 'osfstorage'}
+ }
+ quota.file_moved(mock_node, payload)
+ mock_update_quota.assert_called_once()
+
+@mock.patch('website.util.quota.ProjectStorageType.objects.get')
+@mock.patch('website.util.quota.AbstractNode.objects.get')
+@mock.patch('website.util.quota.update_quota')
+@pytest.mark.django_db
+def test_file_moved_folder(mock_update_quota, mock_node_get, mock_pst_get, mock_node):
+ mock_pst_get.return_value = mock.Mock(storage_type=1)
+ mock_node_get.return_value = mock_node
+ payload = {
+ 'destination': {'children': [{'kind': 'file', 'size': 100}]},
+ 'source': {'nid': 'nid1', 'provider': 'osfstorage'}
+ }
+ quota.file_moved(mock_node, payload)
+ mock_update_quota.assert_called_once()
+
+@mock.patch('website.util.quota.ProjectStorageType.objects.get', side_effect=Exception)
+@mock.patch('website.util.quota.AbstractNode.objects.get')
+@mock.patch('website.util.quota.update_quota')
+@pytest.mark.django_db
+def test_file_moved_exception(mock_update_quota, mock_node_get, mock_pst_get, mock_node):
+ mock_node_get.return_value = mock_node
+ payload = {
+ 'destination': {'size': 123, 'children': None},
+ 'source': {'nid': 'nid1', 'provider': 'osfstorage'}
+ }
+ with pytest.raises(Exception):
+ quota.file_moved(mock_node, payload)
+ mock_update_quota.assert_not_called()
+
+@mock.patch('website.util.quota.UserQuota.objects.get')
+@mock.patch('website.util.quota.UserQuota.objects.filter')
+@pytest.mark.django_db
+def test_update_quota_add(mock_filter, mock_get, mock_node):
+ uq = mock.Mock()
+ uq.used = 0
+ mock_get.return_value = uq
+ mock_filter.return_value.select_for_update.return_value.get.return_value = uq
+ quota.update_quota(mock_node, 100, 1, add=True)
+ assert uq.used == 100
+
+@mock.patch('website.util.quota.UserQuota.objects.get')
+@mock.patch('website.util.quota.UserQuota.objects.filter')
+@pytest.mark.django_db
+def test_update_quota_subtract(mock_filter, mock_get, mock_node):
+ uq = mock.Mock()
+ uq.used = 200
+ mock_get.return_value = uq
+ mock_filter.return_value.select_for_update.return_value.get.return_value = uq
+ quota.update_quota(mock_node, 50, 1, add=False)
+ assert uq.used == 150
+
+@mock.patch('website.util.quota.check_select_for_update', return_value=False)
+@mock.patch('website.util.quota.transaction.atomic')
+@mock.patch('website.util.quota.UserQuota.objects.create')
+@mock.patch('website.util.quota.UserQuota.objects.filter')
+@mock.patch('website.util.quota.UserQuota.objects.get', side_effect=quota.UserQuota.DoesNotExist)
+@pytest.mark.django_db
+def test_update_quota_userquota_not_exist(mock_get, mock_filter, mock_create, mock_atomic, mock_node):
+ mock_filter.return_value.select_for_update.return_value.get.side_effect = quota.UserQuota.DoesNotExist
+ quota.update_quota(mock_node, 50, 1, add=True)
+ mock_get.assert_called_once()
+
+@mock.patch('website.util.quota.UserQuota.objects.get', side_effect=Exception)
+@mock.patch('website.util.quota.UserQuota.objects.filter')
+@pytest.mark.django_db
+def test_update_quota_other_exception(mock_filter, mock_get, mock_node):
+ mock_filter.return_value.select_for_update.return_value.get.side_effect = Exception
+ with pytest.raises(Exception):
+ quota.update_quota(mock_node, 50, 1, add=True)
+ mock_get.assert_not_called()
+
+@mock.patch('website.util.quota._handle_move_copy')
+def test_update_used_quota_move_event(mock_handle_move_copy, mock_node, mock_user):
+ payload = {
+ 'destination': {'provider': 's3compatinstitutions'},
+ 'source': {'provider': 'osfstorage'}
+ }
+ quota.update_used_quota(None, mock_node, mock_user, quota.FileLog.FILE_MOVED, payload)
+ mock_handle_move_copy.assert_called_once()
+
+@mock.patch('website.util.quota._handle_move_copy')
+def test_update_used_quota_dest_osfstorage_returns_early(mock_handle_move_copy, mock_node, mock_user):
+ payload = {
+ 'destination': {'provider': 'osfstorage'}
+ }
+ result = quota.update_used_quota(None, mock_node, mock_user, quota.FileLog.FILE_MOVED, payload)
+ assert result is None
+ mock_handle_move_copy.assert_not_called()
+
+@mock.patch('website.util.quota.check_select_for_update', return_value=True)
+@mock.patch('website.util.quota.transaction.atomic')
+@mock.patch('website.util.quota.UserQuota.objects.create', side_effect=IntegrityError)
+@mock.patch('website.util.quota.UserQuota.objects.filter')
+@pytest.mark.django_db
+def test_update_quota_integrityerror_select_for_update_add(
+ mock_filter, mock_create, mock_atomic, mock_check, mock_node
+):
+ uq = mock.Mock()
+ uq.used = 10
+ mock_filter.return_value.select_for_update.return_value.get.side_effect = [
+ quota.UserQuota.DoesNotExist,
+ uq
+ ]
+ quota.update_quota(mock_node, 5, 1, add=True)
+ assert mock_filter.return_value.select_for_update.return_value.get.call_count == 2
+ assert uq.used == 15
+ uq.save.assert_called_once()
+
+@mock.patch('website.util.quota.check_select_for_update', return_value=True)
+@mock.patch('website.util.quota.transaction.atomic')
+@mock.patch('website.util.quota.UserQuota.objects.create', side_effect=IntegrityError)
+@mock.patch('website.util.quota.UserQuota.objects.filter')
+@pytest.mark.django_db
+def test_update_quota_integrityerror_select_for_update_subtract(
+ mock_filter, mock_create, mock_atomic, mock_check, mock_node
+):
+ uq = mock.Mock()
+ uq.used = 10
+ mock_filter.return_value.select_for_update.return_value.get.side_effect = [
+ quota.UserQuota.DoesNotExist,
+ uq
+ ]
+ quota.update_quota(mock_node, 3, 1, add=False)
+ assert mock_filter.return_value.select_for_update.return_value.get.call_count == 1
+ assert uq.used == 10
+ uq.save.assert_not_called()
+
+@mock.patch('website.util.quota.check_select_for_update', return_value=False)
+@mock.patch('website.util.quota.transaction.atomic')
+@mock.patch('website.util.quota.UserQuota.objects.create', side_effect=IntegrityError)
+@mock.patch('website.util.quota.UserQuota.objects.get')
+@mock.patch('website.util.quota.UserQuota.objects.filter')
+@pytest.mark.django_db
+def test_update_quota_integrityerror_get_add(
+ mock_filter, mock_get, mock_create, mock_atomic, mock_check, mock_node
+):
+ uq = mock.Mock()
+ uq.used = 20
+ mock_get.side_effect = [quota.UserQuota.DoesNotExist, uq]
+ quota.update_quota(mock_node, 8, 1, add=True)
+ assert uq.used == 28
+ uq.save.assert_called_once()
+
+@mock.patch('website.util.quota.check_select_for_update', return_value=False)
+@mock.patch('website.util.quota.transaction.atomic')
+@mock.patch('website.util.quota.UserQuota.objects.create', side_effect=IntegrityError)
+@mock.patch('website.util.quota.UserQuota.objects.get')
+@mock.patch('website.util.quota.UserQuota.objects.filter')
+@pytest.mark.django_db
+def test_update_quota_integrityerror_get_subtract(
+ mock_filter, mock_get, mock_create, mock_atomic, mock_check, mock_node
+):
+ uq = mock.Mock()
+ uq.used = 30
+ mock_get.side_effect = [quota.UserQuota.DoesNotExist, uq]
+ quota.update_quota(mock_node, 12, 1, add=False)
+ assert uq.used == 30
+ uq.save.assert_not_called()
diff --git a/website/project/views/quota.py b/website/project/views/quota.py
index 88814f268a0..8f78fbd5d83 100644
--- a/website/project/views/quota.py
+++ b/website/project/views/quota.py
@@ -17,10 +17,11 @@ def get_quota_from_pid(pid):
"""Auxiliary function for getting the quota from a project ID.
Used on requests by waterbutler and the user (from browser)."""
node = AbstractNode.load(pid)
- max_quota, used_quota = quota.get_quota_info(
- node.creator, quota.get_project_storage_type(node)
- )
+ storage_type = quota.get_project_storage_type(node)
+ max_quota, used_quota = quota.get_quota_info(node.creator, storage_type)
return {
'max': max_quota * api_settings.SIZE_UNIT_GB,
- 'used': used_quota
+ 'used': used_quota,
+ 'user_guid': node.creator._id,
+ 'storage_type': storage_type,
}
diff --git a/website/settings/__init__.py b/website/settings/__init__.py
index 38da961d50b..e533b3a4619 100644
--- a/website/settings/__init__.py
+++ b/website/settings/__init__.py
@@ -21,7 +21,7 @@
if not DEV_MODE:
from . import local
from . import defaults
- for setting in ('WATERBUTLER_JWE_SECRET', 'WATERBUTLER_JWE_SALT', 'WATERBUTLER_JWT_SECRET', 'JWT_SECRET', 'DEFAULT_HMAC_SECRET', 'POPULAR_LINKS_NODE', 'NEW_AND_NOTEWORTHY_LINKS_NODE', 'SENSITIVE_DATA_SALT', 'SENSITIVE_DATA_SECRET'):
+ for setting in ('WATERBUTLER_JWE_SECRET', 'WATERBUTLER_JWE_SALT', 'WATERBUTLER_JWT_SECRET', 'JWT_SECRET', 'DEFAULT_HMAC_SECRET', 'CRON_SIGNED_URL_SECRET', 'POPULAR_LINKS_NODE', 'NEW_AND_NOTEWORTHY_LINKS_NODE', 'SENSITIVE_DATA_SALT', 'SENSITIVE_DATA_SECRET'):
assert getattr(local, setting, None) and getattr(local, setting, None) != getattr(defaults, setting, None), '{} must be specified in local.py when DEV_MODE is False'.format(setting)
diff --git a/website/settings/defaults.py b/website/settings/defaults.py
index e808456a88a..12f9a27782d 100644
--- a/website/settings/defaults.py
+++ b/website/settings/defaults.py
@@ -330,6 +330,13 @@ def parent_dir(path):
DEFAULT_HMAC_SECRET = 'changeme'
DEFAULT_HMAC_ALGORITHM = hashlib.sha256
+
+# signed-URL auth for cron-triggered admin endpoints (No.1 external_acc_update,
+# No.82 GatherView). Deliberately separate from DEFAULT_HMAC_SECRET (WaterButler<->OSF)
+# so a leak of one does not affect the other.
+CRON_SIGNED_URL_SECRET = 'changeme'
+CRON_SIGNED_URL_HMAC_ALGORITHM = hashlib.sha256
+CRON_SIGNED_URL_TTL_SECONDS = 100
WATERBUTLER_URL = 'http://localhost:7777'
WATERBUTLER_INTERNAL_URL = WATERBUTLER_URL
diff --git a/website/static/img/institutions/banners/edugain-logo.png b/website/static/img/institutions/banners/edugain-logo.png
new file mode 100644
index 00000000000..812f470d210
Binary files /dev/null and b/website/static/img/institutions/banners/edugain-logo.png differ
diff --git a/website/static/img/institutions/shields-rounded-corners/edugain-shield-rounded-corners.png b/website/static/img/institutions/shields-rounded-corners/edugain-shield-rounded-corners.png
new file mode 100644
index 00000000000..a42be72e902
Binary files /dev/null and b/website/static/img/institutions/shields-rounded-corners/edugain-shield-rounded-corners.png differ
diff --git a/website/static/img/institutions/shields/edugain-shield.png b/website/static/img/institutions/shields/edugain-shield.png
new file mode 100644
index 00000000000..a42be72e902
Binary files /dev/null and b/website/static/img/institutions/shields/edugain-shield.png differ
diff --git a/website/static/js/fangorn.js b/website/static/js/fangorn.js
index bb0e87f283f..d6b6b44829c 100644
--- a/website/static/js/fangorn.js
+++ b/website/static/js/fangorn.js
@@ -89,6 +89,10 @@ var OPERATIONS = {
}
};
+var MESSAGE_MAP = {
+ 'quota_exceeded': gettext('You do not have enough available quota.'),
+};
+
var isInUploadFolderProcess = false;
var isOngoingUploadFolder = false;
@@ -706,6 +710,111 @@ function checkConflictsRename(tb, item, name, cb) {
cb('replace');
}
+/**
+ * Recursively finds all individual files that exceed maxSizeBytes.
+ * Returns an array of offending item objects.
+ */
+function _findOversizedFiles(item, maxSizeBytes) {
+ var oversized = [];
+ if (item.kind === 'file') {
+ var size = item.data.size || 0;
+ if (size > 0 && size > maxSizeBytes) {
+ oversized.push(item);
+ }
+ } else if (item.kind === 'folder' && item.children && item.children.length > 0) {
+ for (var i = 0; i < item.children.length; i++) {
+ oversized = oversized.concat(_findOversizedFiles(item.children[i], maxSizeBytes));
+ }
+ }
+ return oversized;
+}
+
+/**
+ * Helper function to reject a conflict item (e.g. if file size or quota check fails)
+ * and advance the sync move/copy queue.
+ */
+function _rejectConflictItem(tb, from, to, notRenameOp, conflict) {
+ from.inProgress = false;
+ if (notRenameOp) {
+ addFileStatus(tb, from, false, '', '', conflict);
+ // Use object identity (not data.id): for MOVE, .done() replaces from.data
+ // in-place so from.data.id changes before .always() runs.
+ tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function(file) {
+ return file !== from;
+ });
+ if (!tb.pendingReadyFiles.length) {
+ doSyncMove(tb, to.data.provider);
+ }
+ }
+}
+
+/**
+ * True when `node` itself, or any folder anywhere beneath it, has not been
+ * lazy-loaded yet — meaning the subtree's total size cannot be known client-side.
+ *
+ * The file browser lazy-loads a single level at a time, so an expanded folder can
+ * still contain sub-folders whose own children have never been fetched. Summing
+ * only what happens to be loaded would under-count the freed size.
+ */
+function hasUnloadedFolder(node) {
+ if (node.kind === 'folder' && !node.load) {
+ return true;
+ }
+ return (node.children || []).some(hasUnloadedFolder);
+}
+
+/**
+ * Size freed up at the destination by an overwrite, in bytes.
+ *
+ * Mirrors WaterButler's get_replaced_size(): only conflict === 'replace' overwrites
+ * anything ('keep' renames the incoming item; omitted conflict defaults to 'warn' —
+ * both free 0 bytes), and only a same-kind destination item is replaced (osfstorage
+ * allows a file and folder to share a name).
+ *
+ * Returns null if the destination folder's subtree isn't fully loaded (size unknown) —
+ * callers must skip the client-side check and defer to the server.
+ */
+function getReplacedSize(to, from, conflict) {
+ if (conflict !== 'replace') {
+ return 0;
+ }
+ var existing = (to.children || []).filter(function(child) {
+ return child.data.name === from.data.name && child.kind === from.kind;
+ })[0];
+ if (!existing) {
+ return 0;
+ }
+ if (from.kind !== 'folder') {
+ return parseInt(existing.data.size, 10) || 0;
+ }
+ if (hasUnloadedFolder(existing)) {
+ // Part of the subtree is not lazy-loaded yet; total size unknown, let the
+ // server-side check decide.
+ return null;
+ }
+ return getAllChildren(existing).reduce(function(total, child) {
+ return child.kind === 'file' ? total + (parseInt(child.data.size, 10) || 0) : total;
+ }, 0);
+}
+
+// max_file_size skip narrows by provider only (browser can't know region);
+// server gives the authoritative 413 if it turns out not to qualify.
+function shouldSkipSizeCheck(operation, fromProvider, destProvider) {
+ return operation.status === 'move' && fromProvider === destProvider;
+}
+
+function isSameUserQuota(srcQuota, destQuota) {
+ return !!(srcQuota && destQuota &&
+ srcQuota.user_guid !== undefined && srcQuota.user_guid !== null &&
+ srcQuota.user_guid === destQuota.user_guid &&
+ srcQuota.storage_type === destQuota.storage_type);
+}
+
+function quotaCheckExceeds(destQuota, totalSize, replacedSize) {
+ replacedSize = replacedSize || 0;
+ return (destQuota.used + totalSize - replacedSize) > destQuota.max;
+}
+
function doItemOp(operation, to, from, rename, conflict) {
var tb = this;
// dismiss old modal immediately to prevent button mashing
@@ -714,6 +823,7 @@ function doItemOp(operation, to, from, rename, conflict) {
var filesRemaining;
var inConflictsQueue = false;
var syncMoves;
+ var showQuotaUsageAlert = false;
var notRenameOp = typeof rename === 'undefined';
if (notRenameOp) {
@@ -749,7 +859,113 @@ function doItemOp(operation, to, from, rename, conflict) {
return;
}
+ if (operation !== OPERATIONS.RENAME) {
+ var destProvider = to.data.provider;
+
+ // Sum size of all files in the item being moved/copied
+ var totalMoveSize = 0;
+ if (from.kind === 'file') {
+ totalMoveSize = parseInt(from.data.size, 10) || 0;
+ } else {
+ getAllChildren(from).forEach(function(child) {
+ if (child.kind === 'file') {
+ totalMoveSize += parseInt(child.data.size, 10) || 0;
+ }
+ });
+ }
+
+ var skipSizeCheck = shouldSkipSizeCheck(operation, from.data.provider, destProvider);
+
+ if (!skipSizeCheck) {
+ var destMaxSize = to.data && to.data.accept && to.data.accept.maxSize;
+ if (destMaxSize) {
+ var maxSizeBytes = destMaxSize * 1000000;
+ var oversizedFiles = _findOversizedFiles(from, maxSizeBytes);
+ if (oversizedFiles.length > 0) {
+ var maxSizeDisplay = $osf.humanFileSize(maxSizeBytes, true);
+ oversizedFiles.forEach(function(oversized) {
+ var displaySize = $osf.humanFileSize(oversized.data.size, true);
+ $osf.growl(sprintf(
+ gettext('File「%1$s」is too large (%2$s). Max file size is %3$s.'),
+ oversized.data.name, displaySize, maxSizeDisplay
+ ));
+ });
+ _rejectConflictItem(tb, from, to, notRenameOp, conflict);
+ return;
+ }
+ }
+ }
+
+ // Quota context is resolved only after the size check above has run, mirroring
+ // the server's run_pre_checks order: an oversized file is rejected with 413
+ // before creator_quota is ever fetched.
+ var destQuota = null;
+ var srcQuota = null;
+ var skipQuotaCheck = false;
+ if (destProvider === 'osfstorage' && totalMoveSize > 0) {
+ // Sequential async:false calls (kept in sync with doItemOp's control flow)
+ var destQuotaResp = $.ajax({
+ async: false,
+ method: 'GET',
+ url: to.data.nodeApiUrl + 'get_creator_quota/'
+ });
+ // Only a MOVE from osfstorage needs src quota — it serves just to detect a
+ // same-record move. A falsy srcQuota means "adds usage".
+ var srcQuotaResp = null;
+ if (operation.status === 'move' && from.data.provider === 'osfstorage') {
+ // Same node ⇒ same UserQuota record: reuse the response already in hand
+ // instead of asking the very same endpoint twice.
+ srcQuotaResp = (from.data.nodeApiUrl === to.data.nodeApiUrl) ? destQuotaResp : $.ajax({
+ async: false,
+ method: 'GET',
+ url: from.data.nodeApiUrl + 'get_creator_quota/'
+ });
+ }
+ if (destQuotaResp.responseJSON && (!srcQuotaResp || srcQuotaResp.responseJSON)) {
+ destQuota = destQuotaResp.responseJSON;
+ srcQuota = srcQuotaResp ? srcQuotaResp.responseJSON : null;
+ skipQuotaCheck = (operation.status === 'move' &&
+ isSameUserQuota(srcQuota, destQuota));
+ }
+ // A failed request leaves destQuota null: the browser evaluates nothing here —
+ // neither the quota check nor the threshold warning runs below — and defers
+ // entirely to the server-side check.
+ }
+
+ // destQuota is only set for an osfstorage destination with something to weigh.
+ // skipQuotaCheck is the single source of truth for same-UserQuota-record moves:
+ // by this point that case is already excluded, so nothing below needs to re-check it.
+ if (destQuota && !skipQuotaCheck) {
+ var replacedSize = getReplacedSize(to, from, conflict);
+ var quotaMsgText = '';
+
+ // replacedSize === null: destination folder not yet loaded, so its size
+ // is unknown — skip only this blocking check and let the move/copy proceed;
+ // the server-side check is authoritative.
+ if (replacedSize !== null && quotaCheckExceeds(destQuota, totalMoveSize, replacedSize)) {
+ if (from.kind === 'file') {
+ quotaMsgText = sprintf(gettext('Not enough quota to move/copy the file.'));
+ } else {
+ var folderSize = $osf.humanFileSize(totalMoveSize, true);
+ quotaMsgText = sprintf(gettext('Not enough quota to move/copy. The total size of the folder %1$s.'), folderSize);
+ }
+ from.notify.update(quotaMsgText, 'warning', undefined, 3000);
+ _rejectConflictItem(tb, from, to, notRenameOp, conflict);
+ return;
+ }
+ // Unknown replacedSize is treated as 0 (worst case) so the alert can still fire.
+ var quotaDelta = totalMoveSize - (replacedSize || 0);
+ // Shown only after a confirmed success below. The alert announces *crossing*
+ // the threshold, so an operation that adds no usage must stay silent.
+ showQuotaUsageAlert = (quotaDelta > 0 &&
+ destQuota.used + quotaDelta > destQuota.max * window.contextVars.threshold);
+ }
+ }
+
var origFrom = Object.assign({}, from);
+ // Must capture before COPY rebinds `from` to a new clone.
+ // .done() replaces from.data in-place, so ID-based filtering breaks for MOVE ops.
+ var originalFrom = from;
if (operation === OPERATIONS.COPY) {
from = tb.createItem($.extend(true, {status: operation.status}, from.data), to.id);
} else {
@@ -815,6 +1031,17 @@ function doItemOp(operation, to, from, rename, conflict) {
from.data.status = undefined;
from.notify.update(sprintf(gettext('Successfully %1$s.') , gettext(operation.passed)), 'success', null, 1000);
+ if (showQuotaUsageAlert) {
+ $osf.growl(
+ gettext('Quota usage alert'),
+ sprintf(
+ gettext('You have used more than %1$s%% of your quota.'),
+ (window.contextVars.threshold * 100)
+ ),
+ 'warning'
+ );
+ }
+
if (xhr.status === 200) {
to.children.forEach(function(child) {
if (child.data.name === from.data.name && child.id !== from.id) {
@@ -854,6 +1081,41 @@ function doItemOp(operation, to, from, rename, conflict) {
from.data.status = undefined;
}
+ if (xhr.status === 413 && xhr.responseJSON && xhr.responseJSON.oversized_files) {
+ var serverMaxSize = xhr.responseJSON.max_size;
+ var destMaxSize = to.data && to.data.accept && to.data.accept.maxSize;
+ var maxSizeDisplay = serverMaxSize ?
+ $osf.humanFileSize(serverMaxSize, true) :
+ (destMaxSize ? $osf.humanFileSize(destMaxSize * 1000000, true) : null);
+ xhr.responseJSON.oversized_files.forEach(function(fileObj) {
+ var displaySize = $osf.humanFileSize(fileObj.size, true);
+ var msg = maxSizeDisplay ?
+ sprintf(gettext('File「%1$s」is too large (%2$s). Max file size is %3$s.'), fileObj.name, displaySize, maxSizeDisplay) :
+ sprintf(gettext('File「%1$s」is too large (%2$s).'), fileObj.name, displaySize);
+ $osf.growl(msg);
+ });
+
+ if (notRenameOp) {
+ addFileStatus(tb, from, false, '', '', conflict);
+ }
+ orderFolder.call(tb, from.parent());
+ // NOTE: pendingReadyFiles filter, from.inProgress reset, and doSyncMove
+ // are all handled by .always() below. Do NOT duplicate them here.
+ return;
+ }
+ if (xhr.status === 406 && xhr.responseJSON && xhr.responseJSON.message) {
+ var key = xhr.responseJSON.message_key;
+ var msg = (key && MESSAGE_MAP[key]) ? MESSAGE_MAP[key] : xhr.responseJSON.message;
+ $osf.growl(msg);
+ if (notRenameOp) {
+ addFileStatus(tb, from, false, '', '', conflict);
+ }
+ orderFolder.call(tb, from.parent());
+ // NOTE: pendingReadyFiles filter, from.inProgress reset, and doSyncMove
+ // are all handled by .always() below. Do NOT duplicate them here.
+ return;
+ }
+
var message;
if (xhr.status !== 500 && xhr.responseJSON && (xhr.responseJSON.message || xhr.responseJSON.message_long)) {
@@ -878,7 +1140,9 @@ function doItemOp(operation, to, from, rename, conflict) {
orderFolder.call(tb, from.parent());
}).always(function(){
- tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function (file) { return file.data.id !== from.data.id; });
+ // Use object identity: .done() replaces from.data in-place (MOVE op),
+ // making ID-based comparison unreliable by the time .always() runs.
+ tb.pendingReadyFiles = (tb.pendingReadyFiles || []).filter(function (file) { return file !== originalFrom; });
from.inProgress = false;
if (notRenameOp && !tb.pendingReadyFiles.length){
doSyncMove(tb, to.data.provider);
@@ -3647,6 +3911,7 @@ function _dropLogic(event, items, folder) {
if (toMove.conflicts.length > 0) {
tb.syncFileMoveCache[folder.data.provider].conflicts = tb.syncFileMoveCache[folder.data.provider].conflicts || [];
+ tb.syncFileMoveCache[folder.data.provider].ready = tb.syncFileMoveCache[folder.data.provider].ready || [];
toMove.conflicts.forEach(function(item) {
tb.syncFileMoveCache[folder.data.provider].conflicts.push({'item' : item, 'folder' : folder});
});
@@ -4412,5 +4677,11 @@ module.exports = {
getCopyMode : getCopyMode,
showDeleteMultiple : showDeleteMultiple,
checkConflicts : checkConflicts,
- getPersistentLinkFor: getPersistentLinkFor
+ getPersistentLinkFor: getPersistentLinkFor,
+ quotaCheckExceeds: quotaCheckExceeds,
+ getReplacedSize: getReplacedSize,
+ isSameUserQuota: isSameUserQuota,
+ shouldSkipSizeCheck: shouldSkipSizeCheck,
+ doItemOp: doItemOp,
+ OPERATIONS: OPERATIONS
};
diff --git a/website/static/js/tests/fangorn.test.js b/website/static/js/tests/fangorn.test.js
index 10a18c56f1e..c34f3bd5c88 100644
--- a/website/static/js/tests/fangorn.test.js
+++ b/website/static/js/tests/fangorn.test.js
@@ -14,6 +14,8 @@ var Raven = require('raven-js');
window.contextVars = {
osfSupportEmail : 'fake-support@osf.io',
+ waterbutlerURL : 'https://files.osf.io/',
+ threshold : 0.9,
};
var language = require('js/osfLanguage').projectSettings;
@@ -423,6 +425,647 @@ describe('fangorn', () => {
assert.equal(Fangorn.showDeleteMultiple([folder, item]), true);
});
});
+
+ describe('shouldSkipSizeCheck', () => {
+ it('returns true for a move where source and destination provider match', () => {
+ var operation = {status: 'move'};
+ assert.equal(Fangorn.shouldSkipSizeCheck(operation, 'osfstorage', 'osfstorage'), true);
+ });
+
+ it('returns false for a move where source and destination provider differ', () => {
+ var operation = {status: 'move'};
+ assert.equal(Fangorn.shouldSkipSizeCheck(operation, 'osfstorage', 's3'), false);
+ });
+
+ it('returns false for a copy even when provider matches', () => {
+ var operation = {status: 'copy'};
+ assert.equal(Fangorn.shouldSkipSizeCheck(operation, 'osfstorage', 'osfstorage'), false);
+ });
+
+ it('returns false for a copy where provider also differs', () => {
+ var operation = {status: 'copy'};
+ assert.equal(Fangorn.shouldSkipSizeCheck(operation, 'osfstorage', 's3'), false);
+ });
+ });
+
+ describe('isSameUserQuota', () => {
+ it('returns true when user_guid and storage_type both match', () => {
+ var src = {user_guid: 'abc', storage_type: 'NII_STORAGE'};
+ var dest = {user_guid: 'abc', storage_type: 'NII_STORAGE'};
+ assert.equal(Fangorn.isSameUserQuota(src, dest), true);
+ });
+
+ it('returns false when user_guid differs', () => {
+ var src = {user_guid: 'abc', storage_type: 'NII_STORAGE'};
+ var dest = {user_guid: 'xyz', storage_type: 'NII_STORAGE'};
+ assert.equal(Fangorn.isSameUserQuota(src, dest), false);
+ });
+
+ it('returns false when storage_type differs', () => {
+ var src = {user_guid: 'abc', storage_type: 'NII_STORAGE'};
+ var dest = {user_guid: 'abc', storage_type: 'CUSTOM_STORAGE'};
+ assert.equal(Fangorn.isSameUserQuota(src, dest), false);
+ });
+
+ it('returns false when user_guid is undefined on either side', () => {
+ var src = {storage_type: 'NII_STORAGE'};
+ var dest = {user_guid: 'abc', storage_type: 'NII_STORAGE'};
+ assert.equal(Fangorn.isSameUserQuota(src, dest), false);
+ });
+
+ it('returns false when user_guid is null on both sides', () => {
+ var src = {user_guid: null, storage_type: 'NII_STORAGE'};
+ var dest = {user_guid: null, storage_type: 'NII_STORAGE'};
+ assert.equal(Fangorn.isSameUserQuota(src, dest), false);
+ });
+
+ it('returns false when srcQuota is falsy', () => {
+ var dest = {user_guid: 'abc', storage_type: 'NII_STORAGE'};
+ assert.equal(Fangorn.isSameUserQuota(null, dest), false);
+ });
+
+ it('returns false when destQuota is falsy', () => {
+ var src = {user_guid: 'abc', storage_type: 'NII_STORAGE'};
+ assert.equal(Fangorn.isSameUserQuota(src, null), false);
+ });
+ });
+
+ describe('quotaCheckExceeds', () => {
+ it('returns true when used + totalSize exceeds max', () => {
+ var destQuota = {used: 800, max: 1000};
+ assert.equal(Fangorn.quotaCheckExceeds(destQuota, 300, 0), true);
+ });
+
+ it('returns false when used + totalSize stays within max', () => {
+ var destQuota = {used: 800, max: 1000};
+ assert.equal(Fangorn.quotaCheckExceeds(destQuota, 100, 0), false);
+ });
+
+ it('returns false when used + totalSize lands exactly on max', () => {
+ var destQuota = {used: 800, max: 1000};
+ assert.equal(Fangorn.quotaCheckExceeds(destQuota, 200, 0), false);
+ });
+
+ it('subtracts replacedSize before comparing to max', () => {
+ var destQuota = {used: 800, max: 1000};
+ // Without replacedSize this would exceed (800 + 300 > 1000),
+ // but the 150 freed by the overwrite brings it back within max.
+ assert.equal(Fangorn.quotaCheckExceeds(destQuota, 300, 150), false);
+ });
+
+ it('treats a missing replacedSize as 0', () => {
+ var destQuota = {used: 800, max: 1000};
+ assert.equal(Fangorn.quotaCheckExceeds(destQuota, 300), true);
+ });
+ });
+
+ describe('getReplacedSize', () => {
+ // Fixture builder for treebeard-like nodes: getReplacedSize/hasUnloadedFolder
+ // read top-level `kind`/`load`, unlike the `getItem` helper above which only
+ // sets `data.kind`.
+ var makeNode = function(kind, name, opts) {
+ opts = opts || {};
+ return {
+ kind: kind,
+ data: {
+ name: name,
+ size: opts.size
+ },
+ children: opts.children || [],
+ load: opts.load !== undefined ? opts.load : true
+ };
+ };
+
+ it('returns 0 when conflict is not replace', () => {
+ var to = {children: []};
+ var from = makeNode('file', 'a.txt', {size: 10});
+ assert.equal(Fangorn.getReplacedSize(to, from, 'keep'), 0);
+ });
+
+ it('returns 0 when no existing item matches name and kind', () => {
+ var to = {children: [makeNode('file', 'other.txt', {size: 10})]};
+ var from = makeNode('file', 'a.txt', {size: 10});
+ assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 0);
+ });
+
+ it('does not match when name is the same but kind differs', () => {
+ var to = {children: [makeNode('folder', 'a', {})]};
+ var from = makeNode('file', 'a', {size: 10});
+ assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 0);
+ });
+
+ it('returns the existing file size when replacing a file', () => {
+ var existing = makeNode('file', 'a.txt', {size: 500});
+ var to = {children: [existing]};
+ var from = makeNode('file', 'a.txt', {size: 10});
+ assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 500);
+ });
+
+ it('returns 0 when the existing file has no size', () => {
+ var existing = makeNode('file', 'a.txt', {});
+ var to = {children: [existing]};
+ var from = makeNode('file', 'a.txt', {size: 10});
+ assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 0);
+ });
+
+ it('sums the sizes of all files in a fully loaded existing folder', () => {
+ var child1 = makeNode('file', 'x.txt', {size: 100});
+ var child2 = makeNode('file', 'y.txt', {size: 200});
+ var subfolder = makeNode('folder', 'sub', {children: [child1, child2], load: true});
+ var existing = makeNode('folder', 'a', {children: [subfolder], load: true});
+ var to = {children: [existing]};
+ var from = makeNode('folder', 'a', {});
+ assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), 300);
+ });
+
+ it('returns null when the existing folder itself has not been lazy-loaded', () => {
+ var existing = makeNode('folder', 'a', {load: false});
+ var to = {children: [existing]};
+ var from = makeNode('folder', 'a', {});
+ assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), null);
+ });
+
+ it('returns null when a nested subfolder has not been lazy-loaded', () => {
+ var unloadedSub = makeNode('folder', 'sub', {load: false});
+ var existing = makeNode('folder', 'a', {children: [unloadedSub], load: true});
+ var to = {children: [existing]};
+ var from = makeNode('folder', 'a', {});
+ assert.equal(Fangorn.getReplacedSize(to, from, 'replace'), null);
+ });
+ });
+ });
+
+ describe('doItemOp', () => {
+ var ajaxStub;
+ var growlStub;
+ var ravenStub;
+
+ beforeEach(() => {
+ growlStub = sinon.stub($osf, 'growl');
+ ravenStub = sinon.stub(Raven, 'captureMessage');
+ });
+
+ afterEach(() => {
+ if (ajaxStub) { ajaxStub.restore(); }
+ growlStub.restore();
+ ravenStub.restore();
+ });
+
+ // Fixture builder for a fake Treebeard node with just enough of the
+ // real API (move/parent/notify/add/removeSelf) for doItemOp to run.
+ function makeParent() {
+ return {
+ data: {
+ nodeUrl: 'https://osf.io/proj1/',
+ nodeApiUrl: 'https://osf.io/proj1/api/',
+ nodeId: 'proj1',
+ permissions: {edit: true},
+ provider: 'osfstorage'
+ },
+ sortChildren: sinon.stub()
+ };
+ }
+
+ function makeItem(kind, data, id, parentID) {
+ return {
+ kind: kind,
+ data: $.extend({provider: 'osfstorage'}, data),
+ children: [],
+ id: id,
+ parentID: parentID,
+ inProgress: false,
+ notify: {update: sinon.stub()},
+ move: sinon.stub(),
+ add: sinon.stub(),
+ removeSelf: sinon.stub(),
+ parent: sinon.stub().returns(makeParent())
+ };
+ }
+
+ // Fixture builder for a fake Treebeard.controller (`this` inside doItemOp).
+ function makeTb() {
+ var tb = {
+ modal: {dismiss: sinon.stub(), update: sinon.stub()},
+ pendingFileOps: [],
+ pendingReadyFiles: [],
+ moveStates: [],
+ uploadStates: [],
+ syncFileMoveCache: {osfstorage: {conflicts: [], ready: []}, s3compat: {conflicts: [], ready: []}},
+ clearMultiselect: sinon.stub(),
+ redraw: sinon.stub(),
+ select: sinon.stub().returns({hide: sinon.stub(), css: sinon.stub()}),
+ options: {
+ lazyLoadPreprocess: function(resp) { return {data: resp}; }
+ }
+ };
+ tb.createItem = sinon.stub().callsFake(function(data) {
+ return makeItem(data.kind, data);
+ });
+ tb.buildTree = sinon.stub().callsFake(function(data) {
+ return makeItem(data.kind, data);
+ });
+ return tb;
+ }
+
+ // A minimal fake jQuery Deferred supporting the .done()/.fail()/.always()
+ // chain doItemOp attaches to the final POST call.
+ function fakeDeferred(config) {
+ var obj = {
+ done: function(cb) {
+ if (config.status === 'done') { cb.apply(null, config.args || []); }
+ return obj;
+ },
+ fail: function(cb) {
+ if (config.status === 'fail') { cb.apply(null, config.args || []); }
+ return obj;
+ },
+ always: function(cb) { cb(); return obj; }
+ };
+ return obj;
+ }
+
+ function stubAjax(responder) {
+ ajaxStub = sinon.stub($, 'ajax').callsFake(responder);
+ }
+
+ function growlCallWithType(type) {
+ return growlStub.getCalls().some(function(call) { return call.args[2] === type; });
+ }
+
+ it('returns without calling the API when dropped back onto its own parent', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/'}, 5, undefined);
+ var from = makeItem('file', {name: 'a.txt', size: 10}, 3, 5);
+ ajaxStub = sinon.stub($, 'ajax');
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ assert.equal(ajaxStub.called, false);
+ });
+
+ it('rejects the move without calling the API when a file exceeds the destination max size', () => {
+ var tb = makeTb();
+ // Different providers: shouldSkipSizeCheck only shortcuts a same-provider move,
+ // so this keeps the client-side oversized check from being bypassed.
+ var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', accept: {maxSize: 1}}, 5, 1);
+ var from = makeItem('file', {name: 'big.txt', size: 5000000}, 3, 1);
+ ajaxStub = sinon.stub($, 'ajax');
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ assert.equal(ajaxStub.called, false);
+ assert.equal(growlStub.called, true);
+ assert.equal(from.inProgress, false);
+ });
+
+ it('rejects the copy without calling the API when it would exceed quota', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/'}, 5, 1);
+ var from = makeItem('file', {name: 'a.txt', size: 100}, 3, 1);
+ stubAjax(function(opts) {
+ if (opts.method === 'GET') {
+ return {responseJSON: {used: 950, max: 1000, user_guid: 'u1', storage_type: 'NII_STORAGE'}};
+ }
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined);
+ assert.equal(ajaxStub.callCount, 1);
+ assert.equal(from.notify.update.called, true);
+ assert.equal(from.inProgress, false);
+ });
+
+ it('skips the quota check and proceeds when source and destination share the same UserQuota record', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1);
+ var from = makeItem('file', {nodeApiUrl: 'https://osf.io/from/', name: 'a.txt', size: 999999, path: '/a.txt'}, 3, 1);
+ var sameQuota = {used: 10, max: 1000000000, user_guid: 'u1', storage_type: 'NII_STORAGE'};
+ stubAjax(function(opts) {
+ if (opts.method === 'GET') {
+ return {responseJSON: sameQuota};
+ }
+ if (opts.type === 'POST') {
+ return fakeDeferred({status: 'done', args: [
+ {name: 'a.txt', provider: 'osfstorage', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/to/'},
+ 'success',
+ {status: 200}
+ ]});
+ }
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ // dest quota GET + src quota GET (different node) + the real POST
+ assert.equal(ajaxStub.callCount, 3);
+ assert.equal(tb.uploadStates.length, 1);
+ assert.equal(tb.uploadStates[0].success, true);
+ });
+
+ it('shows a quota usage alert on success once the destination crosses the warning threshold', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1);
+ var from = makeItem('file', {name: 'a.txt', size: 150}, 3, 1);
+ stubAjax(function(opts) {
+ if (opts.method === 'GET') {
+ // used(800) + totalSize(150) = 950 <= max(1000) -> not rejected,
+ // but 950 > max * threshold(0.9) = 900 -> alert should fire.
+ return {responseJSON: {used: 800, max: 1000, user_guid: 'u1', storage_type: 'NII_STORAGE'}};
+ }
+ if (opts.type === 'POST') {
+ return fakeDeferred({status: 'done', args: [
+ {name: 'a.txt', provider: 'osfstorage', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/to/'},
+ 'success',
+ {status: 200}
+ ]});
+ }
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined);
+ assert.equal(growlCallWithType('warning'), true);
+ });
+
+ it('creates a new item via tb.createItem for a COPY and posts a copy action', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1);
+ var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1);
+ var postedData;
+ stubAjax(function(opts) {
+ postedData = JSON.parse(opts.data);
+ return fakeDeferred({status: 'done', args: [
+ {name: 'a.txt', provider: 's3compat', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/to/'},
+ 'success',
+ {status: 200}
+ ]});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined);
+ assert.equal(tb.createItem.called, true);
+ assert.equal(postedData.action, 'copy');
+ });
+
+ it('posts a rename action for a RENAME operation', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', path: '/'}, 5, 1);
+ var from = makeItem('file', {provider: 's3compat', name: 'old.txt'}, 3, 9);
+ var postedData;
+ stubAjax(function(opts) {
+ postedData = JSON.parse(opts.data);
+ return fakeDeferred({status: 'done', args: [
+ {name: 'new.txt', provider: 's3compat', path: '/new.txt', kind: 'file', nodeUrl: 'https://osf.io/proj1/'},
+ 'success',
+ {status: 200}
+ ]});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.RENAME, to, from, 'new.txt', undefined);
+ assert.equal(postedData.action, 'rename');
+ assert.equal(postedData.rename, 'new.txt');
+ });
+
+ it('shows a pending growl and stops when the API responds 202', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1);
+ var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1);
+ stubAjax(function() {
+ return fakeDeferred({status: 'done', args: [{}, 'success', {status: 202}]});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ assert.equal(growlCallWithType('info'), true);
+ assert.equal(tb.uploadStates.length, 0);
+ });
+
+ it('shows an oversized-file growl and records a failure when the API rejects with 413', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1);
+ var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1);
+ stubAjax(function() {
+ return fakeDeferred({status: 'fail', args: [
+ {status: 413, responseJSON: {oversized_files: [{name: 'big.txt', size: 5000000}], max_size: 1000000}},
+ 'error'
+ ]});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ assert.equal(growlStub.called, true);
+ assert.equal(tb.uploadStates.length, 1);
+ assert.equal(tb.uploadStates[0].success, false);
+ });
+
+ it('shows the mapped MESSAGE_MAP text when the API rejects with 406 and a known message_key', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1);
+ var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1);
+ stubAjax(function() {
+ return fakeDeferred({status: 'fail', args: [
+ {status: 406, responseJSON: {message: 'raw message', message_key: 'quota_exceeded'}},
+ 'error'
+ ]});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ assert.equal(growlStub.called, true);
+ assert.equal(tb.uploadStates[0].success, false);
+ });
+
+ it('reports to Raven and shows a fallback message when the API fails with a generic 500', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1);
+ var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1);
+ stubAjax(function() {
+ return fakeDeferred({status: 'fail', args: [{status: 500}, 'error']});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ assert.equal(ravenStub.called, true);
+ assert.equal(growlStub.called, true);
+ });
+
+ // The cases below close the gaps found in the coverage report
+ // (RDM-osf.io/coverage/html/js/fangorn.js.html) for _findOversizedFiles
+ // and doItemOp. See investigation/2026-09-03-fangorn-coverage-findoversizedfiles-doitemop.md.
+
+ it('rejects a folder move and reports oversized files found inside it', () => {
+ var tb = makeTb();
+ // Different providers: shouldSkipSizeCheck only shortcuts a same-provider move.
+ var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', accept: {maxSize: 1}}, 5, 1);
+ var from = makeItem('folder', {name: 'myFolder'}, 3, 1);
+ var smallChild = makeItem('file', {name: 'small.txt', size: 100}, 10, 3);
+ var bigChild = makeItem('file', {name: 'big.txt', size: 5000000}, 11, 3);
+ from.children = [smallChild, bigChild];
+ ajaxStub = sinon.stub($, 'ajax');
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ assert.equal(ajaxStub.called, false);
+ assert.equal(growlStub.called, true);
+ assert.equal(from.inProgress, false);
+ });
+
+ it('sums child file sizes when moving a folder and rejects if quota would be exceeded', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {nodeApiUrl: 'https://osf.io/to/'}, 5, 1);
+ var from = makeItem('folder', {name: 'myFolder'}, 3, 1);
+ var child = makeItem('file', {name: 'a.txt', size: 200}, 10, 3);
+ from.children = [child];
+ stubAjax(function(opts) {
+ if (opts.method === 'GET') {
+ return {responseJSON: {used: 900, max: 1000, user_guid: 'u1', storage_type: 'NII_STORAGE'}};
+ }
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined);
+ assert.equal(ajaxStub.callCount, 1);
+ assert.equal(from.notify.update.called, true);
+ assert.equal(from.inProgress, false);
+ });
+
+ // _rejectConflictItem isn't exported, so these two exercise it through its only
+ // real entry points (the oversized-file and quota pre-flight checks in doItemOp)
+ // to cover the object-identity pendingReadyFiles filter and its doSyncMove advance.
+ it('keeps other pending files queued when a pre-flight reject leaves the sync queue non-empty', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', accept: {maxSize: 1}}, 5, 1);
+ var from = makeItem('file', {name: 'big.txt', size: 5000000}, 3, 1);
+ var other = makeItem('file', {name: 'other.txt'}, 77, 1);
+ tb.pendingReadyFiles = [from, other];
+ ajaxStub = sinon.stub($, 'ajax');
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, 'replace');
+ // Filtered by object identity, not id: "other" survives untouched.
+ assert.deepEqual(tb.pendingReadyFiles, [other]);
+ assert.equal(ajaxStub.called, false);
+ // Queue still has "other" pending, so doSyncMove must not have run yet.
+ assert.equal(tb.modal.dismiss.callCount, 1);
+ assert.equal(tb.modal.update.called, false);
+ });
+
+ it('advances the sync move queue via doSyncMove once a pre-flight reject empties pendingReadyFiles', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', accept: {maxSize: 1}}, 5, 1);
+ var from = makeItem('file', {name: 'big.txt', size: 5000000}, 3, 1);
+ tb.pendingReadyFiles = [from];
+ ajaxStub = sinon.stub($, 'ajax');
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, 'replace');
+ assert.equal(tb.pendingReadyFiles.length, 0);
+ // Emptying the queue must trigger doSyncMove -> displayMoveStats, which shows
+ // the move-status modal because addFileStatus already recorded the rejection
+ // (displayMoveStats resets tb.moveStates back to [] once it builds the modal).
+ assert.equal(tb.modal.update.called, true);
+ });
+
+ it('shows a "conflicts left to resolve" modal when the sync queue has pending conflicts', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1);
+ var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1);
+ tb.syncFileMoveCache.s3compat = {conflicts: [{item: from, folder: to}], ready: []};
+ stubAjax(function() {
+ return fakeDeferred({status: 'done', args: [
+ {name: 'a.txt', provider: 's3compat', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/proj1/'},
+ 'success',
+ {status: 200}
+ ]});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ assert.equal(tb.modal.update.called, true);
+ });
+
+ it('preserves the branch and checks the sync-ready queue when moving a github item', () => {
+ var tb = makeTb();
+ // github isn't in the default syncFileMoveCache fixture; add it so the
+ // notRenameOp conflicts-check doesn't throw on an undefined entry.
+ tb.syncFileMoveCache.github = {conflicts: [], ready: []};
+ var to = makeItem('folder', {provider: 'github', path: '/'}, 5, 1);
+ var from = makeItem('file', {provider: 'github', name: 'a.txt', branch: 'main'}, 3, 1);
+ var postedData;
+ stubAjax(function(opts) {
+ postedData = JSON.parse(opts.data);
+ return fakeDeferred({status: 'done', args: [
+ {name: 'a.txt', provider: 'github', path: '/a.txt', kind: 'file', nodeUrl: 'https://osf.io/proj1/'},
+ 'success',
+ {status: 200}
+ ]});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ assert.equal(postedData.branch, 'main');
+ });
+
+ it('removes the existing item at destination when the server responds 200 with a name conflict', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1);
+ var existingChild = makeItem('file', {name: 'a.txt', provider: 's3compat'}, 99, 5);
+ to.children = [existingChild];
+ var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1);
+ stubAjax(function() {
+ return fakeDeferred({status: 'done', args: [
+ {name: 'a.txt', provider: 's3compat', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/to/'},
+ 'success',
+ {status: 200}
+ ]});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ assert.equal(existingChild.removeSelf.called, true);
+ });
+
+ it('rebuilds the child subtree when the moved item is a folder with children in the response', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', nodeApiUrl: 'https://osf.io/to/', path: '/dest/'}, 5, 1);
+ var from = makeItem('folder', {provider: 's3compat', name: 'myFolder'}, 3, 1);
+ stubAjax(function() {
+ return fakeDeferred({status: 'done', args: [
+ {
+ name: 'myFolder', provider: 's3compat', path: '/dest/myFolder/', kind: 'folder',
+ nodeUrl: 'https://osf.io/to/',
+ children: [{name: 'child.txt', kind: 'file', provider: 's3compat'}]
+ },
+ 'success',
+ {status: 200}
+ ]});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ assert.equal(tb.buildTree.called, true);
+ assert.equal(from.add.called, true);
+ assert.equal(from.open, true);
+ assert.equal(from.load, true);
+ });
+
+ it('removes the newly created item when a COPY operation fails', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1);
+ var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1);
+ stubAjax(function() {
+ return fakeDeferred({status: 'fail', args: [{status: 500}, 'error']});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.COPY, to, from, undefined, undefined);
+ assert.equal(tb.createItem.called, true);
+ var createdItem = tb.createItem.returnValues[0];
+ assert.equal(createdItem.removeSelf.called, true);
+ });
+
+ it('shows the responseJSON message for a non-413/406/500/503 failure', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1);
+ var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1);
+ stubAjax(function() {
+ return fakeDeferred({status: 'fail', args: [
+ {status: 400, responseJSON: {message: 'custom failure message'}},
+ 'error'
+ ]});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ var calledWithMessage = growlStub.getCalls().some(function(call) { return call.args[1] === 'custom failure message'; });
+ assert.equal(calledWithMessage, true);
+ });
+
+ it('uses textStatus as the error message on a 503 response', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1);
+ var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1);
+ stubAjax(function() {
+ return fakeDeferred({status: 'fail', args: [{status: 503}, 'Service Unavailable']});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ var calledWithMessage = growlStub.getCalls().some(function(call) { return call.args[1] === 'Service Unavailable'; });
+ assert.equal(calledWithMessage, true);
+ });
+
+ it('removes the settled item from pendingReadyFiles once the operation completes', () => {
+ var tb = makeTb();
+ var to = makeItem('folder', {provider: 's3compat', path: '/dest/'}, 5, 1);
+ var from = makeItem('file', {provider: 's3compat', name: 'a.txt', size: 10}, 3, 1);
+ var other = makeItem('file', {name: 'other.txt'}, 77, 1);
+ tb.pendingReadyFiles = [from, other];
+ stubAjax(function() {
+ return fakeDeferred({status: 'done', args: [
+ {name: 'a.txt', provider: 's3compat', path: '/dest/a.txt', kind: 'file', nodeUrl: 'https://osf.io/proj1/'},
+ 'success',
+ {status: 200}
+ ]});
+ });
+ Fangorn.doItemOp.call(tb, Fangorn.OPERATIONS.MOVE, to, from, undefined, undefined);
+ assert.deepEqual(tb.pendingReadyFiles, [other]);
+ });
});
describe('FGToolbar IME Keydown Handling', function() {
diff --git a/website/templates/emails/file_operation_failed.html.mako b/website/templates/emails/file_operation_failed.html.mako
index 2c72fed9b16..e7ab9105d15 100644
--- a/website/templates/emails/file_operation_failed.html.mako
+++ b/website/templates/emails/file_operation_failed.html.mako
@@ -59,8 +59,30 @@
- An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'}.
- Please log in and try this action again. If the problem persists, please email ${osf_support_email}.
+ <%
+ ei = context.get('error_info') or {}
+ err_type = ei.get('type', '')
+ %>
+ % if err_type == 'quota_exceeded':
+ An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'} because you do not have enough available storage quota.
+ Please free up storage space or contact your administrator to increase your quota, then try again. If the problem persists, please email ${osf_support_email}.
+ % elif err_type == 'oversized':
+ <%
+ oversized_files = ei.get('oversized_files', [])
+ max_size_bytes = ei.get('max_size') or 0
+ max_size_mb = max_size_bytes // (1024 * 1024) if max_size_bytes else 0
+ %>
+ An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'} because the following file(s) exceed the maximum allowed file size${' of {} MB'.format(max_size_mb) if max_size_mb else ''}:
+
+ % for f in oversized_files:
+ - ${f.get('name', 'Unknown file')} (${'{:.1f}'.format(f.get('size', 0) / (1024 * 1024))} MB)
+ % endfor
+
+ Please reduce the file size and try again. If the problem persists, please email ${osf_support_email}.
+ % else:
+ An error has occurred, and the ${'folder' if source_path.endswith('/') else 'file'} from ${source_node.title} on the GakuNin RDM was not successfully ${'moved' if action == 'move' else 'copied'}.
+ Please log in and try this action again. If the problem persists, please email ${osf_support_email}.
+ % endif
|
diff --git a/website/translations/en/LC_MESSAGES/js_messages.po b/website/translations/en/LC_MESSAGES/js_messages.po
index 7b54523cf44..32f8b61b92a 100644
--- a/website/translations/en/LC_MESSAGES/js_messages.po
+++ b/website/translations/en/LC_MESSAGES/js_messages.po
@@ -9475,3 +9475,15 @@ msgstr ""
"GakuNin RDM is not responsible for the service or for your use "
"thereof.\n"
"\n"
+
+msgid "File「%1$s」is too large (%2$s). Max file size is %3$s."
+msgstr ""
+
+msgid "You do not have enough available quota."
+msgstr ""
+
+msgid "Not enough quota to move/copy the file."
+msgstr ""
+
+msgid "Not enough quota to move/copy. The total size of the folder %1$s."
+msgstr ""
diff --git a/website/translations/ja/LC_MESSAGES/js_messages.po b/website/translations/ja/LC_MESSAGES/js_messages.po
index 8b8775c7c4f..18651932072 100644
--- a/website/translations/ja/LC_MESSAGES/js_messages.po
+++ b/website/translations/ja/LC_MESSAGES/js_messages.po
@@ -11021,3 +11021,15 @@ msgstr ""
"RDMプロジェクトは外部サービスに接続されます。このサービスを利用することで、それら外部サービスの利用規約に拘束されます。GakuNin "
"RDMは、それらサービスまたはユーザーによるその利用に対して責任を負いません。\n"
"\n"
+
+msgid "File「%1$s」is too large (%2$s). Max file size is %3$s."
+msgstr "ファイル「%1$s」は大きすぎます (%2$s)。最大ファイルサイズは %3$s です。"
+
+msgid "You do not have enough available quota."
+msgstr "利用可能なクォータが不足しています。"
+
+msgid "Not enough quota to move/copy the file."
+msgstr "ファイルを移動/コピーするのに十分なクォータがありません。"
+
+msgid "Not enough quota to move/copy. The total size of the folder %1$s."
+msgstr "移動/コピーするための空き容量が足りません。フォルダのサイズは%1$sです。"
diff --git a/website/translations/js_messages.pot b/website/translations/js_messages.pot
index f9bba444c1c..bd6c80d03c1 100644
--- a/website/translations/js_messages.pot
+++ b/website/translations/js_messages.pot
@@ -9314,3 +9314,15 @@ msgid ""
"${user} made bibliographic group ${mapcore_groups} a "
"non-bibliographic group on ${node}"
msgstr ""
+
+msgid "File「%1$s」is too large (%2$s). Max file size is %3$s."
+msgstr ""
+
+msgid "You do not have enough available quota."
+msgstr ""
+
+msgid "Not enough quota to move/copy the file."
+msgstr ""
+
+msgid "Not enough quota to move/copy. The total size of the folder %1$s."
+msgstr ""
diff --git a/website/util/quota.py b/website/util/quota.py
index 1990073a444..28980452fe1 100644
--- a/website/util/quota.py
+++ b/website/util/quota.py
@@ -13,6 +13,7 @@
ProjectStorageType
)
from django.utils import timezone
+from osf.models.institution_default_max_quota import InstitutionDefaultMaxQuota
from osf.utils.requests import check_select_for_update
@@ -72,6 +73,7 @@ def update_user_used_quota(user, storage_type=UserQuota.NII_STORAGE, is_recalcul
# Get total file size of projects with specified storage_type
used = used_quota(user._id, storage_type)
+ max_quota = get_default_max_quota(user, storage_type)
try:
if check_select_for_update():
user_quota = UserQuota.objects.filter(
@@ -91,10 +93,11 @@ def update_user_used_quota(user, storage_type=UserQuota.NII_STORAGE, is_recalcul
UserQuota.objects.create(
user=user,
storage_type=storage_type,
- max_quota=api_settings.DEFAULT_MAX_QUOTA,
+ max_quota=max_quota,
used=used,
)
- except IntegrityError:
+ except IntegrityError as e:
+ logger.warning(u'IntegrityError while creating UserQuota: user={}, storage_type={}: {}'.format(user.id, storage_type, str(e)))
if is_recalculating_quota and storage_type == UserQuota.CUSTOM_STORAGE:
used_quota_for_nii_default_storage = used_quota(user._id, UserQuota.NII_STORAGE)
used_quota_for_nii_custom_storage = used_quota(user._id, UserQuota.CUSTOM_STORAGE)
@@ -128,11 +131,12 @@ def abbreviate_size(size):
return (size, abbr_dict[power])
def get_quota_info(user, storage_type=UserQuota.NII_STORAGE):
+ max_quota = get_default_max_quota(user, storage_type)
try:
user_quota = user.userquota_set.get(storage_type=storage_type)
return (user_quota.max_quota, user_quota.used)
except UserQuota.DoesNotExist:
- return (api_settings.DEFAULT_MAX_QUOTA, used_quota(user._id, storage_type))
+ return (max_quota, used_quota(user._id, storage_type))
def get_project_storage_type(node):
try:
@@ -142,6 +146,17 @@ def get_project_storage_type(node):
@file_signals.file_updated.connect
def update_used_quota(self, target, user, event_type, payload):
+ # FILE_MOVED and FILE_COPIED carry 'source'/'destination', not 'metadata'.
+ # Handle them first before the metadata_provider guard below.
+ if event_type in (FileLog.FILE_MOVED, FileLog.FILE_COPIED):
+ dest_provider = payload.get('destination', {}).get('provider')
+ if dest_provider == 'osfstorage':
+ return # Hooks already handled (create_child), SKIP double counting
+ src_provider = payload.get('source', {}).get('provider')
+ if src_provider == 'osfstorage' or src_provider in PROVIDERS:
+ _handle_move_copy(event_type, target, user, payload)
+ return
+
data = dict(payload.get('metadata')) if payload.get('metadata') else None
metadata_provider = data.get('provider') if payload.get('metadata') else None
if metadata_provider == 'osfstorage' or metadata_provider in PROVIDERS:
@@ -180,6 +195,9 @@ def update_used_quota(self, target, user, event_type, payload):
storage_type = get_project_storage_type(target)
if event_type == FileLog.FILE_ADDED:
+ metadata_provider = (payload.get('metadata') or {}).get('provider')
+ if metadata_provider == 'osfstorage':
+ return
file_added(target, payload, file_node, storage_type)
elif event_type == FileLog.FILE_REMOVED:
if metadata_provider in PROVIDERS:
@@ -214,7 +232,7 @@ def update_used_quota(self, target, user, event_type, payload):
else:
node_removed(target, user, payload, file_node, storage_type)
elif event_type == FileLog.FILE_UPDATED:
- file_modified(target, user, payload, file_node, storage_type)
+ file_modified(target, payload, file_node, storage_type)
else:
return
@@ -223,97 +241,53 @@ def file_added(target, payload, file_node, storage_type):
file_size = int(payload['metadata']['size'])
if file_size < 0:
return
- try:
- if check_select_for_update():
- user_quota = UserQuota.objects.filter(
- user=target.creator,
- storage_type=storage_type
- ).select_for_update().get()
- else:
- user_quota = UserQuota.objects.get(
- user=target.creator,
- storage_type=storage_type
- )
- user_quota.used += file_size
- user_quota.save()
- except UserQuota.DoesNotExist:
+ update_quota(target, file_size, storage_type, add=True)
+ FileInfo.objects.create(file=file_node, file_size=file_size)
+
+def node_removed(target, user, payload, file_node, storage_type):
+ if 'osf.trashed' not in file_node.type:
+ logging.error('FileNode is not trashed, cannot update used quota!')
+ return
+
+ total_removed = 0
+ for removed_file in get_node_file_list(file_node):
try:
- with transaction.atomic():
- UserQuota.objects.create(
- user=target.creator,
- storage_type=storage_type,
- max_quota=api_settings.DEFAULT_MAX_QUOTA,
- used=file_size
- )
- except IntegrityError:
if check_select_for_update():
- user_quota = UserQuota.objects.filter(
- user=target.creator,
- storage_type=storage_type
- ).select_for_update().get()
+ file_info = FileInfo.objects.filter(file=removed_file).select_for_update().get()
else:
- user_quota = UserQuota.objects.get(
- user=target.creator,
- storage_type=storage_type
- )
- user_quota.used += file_size
- user_quota.save()
-
- FileInfo.objects.create(file=file_node, file_size=file_size)
+ file_info = FileInfo.objects.get(file=removed_file)
+ except FileInfo.DoesNotExist:
+ logging.error('FileInfo not found, cannot update used quota!')
+ continue
-def node_removed(target, user, payload, file_node, storage_type):
- if check_select_for_update():
- user_quota = UserQuota.objects.filter(
- user=target.creator,
- storage_type=storage_type
- ).select_for_update().first()
- else:
- user_quota = UserQuota.objects.filter(
- user=target.creator,
- storage_type=storage_type
- ).first()
- if user_quota is not None:
- if 'osf.trashed' not in file_node.type:
- logging.error('FileNode is not trashed, cannot update used quota!')
- return
+ total_removed += file_info.file_size
+ file_info.file_size = 0
+ file_info.save()
- for removed_file in get_node_file_list(file_node):
- try:
- if check_select_for_update():
- file_info = FileInfo.objects.filter(file=removed_file).select_for_update().get()
- else:
- file_info = FileInfo.objects.get(file=removed_file)
- except FileInfo.DoesNotExist:
- logging.error('FileInfo not found, cannot update used quota!')
- continue
-
- user_quota.used -= file_info.file_size
- if user_quota.used < 0:
- user_quota.used = 0
- file_info.file_size = 0
- file_info.save()
- user_quota.save()
+ if total_removed > 0:
+ update_quota(target, total_removed, storage_type, add=False)
-def file_modified(target, user, payload, file_node, storage_type):
+def file_modified(target, payload, file_node, storage_type):
file_size = int(payload['metadata']['size'])
if file_size < 0:
return
-
+ max_quota = get_default_max_quota(target.creator, storage_type)
try:
with transaction.atomic():
if check_select_for_update():
user_quota, _ = UserQuota.objects.select_for_update().get_or_create(
user=target.creator,
storage_type=storage_type,
- defaults={'max_quota': api_settings.DEFAULT_MAX_QUOTA}
+ defaults={'max_quota': max_quota}
)
else:
user_quota, _ = UserQuota.objects.get_or_create(
user=target.creator,
storage_type=storage_type,
- defaults={'max_quota': api_settings.DEFAULT_MAX_QUOTA}
+ defaults={'max_quota': max_quota}
)
- except IntegrityError:
+ except IntegrityError as e:
+ logger.warning(u'IntegrityError while creating UserQuota in file_modified: user={}, storage_type={}: {}'.format(target.creator.id, storage_type, str(e)))
if check_select_for_update():
user_quota = UserQuota.objects.filter(user=target.creator, storage_type=storage_type).select_for_update().get()
else:
@@ -390,3 +364,156 @@ def get_node_file_list(file_node):
file_list.append(child_file_node)
return file_list
+
+def _handle_move_copy(event_type, target, user, payload):
+ """Resolve the destination file_node and dispatch to file_moved or file_copied.
+
+ For FILE_MOVED / FILE_COPIED events the relevant file information lives in
+ ``payload['destination']``, not in ``payload['metadata']``. Only extended-storage
+ providers (PROVIDERS) destinations are processed here.
+
+ Scenarios handled:
+ - osfstorage → extended storage (quota decreases for source node)
+ - same provider moves / copies (if it involves extended storage)
+
+ (Note: Moves TO osfstorage are handled entirely by OSF hooks, bypassing this flow).
+
+ :param str event_type: FileLog.FILE_MOVED or FileLog.FILE_COPIED
+ :param AbstractNode target: destination project node
+ :param OSFUser user: acting user
+ :param dict payload: full webhook payload
+ """
+
+ if event_type == FileLog.FILE_MOVED:
+ file_moved(target, payload)
+
+def file_moved(target, payload):
+ """Update per-user-per-storage used quota when moving file
+
+ :param Object target: Id of project
+ :param dict payload: The payload of the move operation
+ """
+ if isinstance(target, AbstractNode):
+ file_size = -1
+ dest_size = payload['destination'].get('size', None)
+ children = payload['destination'].get('children', None)
+ # Move file
+ if dest_size is not None:
+ file_size = int(dest_size)
+ # Move folder
+ elif children is not None:
+ file_size = get_file_size(payload['destination'])
+ if file_size < 0:
+ return
+
+ source_node_id = payload['source']['nid']
+ source_node = AbstractNode.objects.get(guids___id=source_node_id)
+ # Move from bulk-mount
+ if payload['source']['provider'] == 'osfstorage' \
+ and source_node.type != 'osf.quickfilesnode':
+ source_storage_type = get_project_storage_type(source_node)
+ update_quota(source_node, file_size, source_storage_type, add=False)
+
+def update_quota(target, file_size, storage_type, add=True):
+ """Add or subtract file_size from UserQuota.used for target.creator.
+
+ This is the single source of truth for UserQuota updates.
+ Both ``file_added`` and ``node_removed`` delegate to this function.
+
+ :param target: AbstractNode whose creator's quota is updated
+ :param int file_size: size in bytes to add or subtract
+ :param int storage_type: UserQuota.NII_STORAGE or CUSTOM_STORAGE
+ :param bool add: True to add (upload), False to subtract (delete)
+ """
+ try:
+ if check_select_for_update():
+ user_quota = UserQuota.objects.filter(
+ user=target.creator,
+ storage_type=storage_type
+ ).select_for_update().get()
+ else:
+ user_quota = UserQuota.objects.get(
+ user=target.creator,
+ storage_type=storage_type
+ )
+ if add:
+ user_quota.used += file_size
+ else:
+ user_quota.used -= file_size
+ if user_quota.used < 0:
+ user_quota.used = 0
+ user_quota.save()
+ except UserQuota.DoesNotExist:
+ if not add:
+ return
+ try:
+ max_quota = get_default_max_quota(target.creator, storage_type)
+ with transaction.atomic():
+ UserQuota.objects.create(
+ user=target.creator,
+ storage_type=storage_type,
+ max_quota=max_quota,
+ used=file_size
+ )
+ except IntegrityError as e:
+ logger.warning(u'IntegrityError while creating UserQuota in update_quota: user={}, storage_type={}: {}'.format(target.creator.id, storage_type, str(e)))
+ if check_select_for_update():
+ user_quota = UserQuota.objects.filter(
+ user=target.creator,
+ storage_type=storage_type
+ ).select_for_update().get()
+ else:
+ user_quota = UserQuota.objects.get(
+ user=target.creator,
+ storage_type=storage_type
+ )
+ user_quota.used += file_size
+ user_quota.save()
+
+def get_file_size(children):
+ children = children.get('children', [])
+ size = 0
+ for child in children:
+ if child['kind'] == 'file':
+ try:
+ child_size = int(child['size'])
+ except (TypeError, ValueError):
+ child_size = None
+ if child_size is None or child_size < 0:
+ logger.warning(
+ 'child size is %r for %r; treating size as 0 for quota calculation',
+ child_size, child.get('name')
+ )
+ child_size = 0
+ size += int(child_size)
+ else:
+ size += get_file_size(child)
+ return size
+
+def get_default_max_quota(user, storage_type):
+ """
+ Get default max quota for a user based on their affiliated institution.
+
+ If the storage type is CUSTOM_STORAGE, retrieve the default quota
+ configured for the user's affiliated institution. Otherwise, or if no
+ institution-specific quota is configured, return the system default quota.
+ When the institution's region uses NII_STORAGE, always return the system default quota.
+
+ Args:
+ user (OSFUser): The user whose default quota is to be retrieved.
+ storage_type (str): The storage type associated with the user's quota.
+
+ Returns:
+ int: The default maximum quota (in GB) applicable to the user.
+ """
+ if storage_type == UserQuota.CUSTOM_STORAGE:
+ institution = user.representative_affiliated_institution
+ if institution and not Region.objects.filter(
+ _id=institution._id,
+ waterbutler_settings__storage__type=Region.NII_STORAGE
+ ).exists():
+ default_max_quota = InstitutionDefaultMaxQuota.get_quota_by_user(user.id)
+ if default_max_quota is not None:
+ return default_max_quota
+
+ return api_settings.DEFAULT_MAX_QUOTA