diff --git a/ckanext/datastore/blueprint.py b/ckanext/datastore/blueprint.py index d462974707e..2b4f8eea028 100644 --- a/ckanext/datastore/blueprint.py +++ b/ckanext/datastore/blueprint.py @@ -70,7 +70,7 @@ def dump(resource_id): try: dump_to( resource_id, - response.stream, + response, fmt=data[u'format'], offset=data[u'offset'], limit=data.get(u'limit'), diff --git a/ckanext/datastore/tests/test_dump.py b/ckanext/datastore/tests/test_dump.py index 61ccd14d926..79d81aaade6 100644 --- a/ckanext/datastore/tests/test_dump.py +++ b/ckanext/datastore/tests/test_dump.py @@ -156,6 +156,52 @@ def test_dump_q_and_fields(self, app): expected_content = u"nested,author\r\n" u'"{""a"": ""b""}",tolstoy\n' assert content == expected_content + @pytest.mark.ckan_config("ckan.plugins", "datastore") + @pytest.mark.usefixtures("clean_datastore", "with_plugins") + def test_dump_csv_file_extension(self, app): + resource = factories.Resource() + data = { + "resource_id": resource["id"], + "force": True, + "fields": [ + {"id": u"b\xfck", "type": "text"}, + {"id": "author", "type": "text"}, + {"id": "published"}, + {"id": u"characters", u"type": u"_text"}, + {"id": "random_letters", "type": "text[]"}, + ], + "records": [ + { + u"b\xfck": "annakarenina", + "author": "tolstoy", + "published": "2005-03-01", + "nested": ["b", {"moo": "moo"}], + u"characters": [u"Princess Anna", u"Sergius"], + "random_letters": ["a", "e", "x"], + }, + { + u"b\xfck": "warandpeace", + "author": "tolstoy", + "nested": {"a": "b"}, + "random_letters": [], + }, + ], + } + helpers.call_action("datastore_create", **data) + + res = app.get( + u"/datastore/dump/{0}?limit=1&format=csv".format( + resource["id"] + ) + ) + + attachment_filename = res.headers['Content-disposition'] + + expected_attch_filename = 'attachment; filename="{0}.csv"'.format( + resource['id']) + + assert attachment_filename == expected_attch_filename + @pytest.mark.ckan_config("ckan.plugins", "datastore") @pytest.mark.usefixtures("clean_datastore", "with_plugins") def test_dump_tsv(self, app): @@ -205,6 +251,52 @@ def test_dump_tsv(self, app): ) assert content == expected_content + @pytest.mark.ckan_config("ckan.plugins", "datastore") + @pytest.mark.usefixtures("clean_datastore", "with_plugins") + def test_dump_tsv_file_extension(self, app): + resource = factories.Resource() + data = { + "resource_id": resource["id"], + "force": True, + "fields": [ + {"id": u"b\xfck", "type": "text"}, + {"id": "author", "type": "text"}, + {"id": "published"}, + {"id": u"characters", u"type": u"_text"}, + {"id": "random_letters", "type": "text[]"}, + ], + "records": [ + { + u"b\xfck": "annakarenina", + "author": "tolstoy", + "published": "2005-03-01", + "nested": ["b", {"moo": "moo"}], + u"characters": [u"Princess Anna", u"Sergius"], + "random_letters": ["a", "e", "x"], + }, + { + u"b\xfck": "warandpeace", + "author": "tolstoy", + "nested": {"a": "b"}, + "random_letters": [], + }, + ], + } + helpers.call_action("datastore_create", **data) + + res = app.get( + "/datastore/dump/{0}?limit=1&format=tsv".format( + str(resource["id"]) + ) + ) + + attachment_filename = res.headers['Content-disposition'] + + expected_attch_filename = 'attachment; filename="{0}.tsv"'.format( + resource['id']) + + assert attachment_filename == expected_attch_filename + @pytest.mark.ckan_config("ckan.plugins", "datastore") @pytest.mark.usefixtures("clean_datastore", "with_plugins") def test_dump_json(self, app): @@ -266,6 +358,52 @@ def test_dump_json(self, app): } assert content == expected_content + @pytest.mark.ckan_config("ckan.plugins", "datastore") + @pytest.mark.usefixtures("clean_datastore", "with_plugins") + def test_dump_json_file_extension(self, app): + resource = factories.Resource() + data = { + "resource_id": resource["id"], + "force": True, + "fields": [ + {"id": u"b\xfck", "type": "text"}, + {"id": "author", "type": "text"}, + {"id": "published"}, + {"id": u"characters", u"type": u"_text"}, + {"id": "random_letters", "type": "text[]"}, + ], + "records": [ + { + u"b\xfck": "annakarenina", + "author": "tolstoy", + "published": "2005-03-01", + "nested": ["b", {"moo": "moo"}], + u"characters": [u"Princess Anna", u"Sergius"], + "random_letters": ["a", "e", "x"], + }, + { + u"b\xfck": "warandpeace", + "author": "tolstoy", + "nested": {"a": "b"}, + "random_letters": [], + }, + ], + } + helpers.call_action("datastore_create", **data) + + res = app.get( + "/datastore/dump/{0}?limit=1&format=json".format( + resource["id"] + ) + ) + + attachment_filename = res.headers['Content-disposition'] + + expected_attch_filename = 'attachment; filename="{0}.json"'.format( + resource['id']) + + assert attachment_filename == expected_attch_filename + @pytest.mark.ckan_config("ckan.plugins", "datastore") @pytest.mark.usefixtures("clean_datastore", "with_plugins") def test_dump_xml(self, app): @@ -331,6 +469,52 @@ def test_dump_xml(self, app): ) assert content == expected_content + @pytest.mark.ckan_config("ckan.plugins", "datastore") + @pytest.mark.usefixtures("clean_datastore", "with_plugins") + def test_dump_xml_file_extension(self, app): + resource = factories.Resource() + data = { + "resource_id": resource["id"], + "force": True, + "fields": [ + {"id": u"b\xfck", "type": "text"}, + {"id": "author", "type": "text"}, + {"id": "published"}, + {"id": u"characters", u"type": u"_text"}, + {"id": "random_letters", "type": "text[]"}, + ], + "records": [ + { + u"b\xfck": "annakarenina", + "author": "tolstoy", + "published": "2005-03-01", + "nested": ["b", {"moo": "moo"}], + u"characters": [u"Princess Anna", u"Sergius"], + "random_letters": ["a", "e", "x"], + }, + { + u"b\xfck": "warandpeace", + "author": "tolstoy", + "nested": {"a": "b"}, + "random_letters": [], + }, + ], + } + helpers.call_action("datastore_create", **data) + + res = app.get( + "/datastore/dump/{0}?limit=1&format=xml".format( + str(resource["id"]) + ) + ) + + attachment_filename = res.headers['Content-disposition'] + + expected_attch_filename = 'attachment; filename="{0}.xml"'.format( + resource['id']) + + assert attachment_filename == expected_attch_filename + @pytest.mark.ckan_config("ckan.datastore.search.rows_max", "3") @pytest.mark.ckan_config("ckan.plugins", "datastore") @pytest.mark.usefixtures("clean_datastore", "with_plugins") @@ -475,7 +659,10 @@ def test_dump_pagination_json_with_rows_max(self, app): def get_csv_record_values(response_body): - return [int(record.split(",")[1]) for record in six.ensure_text(response_body).split()[1:]] + return [ + int(record.split(",")[1]) for record in six.ensure_text( + response_body).split()[1:] + ] def get_json_record_values(response_body): diff --git a/ckanext/datastore/writer.py b/ckanext/datastore/writer.py index cdb4dd02950..2518c42a236 100644 --- a/ckanext/datastore/writer.py +++ b/ckanext/datastore/writer.py @@ -27,14 +27,14 @@ def csv_writer(response, fields, name=None, bom=False): response.headers['Content-Type'] = b'text/csv; charset=utf-8' if name: response.headers['Content-disposition'] = ( - b'attachment; filename="{name}.csv"'.format( + u'attachment; filename="{name}.csv"'.format( name=encode_rfc2231(name))) if bom: - response.write(BOM_UTF8) + response.stream.write(BOM_UTF8) - unicodecsv.writer(response, encoding=u'utf-8').writerow( + unicodecsv.writer(response.stream, encoding=u'utf-8').writerow( f['id'] for f in fields) - yield TextWriter(response) + yield TextWriter(response.stream) @contextmanager @@ -53,15 +53,17 @@ def tsv_writer(response, fields, name=None, bom=False): b'text/tab-separated-values; charset=utf-8') if name: response.headers['Content-disposition'] = ( - b'attachment; filename="{name}.tsv"'.format( + u'attachment; filename="{name}.tsv"'.format( name=encode_rfc2231(name))) if bom: - response.write(BOM_UTF8) + response.stream.write(BOM_UTF8) unicodecsv.writer( - response, encoding=u'utf-8', dialect=unicodecsv.excel_tab).writerow( + response.stream, + encoding=u'utf-8', + dialect=unicodecsv.excel_tab).writerow( f['id'] for f in fields) - yield TextWriter(response) + yield TextWriter(response.stream) class TextWriter(object): @@ -89,15 +91,15 @@ def json_writer(response, fields, name=None, bom=False): b'application/json; charset=utf-8') if name: response.headers['Content-disposition'] = ( - b'attachment; filename="{name}.json"'.format( + u'attachment; filename="{name}.json"'.format( name=encode_rfc2231(name))) if bom: - response.write(BOM_UTF8) - response.write( + response.stream.write(BOM_UTF8) + response.stream.write( six.ensure_binary(u'{\n "fields": %s,\n "records": [' % dumps( fields, ensure_ascii=False, separators=(u',', u':')))) - yield JSONWriter(response) - response.write(b'\n]}\n') + yield JSONWriter(response.stream) + response.stream.write(b'\n]}\n') class JSONWriter(object): @@ -133,13 +135,13 @@ def xml_writer(response, fields, name=None, bom=False): b'text/xml; charset=utf-8') if name: response.headers['Content-disposition'] = ( - b'attachment; filename="{name}.xml"'.format( + u'attachment; filename="{name}.xml"'.format( name=encode_rfc2231(name))) if bom: - response.write(BOM_UTF8) - response.write(b'\n') - yield XMLWriter(response, [f['id'] for f in fields]) - response.write(b'\n') + response.stream.write(BOM_UTF8) + response.stream.write(b'\n') + yield XMLWriter(response.stream, [f[u'id'] for f in fields]) + response.stream.write(b'\n') class XMLWriter(object):