From 30baf43a6916cf86d3f41e154a5bcd941ad627b8 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Tue, 28 Jun 2016 16:41:59 +1000 Subject: [PATCH 01/21] initial changes to support XLSX - POI dependency and initial commit of responsewriter --- solr/core/ivy.xml | 3 + .../solr/response/XLSXResponseWriter.java | 427 ++++++++++++++++++ 2 files changed, 430 insertions(+) create mode 100644 solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml index 5f8706f0e43f..a1f0b1f9ad15 100644 --- a/solr/core/ivy.xml +++ b/solr/core/ivy.xml @@ -126,6 +126,9 @@ + + + diff --git a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java b/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java new file mode 100644 index 000000000000..f56c81bc3580 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java @@ -0,0 +1,427 @@ +package org.apache.solr.response; + +import org.apache.lucene.index.IndexableField; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; + +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.SchemaField; +import org.apache.solr.schema.StrField; +import org.apache.solr.search.DocList; +import org.apache.solr.search.ReturnFields; +import org.apache.solr.core.SolrCore; + +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.ss.usermodel.IndexedColors; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; + + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.time.format.DateTimeFormatter; +import java.io.IOException; +import java.io.Writer; +import java.io.CharArrayWriter; +import java.io.StringWriter; +import java.io.PrintWriter; +import java.io.OutputStream; +import java.util.Date; +import java.util.Iterator; +import java.util.Map; +import java.util.List; +import java.util.ArrayList; +import java.util.Collection; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; + +/** + * + */ + +public class XLSXResponseWriter extends RawResponseWriter { + Logger log = LoggerFactory.getLogger(SolrCore.class); + + @Override + public void init(NamedList n) { + } + + @Override + public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { + //throwaway arraywriter just to satisfy super requirements; we're grabbing + //all writes before they go to it anyway + XLSXWriter w = new XLSXWriter(new CharArrayWriter(), req, rsp); + try { + w.writeResponse(out); + w.close(); + } catch (IOException e) { + log.warn("Write response failed due to IOException"); + log.warn(e.getMessage()); + w.close(); + } + } + + @Override + public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { + return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; + } +} + +class XLSXWriter extends TextResponseWriter { + + SolrQueryRequest req; + SolrQueryResponse rsp; + + Logger log = LoggerFactory.getLogger(SolrCore.class); + + class SerialWriteWorkbook { + SXSSFWorkbook swb; + Sheet sh; + + XSSFCellStyle headerStyle; + int rowIndex; + Row curRow; + int cellIndex; + + SerialWriteWorkbook() { + this.swb = new SXSSFWorkbook(100); + this.sh = this.swb.createSheet(); + + this.rowIndex = 0; + + this.headerStyle = (XSSFCellStyle)swb.createCellStyle(); + this.headerStyle.setFillBackgroundColor(IndexedColors.BLACK.getIndex()); + //solid fill + this.headerStyle.setFillPattern((short)1); + Font headerFont = swb.createFont(); + headerFont.setFontHeightInPoints((short)14); + headerFont.setBoldweight(Font.BOLDWEIGHT_BOLD); + headerFont.setColor(IndexedColors.WHITE.getIndex()); + this.headerStyle.setFont(headerFont); + } + + void addRow() { + curRow = sh.createRow(rowIndex++); + cellIndex = 0; + } + + void setHeaderRow() { + curRow.setHeightInPoints((short)21); + } + + //sets last created cell to have header style + void setHeaderCell() { + curRow.getCell(cellIndex - 1).setCellStyle(this.headerStyle); + } + + //set the width of the most recently created column + void setColWidth(int charWidth) { + //width is set in units of 1/256th of a character width for some reason + this.sh.setColumnWidth(cellIndex - 1, 256*charWidth); + } + + void writeCell(String value) { + Cell cell = curRow.createCell(cellIndex++); + cell.setCellValue(value); + } + + void flush(OutputStream out) { + try { + swb.write(out); + log.info("Flush complete"); + } catch (IOException e) { + StringWriter sw = new StringWriter(); + e.printStackTrace(new PrintWriter(sw)); + String stacktrace = sw.toString(); + log.warn("Failed to export to XLSX - "+stacktrace); + }finally { + swb.dispose(); + } + } + } + + private SerialWriteWorkbook wb = new SerialWriteWorkbook(); + + static class Field { + String name; + SchemaField sf; + } + + private Map xlFields = new LinkedHashMap(); + + public XLSXWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp){ + super(writer, req, rsp); + this.req = req; + this.rsp = rsp; + } + + public void writeResponse(OutputStream out) throws IOException { + log.info("Beginning export"); + + Collection fields = returnFields.getRequestedFieldNames(); + Object responseObj = rsp.getValues().get("response"); + boolean returnOnlyStored = false; + //TODO check this against CSVResponseWriter, mostly from there + if (fields==null||returnFields.hasPatternMatching()) { + if (responseObj instanceof SolrDocumentList) { + // get the list of fields from the SolrDocumentList + if(fields==null) { + fields = new LinkedHashSet(); + } + for (SolrDocument sdoc: (SolrDocumentList)responseObj) { + fields.addAll(sdoc.getFieldNames()); + } + } else { + // get the list of all fields in the index + Iterable allFields = req.getSearcher().getFieldNames(); + if(fields==null) { + fields = new ArrayList(); + } + for (String fieldName: allFields) { + fields.add(fieldName); + } + } + if (returnFields.wantsScore()) { + fields.add("score"); + } else { + fields.remove("score"); + } + returnOnlyStored = true; + } + + for (String field : fields) { + if (!returnFields.wantsField(field)) { + continue; + } + if (field.equals("score")) { + Field xlField = new Field(); + xlField.name = "score"; + xlFields.put("score", xlField); + continue; + } + + SchemaField sf = schema.getFieldOrNull(field); + if (sf == null) { + FieldType ft = new StrField(); + sf = new SchemaField(field, ft); + } + + // Return only stored fields, unless an explicit field list is specified + if (returnOnlyStored && sf != null && !sf.stored()) { + continue; + } + + Field xlField = new Field(); + xlField.name = field; + xlField.sf = sf; + xlFields.put(field, xlField); + } + + //TODO remove both of these + class NameAndWidth { + private int width; + private String name; + + public NameAndWidth(String name, int width) { + this.name = name; + this.width = width; + } + + public String getName() { + return this.name; + } + + public int getWidth() { + return this.width; + } + } + + class NiceMetadataNames extends LinkedHashMap { + + public NiceMetadataNames() { + super(); + this.put("meta_type_1", new NameAndWidth("Nice Name 1", 14)); + this.put("long_meta_2", new NameAndWidth("Long Metadata name is long", 128)); + } + } + + //TODO and get this from the xml config + NiceMetadataNames niceMap = new NiceMetadataNames(); + + wb.addRow(); + //write header + for (Field xlField : xlFields.values()) { + String printName = xlField.name; + int colWidth = 14; + + NameAndWidth nextField = niceMap.get(xlField.name); + if (nextField != null) { + printName = nextField.getName();; + colWidth = nextField.getWidth(); + } + + writeStr(xlField.name, printName, false); + wb.setColWidth(colWidth); + wb.setHeaderCell(); + } + wb.setHeaderRow(); + wb.addRow(); + + //write rows + //TODO check this against CSVResponseWriter, mostly from there + if (responseObj instanceof ResultContext ) { + writeDocuments("",(ResultContext)responseObj, returnFields ); + } + else if (responseObj instanceof DocList) { + ResultContext ctx = new ResultContext(); + ctx.docs = (DocList)responseObj; + writeDocuments(null, ctx, returnFields ); + } else if (responseObj instanceof SolrDocumentList) { + writeSolrDocumentList(null, (SolrDocumentList)responseObj, returnFields ); + } + log.info("Export complete; flushing document"); + //flush to outputstream + wb.flush(out); + wb = null; + + } + + @Override + public void close() throws IOException { + super.close(); + } + + @Override + public void writeNamedList(String name, NamedList val) throws IOException { + } + + @Override + public void writeStartDocumentList(String name, + long start, int size, long numFound, Float maxScore) throws IOException + { + // nothing + } + + @Override + public void writeEndDocumentList() throws IOException + { + // nothing + } + + //NOTE: a document cannot currently contain another document + List tmpList; + @Override + public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx ) throws IOException { + if (tmpList == null) { + tmpList = new ArrayList(1); + tmpList.add(null); + } + + for (Field xlField : xlFields.values()) { + Object val = doc.getFieldValue(xlField.name); + int nVals = val instanceof Collection ? ((Collection)val).size() : (val==null ? 0 : 1); + if (nVals == 0) { + writeNull(xlField.name); + continue; + } + + if ((xlField.sf != null && xlField.sf.multiValued()) || nVals > 1) { + Collection values; + // normalize to a collection + if (val instanceof Collection) { + values = (Collection)val; + } else { + tmpList.set(0, val); + values = tmpList; + } + + writeArray(xlField.name, values.iterator()); + + } else { + // normalize to first value + if (val instanceof Collection) { + Collection values = (Collection)val; + val = values.iterator().next(); + } + writeVal(xlField.name, val); + } + } + wb.addRow(); + } + + @Override + public void writeStr(String name, String val, boolean needsEscaping) throws IOException { + wb.writeCell(val); + } + + @Override + public void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException { + } + + @Override + public void writeArray(String name, Iterator val) throws IOException { + StringBuffer output = new StringBuffer(); + while (val.hasNext()) { + Object v = val.next(); + if (v instanceof IndexableField) { + IndexableField f = (IndexableField)v; + output.append(f.stringValue() + "; "); + } else { + output.append(v.toString() + "; "); + } + } + if (output.length() > 0) { + output.deleteCharAt(output.length()-1); + output.deleteCharAt(output.length()-1); + } + writeStr(name, output.toString(), false); + } + + @Override + public void writeNull(String name) throws IOException { + wb.writeCell(""); + } + + @Override + public void writeInt(String name, String val) throws IOException { + wb.writeCell(val); + } + + @Override + public void writeLong(String name, String val) throws IOException { + wb.writeCell(val); + } + + @Override + public void writeBool(String name, String val) throws IOException { + wb.writeCell(val); + } + + @Override + public void writeFloat(String name, String val) throws IOException { + wb.writeCell(val); + } + + @Override + public void writeDouble(String name, String val) throws IOException { + wb.writeCell(val); + } + + @Override + public void writeDate(String name, Date val) throws IOException { + String outputDate = DateTimeFormatter.ISO_LOCAL_DATE.format(val.toInstant()); + writeDate(name, outputDate); + } + + @Override + public void writeDate(String name, String val) throws IOException { + wb.writeCell(val); + } +} \ No newline at end of file From 86d922854f9b14ec79fb8d5d7fd8c6bbb4c96b60 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Wed, 29 Jun 2016 16:19:01 +1000 Subject: [PATCH 02/21] further patching for SOLR against 6.0 master --- .../solr/response/XLSXResponseWriter.java | 56 ++++++++----------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java b/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java index f56c81bc3580..4f37b0975170 100644 --- a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java @@ -1,5 +1,7 @@ package org.apache.solr.response; +import com.google.common.collect.Iterables; +import com.google.common.collect.Sets; import org.apache.lucene.index.IndexableField; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -56,14 +58,11 @@ public void init(NamedList n) { @Override public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { //throwaway arraywriter just to satisfy super requirements; we're grabbing - //all writes before they go to it anyway + // all writes before they go to it anyway XLSXWriter w = new XLSXWriter(new CharArrayWriter(), req, rsp); try { w.writeResponse(out); - w.close(); - } catch (IOException e) { - log.warn("Write response failed due to IOException"); - log.warn(e.getMessage()); + } finally { w.close(); } } @@ -123,7 +122,7 @@ void setHeaderCell() { //set the width of the most recently created column void setColWidth(int charWidth) { - //width is set in units of 1/256th of a character width for some reason + //width in poi is units of 1/256th of a character width for some reason this.sh.setColumnWidth(cellIndex - 1, 256*charWidth); } @@ -149,26 +148,23 @@ void flush(OutputStream out) { private SerialWriteWorkbook wb = new SerialWriteWorkbook(); - static class Field { + static class XLField { String name; SchemaField sf; } - private Map xlFields = new LinkedHashMap(); + private Map xlFields = new LinkedHashMap(); public XLSXWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp){ super(writer, req, rsp); - this.req = req; - this.rsp = rsp; } public void writeResponse(OutputStream out) throws IOException { - log.info("Beginning export"); + SolrParams params = req.getParams(); Collection fields = returnFields.getRequestedFieldNames(); Object responseObj = rsp.getValues().get("response"); boolean returnOnlyStored = false; - //TODO check this against CSVResponseWriter, mostly from there if (fields==null||returnFields.hasPatternMatching()) { if (responseObj instanceof SolrDocumentList) { // get the list of fields from the SolrDocumentList @@ -179,13 +175,12 @@ public void writeResponse(OutputStream out) throws IOException { fields.addAll(sdoc.getFieldNames()); } } else { - // get the list of all fields in the index - Iterable allFields = req.getSearcher().getFieldNames(); - if(fields==null) { - fields = new ArrayList(); - } - for (String fieldName: allFields) { - fields.add(fieldName); + // get the list of fields from the index + Iterable all = req.getSearcher().getFieldNames(); + if (fields == null) { + fields = Sets.newHashSet(all); + } else { + Iterables.addAll(fields, all); } } if (returnFields.wantsScore()) { @@ -201,7 +196,7 @@ public void writeResponse(OutputStream out) throws IOException { continue; } if (field.equals("score")) { - Field xlField = new Field(); + XLField xlField = new XLField(); xlField.name = "score"; xlFields.put("score", xlField); continue; @@ -218,7 +213,7 @@ public void writeResponse(OutputStream out) throws IOException { continue; } - Field xlField = new Field(); + XLField xlField = new XLField(); xlField.name = field; xlField.sf = sf; xlFields.put(field, xlField); @@ -257,7 +252,7 @@ public NiceMetadataNames() { wb.addRow(); //write header - for (Field xlField : xlFields.values()) { + for (XLField xlField : xlFields.values()) { String printName = xlField.name; int colWidth = 14; @@ -274,23 +269,18 @@ public NiceMetadataNames() { wb.setHeaderRow(); wb.addRow(); - //write rows - //TODO check this against CSVResponseWriter, mostly from there - if (responseObj instanceof ResultContext ) { - writeDocuments("",(ResultContext)responseObj, returnFields ); + if (responseObj instanceof ResultContext) { + writeDocuments(null, (ResultContext)responseObj ); } else if (responseObj instanceof DocList) { - ResultContext ctx = new ResultContext(); - ctx.docs = (DocList)responseObj; - writeDocuments(null, ctx, returnFields ); + ResultContext ctx = new BasicResultContext((DocList)responseObj, returnFields, null, null, req); + writeDocuments(null, ctx ); } else if (responseObj instanceof SolrDocumentList) { writeSolrDocumentList(null, (SolrDocumentList)responseObj, returnFields ); } - log.info("Export complete; flushing document"); - //flush to outputstream + wb.flush(out); wb = null; - } @Override @@ -324,7 +314,7 @@ public void writeSolrDocument(String name, SolrDocument doc, ReturnFields return tmpList.add(null); } - for (Field xlField : xlFields.values()) { + for (XLField xlField : xlFields.values()) { Object val = doc.getFieldValue(xlField.name); int nVals = val instanceof Collection ? ((Collection)val).size() : (val==null ? 0 : 1); if (nVals == 0) { From 4db64aeddb5458689fab1475248c29118b064f95 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Thu, 14 Jul 2016 15:11:21 +1000 Subject: [PATCH 03/21] add missing dependency on xml-beans (already in lucene) and initial running version of test --- solr/core/ivy.xml | 1 + .../solr/response/XLSXResponseWriter.java | 14 +- .../solr/response/TestXLSXResponseWriter.java | 313 ++++++++++++++++++ 3 files changed, 318 insertions(+), 10 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml index 46d8d2990faf..0f628771316d 100644 --- a/solr/core/ivy.xml +++ b/solr/core/ivy.xml @@ -134,6 +134,7 @@ + diff --git a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java b/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java index 4f37b0975170..5573d320895a 100644 --- a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java @@ -24,10 +24,6 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFCellStyle; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.time.format.DateTimeFormatter; import java.io.IOException; import java.io.Writer; @@ -49,15 +45,15 @@ */ public class XLSXResponseWriter extends RawResponseWriter { - Logger log = LoggerFactory.getLogger(SolrCore.class); @Override public void init(NamedList n) { + //solrconfig calls in here } @Override public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException { - //throwaway arraywriter just to satisfy super requirements; we're grabbing + // throw away arraywriter just to satisfy super requirements; we're grabbing // all writes before they go to it anyway XLSXWriter w = new XLSXWriter(new CharArrayWriter(), req, rsp); try { @@ -78,8 +74,6 @@ class XLSXWriter extends TextResponseWriter { SolrQueryRequest req; SolrQueryResponse rsp; - Logger log = LoggerFactory.getLogger(SolrCore.class); - class SerialWriteWorkbook { SXSSFWorkbook swb; Sheet sh; @@ -134,12 +128,10 @@ void writeCell(String value) { void flush(OutputStream out) { try { swb.write(out); - log.info("Flush complete"); } catch (IOException e) { StringWriter sw = new StringWriter(); e.printStackTrace(new PrintWriter(sw)); String stacktrace = sw.toString(); - log.warn("Failed to export to XLSX - "+stacktrace); }finally { swb.dispose(); } @@ -157,6 +149,8 @@ static class XLField { public XLSXWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp){ super(writer, req, rsp); + this.req = req; + this.rsp = rsp; } public void writeResponse(OutputStream out) throws IOException { diff --git a/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java new file mode 100644 index 000000000000..994f690813cf --- /dev/null +++ b/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java @@ -0,0 +1,313 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.response; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.time.Instant; +import java.util.Arrays; +import java.util.Date; +import java.util.Iterator; + +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.xssf.usermodel.XSSFCell; +import org.apache.poi.xssf.usermodel.XSSFRow; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.apache.poi.xssf.usermodel.XSSFSheet; + +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.impl.BinaryResponseParser; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.search.SolrReturnFields; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestXLSXResponseWriter extends SolrTestCaseJ4 { + + private static XLSXResponseWriter writerXlsx; + + @BeforeClass + public static void beforeClass() throws Exception { + System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_ + initCore("solrconfig.xml","schema12.xml"); + TestCSVResponseWriter.createIndex(); + writerXlsx = new XLSXResponseWriter(); + } + + @AfterClass + public static void cleanupWriter() throws Exception { + writerXlsx = null; + } + + @Test + public void testStructuredDataViaBaseWriters() throws IOException, Exception { + SolrQueryResponse rsp = new SolrQueryResponse(); + // Don't send a ContentStream back, this will fall back to the configured base writer. + // But abuse the CONTENT key to ensure writer is also checking type + rsp.add(RawResponseWriter.CONTENT, "test"); + rsp.add("foo", "bar"); + + SolrQueryRequest r = req(); + + // check Content-Type + assertEquals("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", writerXlsx.getContentType(r, rsp)); + + + // test our basic types,and that fields come back in the requested order + XSSFSheet resultSheet = getWSResultForQuery(req("q","id:1", "wt","xlsx", "fl","id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt")); + + String result = getStringFromSheet(resultSheet); + assertEquals("id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt\n1,hi,-1,12345678987654321,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n" + , result); +/* + // test retrieving score, csv.header + assertEquals("1,0.0,hi\n" + , h.query(req("q","id:1^0", "wt","csv", "csv.header","false", "fl","id,score,foo_s"))); + + // test multivalued + assertEquals("2,\"hi,there\"\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "fl","id,v_ss"))); + + // test separator change + assertEquals("2|\"hi|there\"\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.separator","|", "fl","id,v_ss"))); + + // test mv separator change + assertEquals("2,hi|there\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "fl","id,v_ss"))); + + // test mv separator change for a single field + assertEquals("2,hi|there,nice:output\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "f.v2_ss.csv.separator",":", "fl","id,v_ss,v2_ss"))); + + // test csv field for polyfield (currency) SOLR-3959 + assertEquals("4,\"1.50\\,EUR\"\n" + , h.query(req("q","id:4", "wt","csv", "csv.header","false", "fl","id,amount_c"))); + + // test csv field for polyfield (latlon) SOLR-3959 + assertEquals("5,\"12.434\\,-134.1\"\n" + , h.query(req("q","id:5", "wt","csv", "csv.header","false", "fl","id,store")) ); + // test retrieving fields from index + String result = h.query(req("q","*:*", "wt","csv", "csv.header","true", "fl","*,score")); + for (String field : "id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss,score".split(",")) { + assertTrue(result.indexOf(field) >= 0); + } + + // test null values + assertEquals("2,,hi|there\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "fl","id,foo_s,v_ss"))); + + // test alternate null value + assertEquals("2,NULL,hi|there\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "csv.null","NULL","fl","id,foo_s,v_ss"))); + + // test alternate newline + assertEquals("2,\"hi,there\"\r\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.newline","\r\n", "fl","id,v_ss"))); + + // test alternate encapsulator + assertEquals("2,'hi,there'\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.encapsulator","'", "fl","id,v_ss"))); + + // test using escape instead of encapsulator + assertEquals("2,hi\\,there\n" + , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.escape","\\", "fl","id,v_ss"))); + + // test multiple lines + assertEquals("1,,hi\n2,\"hi,there\",\n" + , h.query(req("q","id:[1 TO 2]", "wt","csv", "csv.header","false", "fl","id,v_ss,foo_s"))); + + // test SOLR-2970 not returning non-stored fields by default. Compare sorted list + assertEquals(sortHeader("amount_c,store,v_ss,foo_b,v2_ss,foo_f,foo_i,foo_d,foo_s,foo_dt,id,foo_l\n") + , sortHeader(h.query(req("q","id:3", "wt","csv", "csv.header","true", "fl","*", "rows","0")))); + + + // now test SolrDocumentList + SolrDocument d = new SolrDocument(); + SolrDocument d1 = d; + d.addField("id","1"); + d.addField("foo_i",-1); + d.addField("foo_s","hi"); + d.addField("foo_l","12345678987654321L"); + d.addField("foo_b",false); + d.addField("foo_f",1.414f); + d.addField("foo_d",-1.0E300); + d.addField("foo_dt", new Date(Instant.parse("2000-01-02T03:04:05Z").toEpochMilli())); + d.addField("score", "2.718"); + + d = new SolrDocument(); + SolrDocument d2 = d; + d.addField("id","2"); + d.addField("v_ss","hi"); + d.addField("v_ss","there"); + d.addField("v2_ss","nice"); + d.addField("v2_ss","output"); + d.addField("score", "89.83"); + d.addField("shouldbeunstored","foo"); + + SolrDocumentList sdl = new SolrDocumentList(); + sdl.add(d1); + sdl.add(d2); + + SolrQueryRequest req = req("q","*:*"); + SolrQueryResponse rsp = new SolrQueryResponse(); + rsp.addResponse(sdl); + QueryResponseWriter w = new CSVResponseWriter(); + + rsp.setReturnFields( new SolrReturnFields("id,foo_s", req) ); + StringWriter buf = new StringWriter(); + w.write(buf, req, rsp); + assertEquals("id,foo_s\n1,hi\n2,\n", buf.toString()); + + // try scores + rsp.setReturnFields( new SolrReturnFields("id,score,foo_s", req) ); + buf = new StringWriter(); + w.write(buf, req, rsp); + assertEquals("id,score,foo_s\n1,2.718,hi\n2,89.83,\n", buf.toString()); + + // get field values from docs... should be ordered and not include score unless requested + rsp.setReturnFields( new SolrReturnFields("*", req) ); + buf = new StringWriter(); + w.write(buf, req, rsp); + assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss\n" + + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z,,\n" + + "2,,,,,,,,\"hi,there\",\"nice,output\"\n", + buf.toString()); + + + // get field values and scores - just check that the scores are there... we don't guarantee where + rsp.setReturnFields( new SolrReturnFields("*,score", req) ); + buf = new StringWriter(); + w.write(buf, req, rsp); + String s = buf.toString(); + assertTrue(s.indexOf("score") >=0 && s.indexOf("2.718") > 0 && s.indexOf("89.83") > 0 ); + + // Test field globs + rsp.setReturnFields( new SolrReturnFields("id,foo*", req) ); + buf = new StringWriter(); + w.write(buf, req, rsp); + assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt\n" + + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n" + + "2,,,,,,,\n", + buf.toString()); + + rsp.setReturnFields( new SolrReturnFields("id,*_d*", req) ); + buf = new StringWriter(); + w.write(buf, req, rsp); + assertEquals("id,foo_d,foo_dt\n" + + "1,-1.0E300,2000-01-02T03:04:05Z\n" + + "2,,\n", + buf.toString()); + + // Test function queries + rsp.setReturnFields( new SolrReturnFields("sum(1,1),id,exists(foo_i),div(9,1),foo_f", req) ); + buf = new StringWriter(); + w.write(buf, req, rsp); + assertEquals("\"sum(1,1)\",id,exists(foo_i),\"div(9,1)\",foo_f\n" + + "\"\",1,,,1.414\n" + + "\"\",2,,,\n", + buf.toString()); + + // Test transformers + rsp.setReturnFields( new SolrReturnFields("mydocid:[docid],[explain]", req) ); + buf = new StringWriter(); + w.write(buf, req, rsp); + assertEquals("mydocid,[explain]\n" + + "\"\",\n" + + "\"\",\n", + buf.toString()); + + req.close(); + */ + } + + + @Test + public void testPseudoFields() throws Exception { + // Use Pseudo Field + /* + assertEquals("1,hi", + h.query(req("q","id:1", "wt","csv", "csv.header","false", "fl","XXX:id,foo_s")).trim()); + + String txt = h.query(req("q","id:1", "wt","csv", "csv.header","true", "fl","XXX:id,YYY:[docid],FOO:foo_s")); + String[] lines = txt.split("\n"); + assertEquals(2, lines.length); + assertEquals("XXX,YYY,FOO", lines[0] ); + assertEquals("1,0,hi", lines[1] ); + + //assertions specific to multiple pseudofields functions like abs, div, exists, etc.. (SOLR-5423) + String funcText = h.query(req("q","*", "wt","csv", "csv.header","true", "fl","XXX:id,YYY:exists(foo_i),exists(shouldbeunstored)")); + String[] funcLines = funcText.split("\n"); + assertEquals(6, funcLines.length); + assertEquals("XXX,YYY,exists(shouldbeunstored)", funcLines[0] ); + assertEquals("1,true,false", funcLines[1] ); + assertEquals("3,false,true", funcLines[3] ); + + + //assertions specific to single function without alias (SOLR-5423) + String singleFuncText = h.query(req("q","*", "wt","csv", "csv.header","true", "fl","exists(shouldbeunstored),XXX:id")); + String[] singleFuncLines = singleFuncText.split("\n"); + assertEquals(6, singleFuncLines.length); + assertEquals("exists(shouldbeunstored),XXX", singleFuncLines[0] ); + assertEquals("false,1", singleFuncLines[1] ); + assertEquals("true,3", singleFuncLines[3] ); + */ + } + + // returns first worksheet as XLSXResponseWriter only returns one sheet + private XSSFSheet getWSResultForQuery(SolrQueryRequest req) throws IOException { + SolrQueryResponse rsp = new SolrQueryResponse(); + ByteArrayOutputStream xmlBout = new ByteArrayOutputStream(); + writerXlsx.write(xmlBout, req, rsp); + XSSFWorkbook output = new XSSFWorkbook(new ByteArrayInputStream(xmlBout.toByteArray())); + + //TODO: DELETEME + File file = new File("/Users/tdm/temp/solr.xlsx"); + FileOutputStream fod = new FileOutputStream(file); + + output.write(fod); + fod.close(); + + return output.getSheetAt(0); + } + + private String getStringFromSheet(XSSFSheet sheet) { + StringBuilder output = new StringBuilder(); + for (Row row: sheet) { + for (Cell cell: row) { + output.append(cell.getStringCellValue()); + output.append(","); + } + output.setLength(output.length() - 1); + output.append("\n"); + } + output.setLength(output.length() - 1); + return output.toString(); + } +} From 4ed97f03e6a52aaac12c7ecbb5ce770d96f53ec7 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Mon, 1 Aug 2016 14:36:22 +1000 Subject: [PATCH 04/21] take column widths/metanames from solrconfig; pull in relevant tests from TestCSVResponseWriter --- .../solr/response/XLSXResponseWriter.java | 69 +- .../conf/solrconfig-xlsxresponsewriter.xml | 597 ++++++++++++++++++ .../solr/response/TestXLSXResponseWriter.java | 197 +++--- 3 files changed, 711 insertions(+), 152 deletions(-) create mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-xlsxresponsewriter.xml diff --git a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java b/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java index 5573d320895a..d5b1c0781f11 100644 --- a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java @@ -45,10 +45,26 @@ */ public class XLSXResponseWriter extends RawResponseWriter { + LinkedHashMap colNamesMap = new LinkedHashMap(); + LinkedHashMap colWidthsMap = new LinkedHashMap(); @Override - public void init(NamedList n) { - //solrconfig calls in here + public void init(NamedList solrconfig) { + NamedList columnNames = (NamedList) solrconfig.get("columnNames"); + if (columnNames != null && columnNames.size() > 0) { + for (Object nameObject: columnNames) { + Map.Entry namePair = (Map.Entry) nameObject; + this.colNamesMap.put(namePair.getKey(), namePair.getValue()); + } + } + + NamedList columnWidths = (NamedList) solrconfig.get("columnWidths"); + if (columnWidths != null && columnWidths.size() > 0) { + for (Object widthObject : columnWidths) { + Map.Entry widthPair = (Map.Entry) widthObject; + this.colWidthsMap.put(widthPair.getKey(), widthPair.getValue()); + } + } } @Override @@ -57,7 +73,7 @@ public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp) // all writes before they go to it anyway XLSXWriter w = new XLSXWriter(new CharArrayWriter(), req, rsp); try { - w.writeResponse(out); + w.writeResponse(out, this.colNamesMap, this.colWidthsMap); } finally { w.close(); } @@ -153,7 +169,8 @@ public XLSXWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp){ this.rsp = rsp; } - public void writeResponse(OutputStream out) throws IOException { + public void writeResponse(OutputStream out, LinkedHashMap colNamesMap, + LinkedHashMap colWidthsMap) throws IOException { SolrParams params = req.getParams(); Collection fields = returnFields.getRequestedFieldNames(); @@ -213,36 +230,7 @@ public void writeResponse(OutputStream out) throws IOException { xlFields.put(field, xlField); } - //TODO remove both of these - class NameAndWidth { - private int width; - private String name; - - public NameAndWidth(String name, int width) { - this.name = name; - this.width = width; - } - - public String getName() { - return this.name; - } - - public int getWidth() { - return this.width; - } - } - - class NiceMetadataNames extends LinkedHashMap { - public NiceMetadataNames() { - super(); - this.put("meta_type_1", new NameAndWidth("Nice Name 1", 14)); - this.put("long_meta_2", new NameAndWidth("Long Metadata name is long", 128)); - } - } - - //TODO and get this from the xml config - NiceMetadataNames niceMap = new NiceMetadataNames(); wb.addRow(); //write header @@ -250,10 +238,14 @@ public NiceMetadataNames() { String printName = xlField.name; int colWidth = 14; - NameAndWidth nextField = niceMap.get(xlField.name); - if (nextField != null) { - printName = nextField.getName();; - colWidth = nextField.getWidth(); + String niceName = colNamesMap.get(xlField.name); + if (niceName != null) { + printName = niceName; + } + + Integer niceWidth = colWidthsMap.get(xlField.name); + if (niceWidth != null) { + colWidth = niceWidth.intValue(); } writeStr(xlField.name, printName, false); @@ -400,8 +392,7 @@ public void writeDouble(String name, String val) throws IOException { @Override public void writeDate(String name, Date val) throws IOException { - String outputDate = DateTimeFormatter.ISO_LOCAL_DATE.format(val.toInstant()); - writeDate(name, outputDate); + writeDate(name, val.toInstant().toString()); } @Override diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-xlsxresponsewriter.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-xlsxresponsewriter.xml new file mode 100644 index 000000000000..f0c6af32af42 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-xlsxresponsewriter.xml @@ -0,0 +1,597 @@ + + + + + + + + + + + + ${solr.data.dir:} + + + + 1000000 + 2000000 + 3000000 + 4000000 + ${solr.hdfs.home:} + ${solr.hdfs.blockcache.enabled:true} + ${solr.hdfs.blockcache.global:true} + ${solr.hdfs.blockcache.write.enabled:false} + 10 + 1 + + + + + ${tests.luceneMatchVersion:LATEST} + + + + + + + + + + + + + + ${solr.ulog.dir:} + + + + ${solr.commitwithin.softcommit:true} + + + + + + + 1024 + + + + + + + + + + + + true + + + + + + 10 + + + + + + + + + + + + + + + 2000 + + + + + + 3 + 10 + 10 + 50 + + + ID + Foo String + Foo Integer + Foo Long + + + + + + + + true + + + + + dismax + *:* + 0.01 + + text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 + + + text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 + + + ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 + + + 3<-1 5<-2 6<90% + + 100 + + + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + 4 + true + text,name,subject,title,whitetok + + + + + + + + lowerpunctfilt + + + default + lowerfilt + spellchecker1 + false + + + direct + DirectSolrSpellChecker + lowerfilt + 3 + + + wordbreak + solr.WordBreakSolrSpellChecker + lowerfilt + true + true + 10 + + + multipleFields + lowerfilt1and2 + spellcheckerMultipleFields + false + + + + jarowinkler + lowerfilt + + org.apache.lucene.search.spell.JaroWinklerDistance + spellchecker2 + + + + solr.FileBasedSpellChecker + external + spellings.txt + UTF-8 + spellchecker3 + + + + freq + lowerfilt + spellcheckerFreq + + freq + false + + + fqcn + lowerfilt + spellcheckerFQCN + org.apache.solr.spelling.SampleComparator + false + + + perDict + org.apache.solr.handler.component.DummyCustomParamSpellChecker + lowerfilt + + + + + + + + termsComp + + + + + + + + + + + false + + false + + 1 + + + spellcheck + + + + + direct + false + false + 1 + + + spellcheck + + + + + default + wordbreak + 20 + + + spellcheck + + + + + direct + wordbreak + 20 + + + spellcheck + + + + + dismax + lowerfilt1^1 + + + spellcheck + + + + + + + + + + + + + + + tvComponent + + + + + + + + + + + + 100 + + + + + + 70 + + + + + + + ]]> + ]]> + + + + + + + + + + + + + 10 + .,!? + + + + + + WORD + en + US + + + + + + + + + + max-age=30, public + + + + + + + explicit + true + + + + + + foo_s + + + foo_s:bar + + + + + foo_s + foo_s:bar + + + + + solr + solrconfig.xml schema.xml admin-extra.html + + + + prefix-${solr.test.sys.prop2}-suffix + + + + + + false + true + v_t,t_field + org.apache.solr.update.processor.TextProfileSignature + + + + + + false + false + id + + org.apache.solr.update.processor.Lookup3Signature + + + + + + + true + non_indexed_signature_sS + false + v_t,t_field + org.apache.solr.update.processor.TextProfileSignature + + + + + + + uniq + uniq2 + uniq3 + + + + + + + + + regex_dup_A_s + x + x_x + + + + regex_dup_B_s + x + x_x + + + + + + + + regex_dup_A_s + x + x_x + + + regex_dup_B_s + x + x_x + + + + + + + org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO + + + + diff --git a/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java index 994f690813cf..ac2eb5ab8f14 100644 --- a/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java @@ -42,6 +42,7 @@ import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.search.SolrReturnFields; import org.junit.AfterClass; @@ -55,9 +56,20 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_ - initCore("solrconfig.xml","schema12.xml"); - TestCSVResponseWriter.createIndex(); - writerXlsx = new XLSXResponseWriter(); + initCore("solrconfig-xlsxresponsewriter.xml","schema12.xml"); + createIndex(); + //find a reference to the default response writer so we can redirect its output later + SolrCore testCore = h.getCore(); + writerXlsx = (XLSXResponseWriter)testCore.getQueryResponseWriter("xlsx"); + } + + public static void createIndex() { + assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false", "foo_f","1.414","foo_d","-1.0E300","foo_dt","2000-01-02T03:04:05Z")); + assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output", "shouldbeunstored","foo")); + assertU(adoc("id","3", "shouldbeunstored","foo")); + assertU(adoc("id","4", "amount_c", "1.50,EUR")); + assertU(adoc("id","5", "store", "12.434,-134.1")); + assertU(commit()); } @AfterClass @@ -78,75 +90,39 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { // check Content-Type assertEquals("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", writerXlsx.getContentType(r, rsp)); + // check it read solrconfig + assertEquals("Foo Integer", writerXlsx.colNamesMap.get("foo_i")); + assertTrue("1 specific width read in from solrconfig", writerXlsx.colWidthsMap.get("foo_l") == 50); + + assertTrue("All names read in from solrconfig", writerXlsx.colNamesMap.size() == 4); + assertTrue("All widths read in from solrconfig", writerXlsx.colWidthsMap.size() == 4); // test our basic types,and that fields come back in the requested order XSSFSheet resultSheet = getWSResultForQuery(req("q","id:1", "wt","xlsx", "fl","id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt")); - String result = getStringFromSheet(resultSheet); - assertEquals("id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt\n1,hi,-1,12345678987654321,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n" - , result); -/* - // test retrieving score, csv.header - assertEquals("1,0.0,hi\n" - , h.query(req("q","id:1^0", "wt","csv", "csv.header","false", "fl","id,score,foo_s"))); + assertEquals("ID,Foo String,Foo Integer,Foo Long,foo_b,foo_f,foo_d,foo_dt\n1,hi,-1,12345678987654321,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n" + , getStringFromSheet(resultSheet)); + + resultSheet = getWSResultForQuery(req("q","id:1^0", "wt","xlsx", "fl","id,score,foo_s")); + // test retrieving score + assertEquals("ID,score,Foo String\n1,0.0,hi\n", getStringFromSheet(resultSheet)); + // test column width (result is in 256ths of a character for some reason) + assertEquals(3*256, resultSheet.getColumnWidth(0)); + resultSheet = getWSResultForQuery(req("q","id:2", "wt","xlsx", "fl","id,v_ss")); // test multivalued - assertEquals("2,\"hi,there\"\n" - , h.query(req("q","id:2", "wt","csv", "csv.header","false", "fl","id,v_ss"))); - - // test separator change - assertEquals("2|\"hi|there\"\n" - , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.separator","|", "fl","id,v_ss"))); - - // test mv separator change - assertEquals("2,hi|there\n" - , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "fl","id,v_ss"))); - - // test mv separator change for a single field - assertEquals("2,hi|there,nice:output\n" - , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "f.v2_ss.csv.separator",":", "fl","id,v_ss,v2_ss"))); - - // test csv field for polyfield (currency) SOLR-3959 - assertEquals("4,\"1.50\\,EUR\"\n" - , h.query(req("q","id:4", "wt","csv", "csv.header","false", "fl","id,amount_c"))); - - // test csv field for polyfield (latlon) SOLR-3959 - assertEquals("5,\"12.434\\,-134.1\"\n" - , h.query(req("q","id:5", "wt","csv", "csv.header","false", "fl","id,store")) ); + assertEquals("ID,v_ss\n2,hi; there\n", getStringFromSheet(resultSheet)); + // test retrieving fields from index - String result = h.query(req("q","*:*", "wt","csv", "csv.header","true", "fl","*,score")); - for (String field : "id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss,score".split(",")) { + resultSheet = getWSResultForQuery(req("q","*:*", "wt","xslx", "fl","*,score")); + String result = getStringFromSheet(resultSheet); + for (String field : "ID,Foo String,Foo Integer,Foo Long,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss,score".split(",")) { assertTrue(result.indexOf(field) >= 0); } // test null values - assertEquals("2,,hi|there\n" - , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "fl","id,foo_s,v_ss"))); - - // test alternate null value - assertEquals("2,NULL,hi|there\n" - , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "csv.null","NULL","fl","id,foo_s,v_ss"))); - - // test alternate newline - assertEquals("2,\"hi,there\"\r\n" - , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.newline","\r\n", "fl","id,v_ss"))); - - // test alternate encapsulator - assertEquals("2,'hi,there'\n" - , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.encapsulator","'", "fl","id,v_ss"))); - - // test using escape instead of encapsulator - assertEquals("2,hi\\,there\n" - , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.escape","\\", "fl","id,v_ss"))); - - // test multiple lines - assertEquals("1,,hi\n2,\"hi,there\",\n" - , h.query(req("q","id:[1 TO 2]", "wt","csv", "csv.header","false", "fl","id,v_ss,foo_s"))); - - // test SOLR-2970 not returning non-stored fields by default. Compare sorted list - assertEquals(sortHeader("amount_c,store,v_ss,foo_b,v2_ss,foo_f,foo_i,foo_d,foo_s,foo_dt,id,foo_l\n") - , sortHeader(h.query(req("q","id:3", "wt","csv", "csv.header","true", "fl","*", "rows","0")))); - + resultSheet = getWSResultForQuery(req("q","id:2", "wt","xlsx", "fl","id,foo_s,v_ss")); + assertEquals("ID,Foo String,v_ss\n2,,hi; there\n", getStringFromSheet(resultSheet)); // now test SolrDocumentList SolrDocument d = new SolrDocument(); @@ -176,93 +152,86 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { sdl.add(d2); SolrQueryRequest req = req("q","*:*"); - SolrQueryResponse rsp = new SolrQueryResponse(); + rsp = new SolrQueryResponse(); rsp.addResponse(sdl); - QueryResponseWriter w = new CSVResponseWriter(); - + rsp.setReturnFields( new SolrReturnFields("id,foo_s", req) ); - StringWriter buf = new StringWriter(); - w.write(buf, req, rsp); - assertEquals("id,foo_s\n1,hi\n2,\n", buf.toString()); + + resultSheet = getWSResultForQuery(req, rsp); + assertEquals("ID,Foo String\n1,hi\n2,\n", getStringFromSheet(resultSheet)); // try scores rsp.setReturnFields( new SolrReturnFields("id,score,foo_s", req) ); - buf = new StringWriter(); - w.write(buf, req, rsp); - assertEquals("id,score,foo_s\n1,2.718,hi\n2,89.83,\n", buf.toString()); + + resultSheet = getWSResultForQuery(req, rsp); + assertEquals("ID,score,Foo String\n1,2.718,hi\n2,89.83,\n", getStringFromSheet(resultSheet)); // get field values from docs... should be ordered and not include score unless requested rsp.setReturnFields( new SolrReturnFields("*", req) ); - buf = new StringWriter(); - w.write(buf, req, rsp); - assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss\n" + + + resultSheet = getWSResultForQuery(req, rsp); + assertEquals("ID,Foo Integer,Foo String,Foo Long,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss\n" + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z,,\n" + - "2,,,,,,,,\"hi,there\",\"nice,output\"\n", - buf.toString()); + "2,,,,,,,,hi; there,nice; output\n", + getStringFromSheet(resultSheet)); // get field values and scores - just check that the scores are there... we don't guarantee where rsp.setReturnFields( new SolrReturnFields("*,score", req) ); - buf = new StringWriter(); - w.write(buf, req, rsp); - String s = buf.toString(); + resultSheet = getWSResultForQuery(req, rsp); + String s = getStringFromSheet(resultSheet); assertTrue(s.indexOf("score") >=0 && s.indexOf("2.718") > 0 && s.indexOf("89.83") > 0 ); // Test field globs rsp.setReturnFields( new SolrReturnFields("id,foo*", req) ); - buf = new StringWriter(); - w.write(buf, req, rsp); - assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt\n" + + resultSheet = getWSResultForQuery(req, rsp); + assertEquals("ID,Foo Integer,Foo String,Foo Long,foo_b,foo_f,foo_d,foo_dt\n" + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n" + "2,,,,,,,\n", - buf.toString()); + getStringFromSheet(resultSheet)); rsp.setReturnFields( new SolrReturnFields("id,*_d*", req) ); - buf = new StringWriter(); - w.write(buf, req, rsp); - assertEquals("id,foo_d,foo_dt\n" + + resultSheet = getWSResultForQuery(req, rsp); + assertEquals("ID,foo_d,foo_dt\n" + "1,-1.0E300,2000-01-02T03:04:05Z\n" + "2,,\n", - buf.toString()); + getStringFromSheet(resultSheet)); // Test function queries rsp.setReturnFields( new SolrReturnFields("sum(1,1),id,exists(foo_i),div(9,1),foo_f", req) ); - buf = new StringWriter(); - w.write(buf, req, rsp); - assertEquals("\"sum(1,1)\",id,exists(foo_i),\"div(9,1)\",foo_f\n" + - "\"\",1,,,1.414\n" + - "\"\",2,,,\n", - buf.toString()); + resultSheet = getWSResultForQuery(req, rsp); + assertEquals("sum(1,1),ID,exists(foo_i),div(9,1),foo_f\n" + + ",1,,,1.414\n" + + ",2,,,\n", + getStringFromSheet(resultSheet)); // Test transformers rsp.setReturnFields( new SolrReturnFields("mydocid:[docid],[explain]", req) ); - buf = new StringWriter(); - w.write(buf, req, rsp); + resultSheet = getWSResultForQuery(req, rsp); assertEquals("mydocid,[explain]\n" + - "\"\",\n" + - "\"\",\n", - buf.toString()); + ",\n" + + ",\n", + getStringFromSheet(resultSheet)); req.close(); - */ } @Test public void testPseudoFields() throws Exception { // Use Pseudo Field - /* - assertEquals("1,hi", - h.query(req("q","id:1", "wt","csv", "csv.header","false", "fl","XXX:id,foo_s")).trim()); + SolrQueryRequest req = req("q","id:1", "wt","xlsx", "fl","XXX:id,foo_s"); + XSSFSheet resultSheet = getWSResultForQuery(req); + assertEquals("XXX,Foo String\n1,hi\n", getStringFromSheet(resultSheet)); - String txt = h.query(req("q","id:1", "wt","csv", "csv.header","true", "fl","XXX:id,YYY:[docid],FOO:foo_s")); + String txt = getStringFromSheet(getWSResultForQuery(req("q","id:1", "wt","xlsx", "fl","XXX:id,YYY:[docid],FOO:foo_s"))); String[] lines = txt.split("\n"); assertEquals(2, lines.length); assertEquals("XXX,YYY,FOO", lines[0] ); assertEquals("1,0,hi", lines[1] ); //assertions specific to multiple pseudofields functions like abs, div, exists, etc.. (SOLR-5423) - String funcText = h.query(req("q","*", "wt","csv", "csv.header","true", "fl","XXX:id,YYY:exists(foo_i),exists(shouldbeunstored)")); + String funcText = getStringFromSheet(getWSResultForQuery(req("q","*", "wt","xlsx", "fl","XXX:id,YYY:exists(foo_i),exists(shouldbeunstored)"))); String[] funcLines = funcText.split("\n"); assertEquals(6, funcLines.length); assertEquals("XXX,YYY,exists(shouldbeunstored)", funcLines[0] ); @@ -271,29 +240,32 @@ public void testPseudoFields() throws Exception { //assertions specific to single function without alias (SOLR-5423) - String singleFuncText = h.query(req("q","*", "wt","csv", "csv.header","true", "fl","exists(shouldbeunstored),XXX:id")); + String singleFuncText = getStringFromSheet(getWSResultForQuery(req("q","*", "wt","xlsx", "fl","exists(shouldbeunstored),XXX:id"))); String[] singleFuncLines = singleFuncText.split("\n"); assertEquals(6, singleFuncLines.length); assertEquals("exists(shouldbeunstored),XXX", singleFuncLines[0] ); assertEquals("false,1", singleFuncLines[1] ); assertEquals("true,3", singleFuncLines[3] ); - */ } // returns first worksheet as XLSXResponseWriter only returns one sheet - private XSSFSheet getWSResultForQuery(SolrQueryRequest req) throws IOException { - SolrQueryResponse rsp = new SolrQueryResponse(); + private XSSFSheet getWSResultForQuery(SolrQueryRequest req) throws IOException, Exception { + SolrQueryResponse rsp = h.queryAndResponse("standard", req); + return getWSResultForQuery(req, rsp); + } + + private XSSFSheet getWSResultForQuery(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, Exception { ByteArrayOutputStream xmlBout = new ByteArrayOutputStream(); writerXlsx.write(xmlBout, req, rsp); XSSFWorkbook output = new XSSFWorkbook(new ByteArrayInputStream(xmlBout.toByteArray())); //TODO: DELETEME - File file = new File("/Users/tdm/temp/solr.xlsx"); - FileOutputStream fod = new FileOutputStream(file); - - output.write(fod); - fod.close(); + //File file = new File("/Users/tdm/temp/solr.xlsx"); + //FileOutputStream fod = new FileOutputStream(file); + //output.write(fod); + //fod.close(); + req.close(); return output.getSheetAt(0); } @@ -307,7 +279,6 @@ private String getStringFromSheet(XSSFSheet sheet) { output.setLength(output.length() - 1); output.append("\n"); } - output.setLength(output.length() - 1); return output.toString(); } } From 4fed3ab1591fd69073e80c4b576b996f44698b33 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Mon, 1 Aug 2016 14:48:52 +1000 Subject: [PATCH 05/21] remove unused write-to-disk of xlsx output --- .../org/apache/solr/response/TestXLSXResponseWriter.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java index ac2eb5ab8f14..4800d8b6f108 100644 --- a/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java @@ -259,12 +259,6 @@ private XSSFSheet getWSResultForQuery(SolrQueryRequest req, SolrQueryResponse rs writerXlsx.write(xmlBout, req, rsp); XSSFWorkbook output = new XSSFWorkbook(new ByteArrayInputStream(xmlBout.toByteArray())); - //TODO: DELETEME - //File file = new File("/Users/tdm/temp/solr.xlsx"); - //FileOutputStream fod = new FileOutputStream(file); - //output.write(fod); - //fod.close(); - req.close(); return output.getSheetAt(0); } From 30c93acfb5f1d22c6f0fcf7640946385957d270d Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Mon, 1 Aug 2016 15:00:17 +1000 Subject: [PATCH 06/21] AL header on new file --- .../apache/solr/response/XLSXResponseWriter.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java b/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java index d5b1c0781f11..17c115895b1a 100644 --- a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.solr.response; import com.google.common.collect.Iterables; From 8edcb84b482e3f9c5fa80cc8984d8a6271d6ceda Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Mon, 1 Aug 2016 15:15:24 +1000 Subject: [PATCH 07/21] remove unused imports --- .../src/java/org/apache/solr/response/XLSXResponseWriter.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java b/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java index 17c115895b1a..b609e5515a39 100644 --- a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java @@ -30,7 +30,6 @@ import org.apache.solr.schema.StrField; import org.apache.solr.search.DocList; import org.apache.solr.search.ReturnFields; -import org.apache.solr.core.SolrCore; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; @@ -40,7 +39,6 @@ import org.apache.poi.xssf.streaming.SXSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFCellStyle; -import java.time.format.DateTimeFormatter; import java.io.IOException; import java.io.Writer; import java.io.CharArrayWriter; From a6ecf0508a67e0f038193c34133e5cdca3e938c0 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Tue, 9 Aug 2016 15:02:11 +1000 Subject: [PATCH 08/21] revert poi -> core --- solr/core/ivy.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/solr/core/ivy.xml b/solr/core/ivy.xml index 822dae945b0f..08272ad74482 100644 --- a/solr/core/ivy.xml +++ b/solr/core/ivy.xml @@ -131,10 +131,6 @@ - - - - From c08324f3695244b13156f4ba788a6fa75a2327af Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Thu, 18 Aug 2016 16:02:58 +1000 Subject: [PATCH 09/21] Moved xlsxresponsewriter into extraction contrib; hook to autoregister with core --- .../extraction}/XLSXResponseWriter.java | 82 ++- .../extraction}/TestXLSXResponseWriter.java | 48 +- .../conf/solrconfig-xlsxresponsewriter.xml | 597 ------------------ 3 files changed, 71 insertions(+), 656 deletions(-) rename solr/{core/src/java/org/apache/solr/response => contrib/extraction/src/java/org/apache/solr/handler/extraction}/XLSXResponseWriter.java (91%) rename solr/{core/src/test/org/apache/solr/response => contrib/extraction/src/test/org/apache/solr/handler/extraction}/TestXLSXResponseWriter.java (87%) delete mode 100644 solr/core/src/test-files/solr/collection1/conf/solrconfig-xlsxresponsewriter.xml diff --git a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java similarity index 91% rename from solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java rename to solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java index b609e5515a39..b3549b6675ea 100644 --- a/solr/core/src/java/org/apache/solr/response/XLSXResponseWriter.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java @@ -14,51 +14,54 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.response; +package org.apache.solr.handler.extraction; + +import java.io.CharArrayWriter; +import java.io.IOException; +import java.io.OutputStream; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.io.Writer; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import org.apache.lucene.index.IndexableField; +import org.apache.poi.ss.usermodel.Cell; +import org.apache.poi.ss.usermodel.Font; +import org.apache.poi.ss.usermodel.IndexedColors; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.streaming.SXSSFWorkbook; +import org.apache.poi.xssf.usermodel.XSSFCellStyle; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; - import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.PluginBag; +import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.BasicResultContext; +import org.apache.solr.response.QueryResponseWriter; +import org.apache.solr.response.RawResponseWriter; +import org.apache.solr.response.ResultContext; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.response.TextResponseWriter; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.StrField; import org.apache.solr.search.DocList; import org.apache.solr.search.ReturnFields; +import org.apache.solr.util.plugin.SolrCoreAware; -import org.apache.poi.ss.usermodel.Cell; -import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.ss.usermodel.Sheet; -import org.apache.poi.ss.usermodel.IndexedColors; -import org.apache.poi.ss.usermodel.Font; -import org.apache.poi.xssf.streaming.SXSSFWorkbook; -import org.apache.poi.xssf.usermodel.XSSFCellStyle; - -import java.io.IOException; -import java.io.Writer; -import java.io.CharArrayWriter; -import java.io.StringWriter; -import java.io.PrintWriter; -import java.io.OutputStream; -import java.util.Date; -import java.util.Iterator; -import java.util.Map; -import java.util.List; -import java.util.ArrayList; -import java.util.Collection; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; - -/** - * - */ - -public class XLSXResponseWriter extends RawResponseWriter { +public class XLSXResponseWriter extends RawResponseWriter implements SolrCoreAware { LinkedHashMap colNamesMap = new LinkedHashMap(); LinkedHashMap colWidthsMap = new LinkedHashMap(); @@ -93,6 +96,19 @@ public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp) } } + @Override + public void inform(SolrCore core) { + PluginBag writerBag = core.getResponseWriters(); + for( String writerKey : writerBag.keySet() ) { + if( writerBag.get(writerKey) == this ) { + //already registered + return; + } + } + + core.registerResponseWriter("xlsx", this); + } + @Override public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; @@ -362,7 +378,11 @@ public void writeArray(String name, Iterator val) throws IOException { Object v = val.next(); if (v instanceof IndexableField) { IndexableField f = (IndexableField)v; - output.append(f.stringValue() + "; "); + if (v instanceof Date) { + output.append(((Date) val).toInstant().toString() + "; "); + } else { + output.append(f.stringValue() + "; "); + } } else { output.append(v.toString() + "; "); } diff --git a/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java similarity index 87% rename from solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java rename to solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java index 4800d8b6f108..cc6257c76289 100644 --- a/solr/core/src/test/org/apache/solr/response/TestXLSXResponseWriter.java +++ b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java @@ -14,7 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.solr.response; +package org.apache.solr.handler.extraction; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -44,6 +44,8 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.response.RawResponseWriter; import org.apache.solr.search.SolrReturnFields; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -56,7 +58,7 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_ - initCore("solrconfig-xlsxresponsewriter.xml","schema12.xml"); + initCore("solrconfig.xml","schema.xml",getFile("extraction/solr").getAbsolutePath()); createIndex(); //find a reference to the default response writer so we can redirect its output later SolrCore testCore = h.getCore(); @@ -64,11 +66,10 @@ public static void beforeClass() throws Exception { } public static void createIndex() { - assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false", "foo_f","1.414","foo_d","-1.0E300","foo_dt","2000-01-02T03:04:05Z")); + assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false", "foo_f","1.414","foo_d","-1.0E300","foo_dt1","2000-01-02T03:04:05Z")); assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output", "shouldbeunstored","foo")); assertU(adoc("id","3", "shouldbeunstored","foo")); - assertU(adoc("id","4", "amount_c", "1.50,EUR")); - assertU(adoc("id","5", "store", "12.434,-134.1")); + assertU(adoc("id","4", "foo_s1","foo")); assertU(commit()); } @@ -98,9 +99,9 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { assertTrue("All widths read in from solrconfig", writerXlsx.colWidthsMap.size() == 4); // test our basic types,and that fields come back in the requested order - XSSFSheet resultSheet = getWSResultForQuery(req("q","id:1", "wt","xlsx", "fl","id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt")); + XSSFSheet resultSheet = getWSResultForQuery(req("q","id:1", "wt","xlsx", "fl","id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt1")); - assertEquals("ID,Foo String,Foo Integer,Foo Long,foo_b,foo_f,foo_d,foo_dt\n1,hi,-1,12345678987654321,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n" + assertEquals("ID,Foo String,Foo Integer,Foo Long,foo_b,foo_f,foo_d,foo_dt1\n1,hi,-1,12345678987654321,F,1.414,-1.0E300,2000-01-02T03:04:05Z\n" , getStringFromSheet(resultSheet)); resultSheet = getWSResultForQuery(req("q","id:1^0", "wt","xlsx", "fl","id,score,foo_s")); @@ -116,7 +117,7 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { // test retrieving fields from index resultSheet = getWSResultForQuery(req("q","*:*", "wt","xslx", "fl","*,score")); String result = getStringFromSheet(resultSheet); - for (String field : "ID,Foo String,Foo Integer,Foo Long,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss,score".split(",")) { + for (String field : "ID,Foo String,Foo Integer,Foo Long,foo_b,foo_f,foo_d,foo_dt1,v_ss,v2_ss,score".split(",")) { assertTrue(result.indexOf(field) >= 0); } @@ -134,7 +135,7 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { d.addField("foo_b",false); d.addField("foo_f",1.414f); d.addField("foo_d",-1.0E300); - d.addField("foo_dt", new Date(Instant.parse("2000-01-02T03:04:05Z").toEpochMilli())); + d.addField("foo_dt1", new Date(Instant.parse("2000-01-02T03:04:05Z").toEpochMilli())); d.addField("score", "2.718"); d = new SolrDocument(); @@ -170,7 +171,7 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { rsp.setReturnFields( new SolrReturnFields("*", req) ); resultSheet = getWSResultForQuery(req, rsp); - assertEquals("ID,Foo Integer,Foo String,Foo Long,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss\n" + + assertEquals("ID,Foo Integer,Foo String,Foo Long,foo_b,foo_f,foo_d,foo_dt1,v_ss,v2_ss\n" + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z,,\n" + "2,,,,,,,,hi; there,nice; output\n", getStringFromSheet(resultSheet)); @@ -185,22 +186,22 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { // Test field globs rsp.setReturnFields( new SolrReturnFields("id,foo*", req) ); resultSheet = getWSResultForQuery(req, rsp); - assertEquals("ID,Foo Integer,Foo String,Foo Long,foo_b,foo_f,foo_d,foo_dt\n" + + assertEquals("ID,Foo Integer,Foo String,Foo Long,foo_b,foo_f,foo_d,foo_dt1\n" + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n" + "2,,,,,,,\n", getStringFromSheet(resultSheet)); rsp.setReturnFields( new SolrReturnFields("id,*_d*", req) ); resultSheet = getWSResultForQuery(req, rsp); - assertEquals("ID,foo_d,foo_dt\n" + + assertEquals("ID,foo_d,foo_dt1\n" + "1,-1.0E300,2000-01-02T03:04:05Z\n" + "2,,\n", getStringFromSheet(resultSheet)); // Test function queries - rsp.setReturnFields( new SolrReturnFields("sum(1,1),id,exists(foo_i),div(9,1),foo_f", req) ); + rsp.setReturnFields( new SolrReturnFields("sum(1,1),id,exists(foo_s1),div(9,1),foo_f", req) ); resultSheet = getWSResultForQuery(req, rsp); - assertEquals("sum(1,1),ID,exists(foo_i),div(9,1),foo_f\n" + + assertEquals("sum(1,1),ID,exists(foo_s1),div(9,1),foo_f\n" + ",1,,,1.414\n" + ",2,,,\n", getStringFromSheet(resultSheet)); @@ -231,21 +232,12 @@ public void testPseudoFields() throws Exception { assertEquals("1,0,hi", lines[1] ); //assertions specific to multiple pseudofields functions like abs, div, exists, etc.. (SOLR-5423) - String funcText = getStringFromSheet(getWSResultForQuery(req("q","*", "wt","xlsx", "fl","XXX:id,YYY:exists(foo_i),exists(shouldbeunstored)"))); + String funcText = getStringFromSheet(getWSResultForQuery(req("q","*", "wt","xlsx", "fl","XXX:id,YYY:exists(foo_s1)"))); String[] funcLines = funcText.split("\n"); - assertEquals(6, funcLines.length); - assertEquals("XXX,YYY,exists(shouldbeunstored)", funcLines[0] ); - assertEquals("1,true,false", funcLines[1] ); - assertEquals("3,false,true", funcLines[3] ); - - - //assertions specific to single function without alias (SOLR-5423) - String singleFuncText = getStringFromSheet(getWSResultForQuery(req("q","*", "wt","xlsx", "fl","exists(shouldbeunstored),XXX:id"))); - String[] singleFuncLines = singleFuncText.split("\n"); - assertEquals(6, singleFuncLines.length); - assertEquals("exists(shouldbeunstored),XXX", singleFuncLines[0] ); - assertEquals("false,1", singleFuncLines[1] ); - assertEquals("true,3", singleFuncLines[3] ); + assertEquals(5, funcLines.length); + assertEquals("XXX,YYY", funcLines[0] ); + assertEquals("1,false", funcLines[1] ); + assertEquals("3,false", funcLines[3] ); } // returns first worksheet as XLSXResponseWriter only returns one sheet diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-xlsxresponsewriter.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-xlsxresponsewriter.xml deleted file mode 100644 index f0c6af32af42..000000000000 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-xlsxresponsewriter.xml +++ /dev/null @@ -1,597 +0,0 @@ - - - - - - - - - - - - ${solr.data.dir:} - - - - 1000000 - 2000000 - 3000000 - 4000000 - ${solr.hdfs.home:} - ${solr.hdfs.blockcache.enabled:true} - ${solr.hdfs.blockcache.global:true} - ${solr.hdfs.blockcache.write.enabled:false} - 10 - 1 - - - - - ${tests.luceneMatchVersion:LATEST} - - - - - - - - - - - - - - ${solr.ulog.dir:} - - - - ${solr.commitwithin.softcommit:true} - - - - - - - 1024 - - - - - - - - - - - - true - - - - - - 10 - - - - - - - - - - - - - - - 2000 - - - - - - 3 - 10 - 10 - 50 - - - ID - Foo String - Foo Integer - Foo Long - - - - - - - - true - - - - - dismax - *:* - 0.01 - - text^0.5 features_t^1.0 subject^1.4 title_stemmed^2.0 - - - text^0.2 features_t^1.1 subject^1.4 title_stemmed^2.0 title^1.5 - - - ord(weight)^0.5 recip(rord(iind),1,1000,1000)^0.3 - - - 3<-1 5<-2 6<90% - - 100 - - - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - 4 - true - text,name,subject,title,whitetok - - - - - - - - lowerpunctfilt - - - default - lowerfilt - spellchecker1 - false - - - direct - DirectSolrSpellChecker - lowerfilt - 3 - - - wordbreak - solr.WordBreakSolrSpellChecker - lowerfilt - true - true - 10 - - - multipleFields - lowerfilt1and2 - spellcheckerMultipleFields - false - - - - jarowinkler - lowerfilt - - org.apache.lucene.search.spell.JaroWinklerDistance - spellchecker2 - - - - solr.FileBasedSpellChecker - external - spellings.txt - UTF-8 - spellchecker3 - - - - freq - lowerfilt - spellcheckerFreq - - freq - false - - - fqcn - lowerfilt - spellcheckerFQCN - org.apache.solr.spelling.SampleComparator - false - - - perDict - org.apache.solr.handler.component.DummyCustomParamSpellChecker - lowerfilt - - - - - - - - termsComp - - - - - - - - - - - false - - false - - 1 - - - spellcheck - - - - - direct - false - false - 1 - - - spellcheck - - - - - default - wordbreak - 20 - - - spellcheck - - - - - direct - wordbreak - 20 - - - spellcheck - - - - - dismax - lowerfilt1^1 - - - spellcheck - - - - - - - - - - - - - - - tvComponent - - - - - - - - - - - - 100 - - - - - - 70 - - - - - - - ]]> - ]]> - - - - - - - - - - - - - 10 - .,!? - - - - - - WORD - en - US - - - - - - - - - - max-age=30, public - - - - - - - explicit - true - - - - - - foo_s - - - foo_s:bar - - - - - foo_s - foo_s:bar - - - - - solr - solrconfig.xml schema.xml admin-extra.html - - - - prefix-${solr.test.sys.prop2}-suffix - - - - - - false - true - v_t,t_field - org.apache.solr.update.processor.TextProfileSignature - - - - - - false - false - id - - org.apache.solr.update.processor.Lookup3Signature - - - - - - - true - non_indexed_signature_sS - false - v_t,t_field - org.apache.solr.update.processor.TextProfileSignature - - - - - - - uniq - uniq2 - uniq3 - - - - - - - - - regex_dup_A_s - x - x_x - - - - regex_dup_B_s - x - x_x - - - - - - - - regex_dup_A_s - x - x_x - - - regex_dup_B_s - x - x_x - - - - - - - org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO - - - - From 480a97be27bb9c8646cf26a0f448c0c42d62ef88 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Thu, 18 Aug 2016 16:20:26 +1000 Subject: [PATCH 10/21] added xlsx writer config for testing --- .../solr/collection1/conf/solrconfig.xml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml index ed33c6b5cd86..3057625203fe 100644 --- a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml +++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml @@ -203,6 +203,22 @@ + + + 3 + 10 + 10 + 50 + + + ID + Foo String + Foo Integer + Foo Long + + + From a7d761d6d16bdcfecc9e4fde3a584079692ac370 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Thu, 18 Aug 2016 17:10:58 +1000 Subject: [PATCH 11/21] deal more gracefully with the xlsx writer being missing in its test --- .../extraction/solr/collection1/conf/schema.xml | 2 ++ .../handler/extraction/TestXLSXResponseWriter.java | 10 ++++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml index 35d123f91676..bd9adbe47ba3 100644 --- a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml +++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml @@ -415,6 +415,7 @@ --> + @@ -422,6 +423,7 @@ + diff --git a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java index cc6257c76289..104a68f3f772 100644 --- a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java +++ b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java @@ -44,6 +44,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.response.RawResponseWriter; import org.apache.solr.search.SolrReturnFields; @@ -57,12 +58,17 @@ public class TestXLSXResponseWriter extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - System.setProperty("enable.update.log", "false"); // schema12 doesn't support _version_ + System.setProperty("enable.update.log", "false"); initCore("solrconfig.xml","schema.xml",getFile("extraction/solr").getAbsolutePath()); createIndex(); //find a reference to the default response writer so we can redirect its output later SolrCore testCore = h.getCore(); - writerXlsx = (XLSXResponseWriter)testCore.getQueryResponseWriter("xlsx"); + QueryResponseWriter writer = testCore.getQueryResponseWriter("xlsx"); + if (writer instanceof XLSXResponseWriter) { + writerXlsx = (XLSXResponseWriter) testCore.getQueryResponseWriter("xlsx"); + } else { + throw new Exception("XLSXResponseWriter not registered - are you using the right solrconfig?"); + } } public static void createIndex() { From 2a08c4f1afca3a472fec2dcf0974bfc6e0a2a7a4 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Thu, 18 Aug 2016 20:08:15 +1000 Subject: [PATCH 12/21] remove inform function; it's no use --- .../handler/extraction/XLSXResponseWriter.java | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java index b3549b6675ea..cd818d48d5a9 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java @@ -61,7 +61,7 @@ import org.apache.solr.search.ReturnFields; import org.apache.solr.util.plugin.SolrCoreAware; -public class XLSXResponseWriter extends RawResponseWriter implements SolrCoreAware { +public class XLSXResponseWriter extends RawResponseWriter { LinkedHashMap colNamesMap = new LinkedHashMap(); LinkedHashMap colWidthsMap = new LinkedHashMap(); @@ -96,19 +96,6 @@ public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp) } } - @Override - public void inform(SolrCore core) { - PluginBag writerBag = core.getResponseWriters(); - for( String writerKey : writerBag.keySet() ) { - if( writerBag.get(writerKey) == this ) { - //already registered - return; - } - } - - core.registerResponseWriter("xlsx", this); - } - @Override public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; From e30f5c722b394cc5fefbaccf5f8c10f4e3de90d6 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Thu, 18 Aug 2016 21:02:17 +1000 Subject: [PATCH 13/21] remove unused imports --- .../apache/solr/handler/extraction/XLSXResponseWriter.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java index cd818d48d5a9..85fff2b6859f 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java @@ -45,11 +45,8 @@ import org.apache.solr.common.SolrDocumentList; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; -import org.apache.solr.core.PluginBag; -import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.BasicResultContext; -import org.apache.solr.response.QueryResponseWriter; import org.apache.solr.response.RawResponseWriter; import org.apache.solr.response.ResultContext; import org.apache.solr.response.SolrQueryResponse; @@ -59,7 +56,6 @@ import org.apache.solr.schema.StrField; import org.apache.solr.search.DocList; import org.apache.solr.search.ReturnFields; -import org.apache.solr.util.plugin.SolrCoreAware; public class XLSXResponseWriter extends RawResponseWriter { LinkedHashMap colNamesMap = new LinkedHashMap(); From 81b69c14022113528f954ed557d5d15a43703857 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Thu, 18 Aug 2016 21:02:46 +1000 Subject: [PATCH 14/21] remove unused imports --- .../extraction/TestXLSXResponseWriter.java | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java index 104a68f3f772..29704f63fe5b 100644 --- a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java +++ b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java @@ -18,30 +18,18 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; -import java.nio.charset.StandardCharsets; import java.time.Instant; -import java.util.Arrays; import java.util.Date; -import java.util.Iterator; import org.apache.poi.ss.usermodel.Cell; import org.apache.poi.ss.usermodel.Row; -import org.apache.poi.xssf.usermodel.XSSFCell; -import org.apache.poi.xssf.usermodel.XSSFRow; import org.apache.poi.xssf.usermodel.XSSFWorkbook; import org.apache.poi.xssf.usermodel.XSSFSheet; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.impl.BinaryResponseParser; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.QueryResponseWriter; @@ -256,9 +244,10 @@ private XSSFSheet getWSResultForQuery(SolrQueryRequest req, SolrQueryResponse rs ByteArrayOutputStream xmlBout = new ByteArrayOutputStream(); writerXlsx.write(xmlBout, req, rsp); XSSFWorkbook output = new XSSFWorkbook(new ByteArrayInputStream(xmlBout.toByteArray())); - + XSSFSheet sheet = output.getSheetAt(0); req.close(); - return output.getSheetAt(0); + output.close(); + return sheet; } private String getStringFromSheet(XSSFSheet sheet) { From 59752c31bd16c58eced12a45d88a46db286aea24 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Sun, 21 Aug 2016 20:00:37 -0700 Subject: [PATCH 15/21] rename column* from xlsxwriter config -> col* --- .../extraction/solr/collection1/conf/solrconfig.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml index 3057625203fe..c38e27ee8f92 100644 --- a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml +++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml @@ -205,13 +205,13 @@ - + 3 10 10 50 - + ID Foo String Foo Integer From ea460a17d7a894445bffefaf1e803db975096317 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Sun, 21 Aug 2016 20:01:29 -0700 Subject: [PATCH 16/21] try to put register xlsx writer if available --- solr/core/src/java/org/apache/solr/core/SolrCore.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 2704e4ac8688..92586f20d2c0 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -2197,6 +2197,12 @@ public PluginBag getResponseWriters() { m.put("smile", new SmileResponseWriter()); m.put(ReplicationHandler.FILE_STREAM, getFileStreamWriter()); DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m); + try { + m.put("xlsx", + (QueryResponseWriter) Class.forName("org.apache.solr.handler.extraction.XLSXResponseWriter").newInstance()); + } catch (Exception e) { + //don't worry; solrcell contrib not in class path + } } private static BinaryResponseWriter getFileStreamWriter() { From 94bfc80ed8e7d11af7867b7dc39a6ccfc0999695 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Sun, 21 Aug 2016 20:02:20 -0700 Subject: [PATCH 17/21] allow colnames.foo and colwidths.foo as request params --- .../extraction/XLSXResponseWriter.java | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java index 85fff2b6859f..e4da9510f7cd 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java @@ -63,7 +63,7 @@ public class XLSXResponseWriter extends RawResponseWriter { @Override public void init(NamedList solrconfig) { - NamedList columnNames = (NamedList) solrconfig.get("columnNames"); + NamedList columnNames = (NamedList) solrconfig.get("colnames"); if (columnNames != null && columnNames.size() > 0) { for (Object nameObject: columnNames) { Map.Entry namePair = (Map.Entry) nameObject; @@ -71,7 +71,7 @@ public void init(NamedList solrconfig) { } } - NamedList columnWidths = (NamedList) solrconfig.get("columnWidths"); + NamedList columnWidths = (NamedList) solrconfig.get("colwidths"); if (columnWidths != null && columnWidths.size() > 0) { for (Object widthObject : columnWidths) { Map.Entry widthPair = (Map.Entry) widthObject; @@ -85,8 +85,25 @@ public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp) // throw away arraywriter just to satisfy super requirements; we're grabbing // all writes before they go to it anyway XLSXWriter w = new XLSXWriter(new CharArrayWriter(), req, rsp); + + // copy these as each request may modify them within its context + LinkedHashMap reqNamesMap = colNamesMap;//.copy(); + LinkedHashMap reqWidthsMap = colWidthsMap;//.copy(); + + Iterator paramNamesIter = req.getParams().getParameterNamesIterator(); + while (paramNamesIter.hasNext()) { + String nextParam = paramNamesIter.next(); + if (nextParam.startsWith("colname.")) { + String field = nextParam.substring("colname.".length()); + reqNamesMap.put(field, req.getParams().get(nextParam)); + } else if (nextParam.startsWith("colwidth.")) { + String field = nextParam.substring("colwidth.".length()); + reqWidthsMap.put(field, req.getParams().getInt(nextParam)); + } + } + try { - w.writeResponse(out, this.colNamesMap, this.colWidthsMap); + w.writeResponse(out, reqNamesMap, reqWidthsMap); } finally { w.close(); } From 212be506dc3f07a7bfb33ebe51e297cdd52c48c5 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Sun, 21 Aug 2016 20:02:55 -0700 Subject: [PATCH 18/21] test colnames.foo and colwidths.foo request params --- .../extraction/TestXLSXResponseWriter.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java index 29704f63fe5b..7a7ee1a6871e 100644 --- a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java +++ b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java @@ -55,7 +55,7 @@ public static void beforeClass() throws Exception { if (writer instanceof XLSXResponseWriter) { writerXlsx = (XLSXResponseWriter) testCore.getQueryResponseWriter("xlsx"); } else { - throw new Exception("XLSXResponseWriter not registered - are you using the right solrconfig?"); + throw new Exception("XLSXResponseWriter not registered with solr core"); } } @@ -104,6 +104,12 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { // test column width (result is in 256ths of a character for some reason) assertEquals(3*256, resultSheet.getColumnWidth(0)); + resultSheet = getWSResultForQuery(req("q","id:1^0", "wt","xlsx", "colname.id", "I.D.", "colwidth.id", "10", + "fl","id,score,foo_s")); + // test override colname/width + assertEquals("I.D.,score,Foo String\n1,0.0,hi\n", getStringFromSheet(resultSheet)); + assertEquals(10*256, resultSheet.getColumnWidth(0)); + resultSheet = getWSResultForQuery(req("q","id:2", "wt","xlsx", "fl","id,v_ss")); // test multivalued assertEquals("ID,v_ss\n2,hi; there\n", getStringFromSheet(resultSheet)); @@ -167,9 +173,7 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { resultSheet = getWSResultForQuery(req, rsp); assertEquals("ID,Foo Integer,Foo String,Foo Long,foo_b,foo_f,foo_d,foo_dt1,v_ss,v2_ss\n" + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z,,\n" + - "2,,,,,,,,hi; there,nice; output\n", - getStringFromSheet(resultSheet)); - + "2,,,,,,,,hi; there,nice; output\n", getStringFromSheet(resultSheet)); // get field values and scores - just check that the scores are there... we don't guarantee where rsp.setReturnFields( new SolrReturnFields("*,score", req) ); @@ -182,31 +186,27 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { resultSheet = getWSResultForQuery(req, rsp); assertEquals("ID,Foo Integer,Foo String,Foo Long,foo_b,foo_f,foo_d,foo_dt1\n" + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n" + - "2,,,,,,,\n", - getStringFromSheet(resultSheet)); + "2,,,,,,,\n", getStringFromSheet(resultSheet)); rsp.setReturnFields( new SolrReturnFields("id,*_d*", req) ); resultSheet = getWSResultForQuery(req, rsp); assertEquals("ID,foo_d,foo_dt1\n" + "1,-1.0E300,2000-01-02T03:04:05Z\n" + - "2,,\n", - getStringFromSheet(resultSheet)); + "2,,\n", getStringFromSheet(resultSheet)); // Test function queries rsp.setReturnFields( new SolrReturnFields("sum(1,1),id,exists(foo_s1),div(9,1),foo_f", req) ); resultSheet = getWSResultForQuery(req, rsp); assertEquals("sum(1,1),ID,exists(foo_s1),div(9,1),foo_f\n" + ",1,,,1.414\n" + - ",2,,,\n", - getStringFromSheet(resultSheet)); + ",2,,,\n", getStringFromSheet(resultSheet)); // Test transformers rsp.setReturnFields( new SolrReturnFields("mydocid:[docid],[explain]", req) ); resultSheet = getWSResultForQuery(req, rsp); assertEquals("mydocid,[explain]\n" + ",\n" + - ",\n", - getStringFromSheet(resultSheet)); + ",\n", getStringFromSheet(resultSheet)); req.close(); } From bfd0c484deec82aaa7ff804257a041ff79a8296c Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Mon, 22 Aug 2016 13:52:47 -0700 Subject: [PATCH 19/21] copy colnames map before modifying per request params --- .../apache/solr/handler/extraction/XLSXResponseWriter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java index e4da9510f7cd..c07c893bb984 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java @@ -87,8 +87,8 @@ public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp) XLSXWriter w = new XLSXWriter(new CharArrayWriter(), req, rsp); // copy these as each request may modify them within its context - LinkedHashMap reqNamesMap = colNamesMap;//.copy(); - LinkedHashMap reqWidthsMap = colWidthsMap;//.copy(); + LinkedHashMap reqNamesMap = new LinkedHashMap<>(colNamesMap); + LinkedHashMap reqWidthsMap = new LinkedHashMap<>(colWidthsMap); Iterator paramNamesIter = req.getParams().getParameterNamesIterator(); while (paramNamesIter.hasNext()) { From 5ed0453cef2977c977b0f3b38555f4e8a6021482 Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Mon, 5 Sep 2016 00:27:36 -0700 Subject: [PATCH 20/21] remove all support for xlsxwriter config in solrconfig --- .../extraction/XLSXResponseWriter.java | 23 ++---------- .../solr/collection1/conf/solrconfig.xml | 18 +--------- .../extraction/TestXLSXResponseWriter.java | 36 ++++++++----------- 3 files changed, 17 insertions(+), 60 deletions(-) diff --git a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java index c07c893bb984..c1517ff75deb 100644 --- a/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java +++ b/solr/contrib/extraction/src/java/org/apache/solr/handler/extraction/XLSXResponseWriter.java @@ -58,26 +58,8 @@ import org.apache.solr.search.ReturnFields; public class XLSXResponseWriter extends RawResponseWriter { - LinkedHashMap colNamesMap = new LinkedHashMap(); - LinkedHashMap colWidthsMap = new LinkedHashMap(); - @Override public void init(NamedList solrconfig) { - NamedList columnNames = (NamedList) solrconfig.get("colnames"); - if (columnNames != null && columnNames.size() > 0) { - for (Object nameObject: columnNames) { - Map.Entry namePair = (Map.Entry) nameObject; - this.colNamesMap.put(namePair.getKey(), namePair.getValue()); - } - } - - NamedList columnWidths = (NamedList) solrconfig.get("colwidths"); - if (columnWidths != null && columnWidths.size() > 0) { - for (Object widthObject : columnWidths) { - Map.Entry widthPair = (Map.Entry) widthObject; - this.colWidthsMap.put(widthPair.getKey(), widthPair.getValue()); - } - } } @Override @@ -86,9 +68,8 @@ public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp) // all writes before they go to it anyway XLSXWriter w = new XLSXWriter(new CharArrayWriter(), req, rsp); - // copy these as each request may modify them within its context - LinkedHashMap reqNamesMap = new LinkedHashMap<>(colNamesMap); - LinkedHashMap reqWidthsMap = new LinkedHashMap<>(colWidthsMap); + LinkedHashMap reqNamesMap = new LinkedHashMap<>(); + LinkedHashMap reqWidthsMap = new LinkedHashMap<>(); Iterator paramNamesIter = req.getParams().getParameterNamesIterator(); while (paramNamesIter.hasNext()) { diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml index c38e27ee8f92..f60e703d6290 100644 --- a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml +++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml @@ -203,23 +203,7 @@ - - - 3 - 10 - 10 - 50 - - - ID - Foo String - Foo Integer - Foo Long - - - - + diff --git a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java index 7a7ee1a6871e..fd4e63d04a53 100644 --- a/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java +++ b/solr/contrib/extraction/src/test/org/apache/solr/handler/extraction/TestXLSXResponseWriter.java @@ -85,45 +85,37 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { // check Content-Type assertEquals("application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", writerXlsx.getContentType(r, rsp)); - // check it read solrconfig - assertEquals("Foo Integer", writerXlsx.colNamesMap.get("foo_i")); - assertTrue("1 specific width read in from solrconfig", writerXlsx.colWidthsMap.get("foo_l") == 50); - - assertTrue("All names read in from solrconfig", writerXlsx.colNamesMap.size() == 4); - assertTrue("All widths read in from solrconfig", writerXlsx.colWidthsMap.size() == 4); - // test our basic types,and that fields come back in the requested order XSSFSheet resultSheet = getWSResultForQuery(req("q","id:1", "wt","xlsx", "fl","id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt1")); - assertEquals("ID,Foo String,Foo Integer,Foo Long,foo_b,foo_f,foo_d,foo_dt1\n1,hi,-1,12345678987654321,F,1.414,-1.0E300,2000-01-02T03:04:05Z\n" + assertEquals("id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt1\n1,hi,-1,12345678987654321,F,1.414,-1.0E300,2000-01-02T03:04:05Z\n" , getStringFromSheet(resultSheet)); resultSheet = getWSResultForQuery(req("q","id:1^0", "wt","xlsx", "fl","id,score,foo_s")); // test retrieving score - assertEquals("ID,score,Foo String\n1,0.0,hi\n", getStringFromSheet(resultSheet)); - // test column width (result is in 256ths of a character for some reason) - assertEquals(3*256, resultSheet.getColumnWidth(0)); + assertEquals("id,score,foo_s\n1,0.0,hi\n", getStringFromSheet(resultSheet)); resultSheet = getWSResultForQuery(req("q","id:1^0", "wt","xlsx", "colname.id", "I.D.", "colwidth.id", "10", "fl","id,score,foo_s")); // test override colname/width - assertEquals("I.D.,score,Foo String\n1,0.0,hi\n", getStringFromSheet(resultSheet)); + assertEquals("I.D.,score,foo_s\n1,0.0,hi\n", getStringFromSheet(resultSheet)); + // test colwidth (value returned is in 256ths of a character as per excel standard) assertEquals(10*256, resultSheet.getColumnWidth(0)); resultSheet = getWSResultForQuery(req("q","id:2", "wt","xlsx", "fl","id,v_ss")); // test multivalued - assertEquals("ID,v_ss\n2,hi; there\n", getStringFromSheet(resultSheet)); + assertEquals("id,v_ss\n2,hi; there\n", getStringFromSheet(resultSheet)); // test retrieving fields from index resultSheet = getWSResultForQuery(req("q","*:*", "wt","xslx", "fl","*,score")); String result = getStringFromSheet(resultSheet); - for (String field : "ID,Foo String,Foo Integer,Foo Long,foo_b,foo_f,foo_d,foo_dt1,v_ss,v2_ss,score".split(",")) { + for (String field : "id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt1,v_ss,v2_ss,score".split(",")) { assertTrue(result.indexOf(field) >= 0); } // test null values resultSheet = getWSResultForQuery(req("q","id:2", "wt","xlsx", "fl","id,foo_s,v_ss")); - assertEquals("ID,Foo String,v_ss\n2,,hi; there\n", getStringFromSheet(resultSheet)); + assertEquals("id,foo_s,v_ss\n2,,hi; there\n", getStringFromSheet(resultSheet)); // now test SolrDocumentList SolrDocument d = new SolrDocument(); @@ -159,19 +151,19 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { rsp.setReturnFields( new SolrReturnFields("id,foo_s", req) ); resultSheet = getWSResultForQuery(req, rsp); - assertEquals("ID,Foo String\n1,hi\n2,\n", getStringFromSheet(resultSheet)); + assertEquals("id,foo_s\n1,hi\n2,\n", getStringFromSheet(resultSheet)); // try scores rsp.setReturnFields( new SolrReturnFields("id,score,foo_s", req) ); resultSheet = getWSResultForQuery(req, rsp); - assertEquals("ID,score,Foo String\n1,2.718,hi\n2,89.83,\n", getStringFromSheet(resultSheet)); + assertEquals("id,score,foo_s\n1,2.718,hi\n2,89.83,\n", getStringFromSheet(resultSheet)); // get field values from docs... should be ordered and not include score unless requested rsp.setReturnFields( new SolrReturnFields("*", req) ); resultSheet = getWSResultForQuery(req, rsp); - assertEquals("ID,Foo Integer,Foo String,Foo Long,foo_b,foo_f,foo_d,foo_dt1,v_ss,v2_ss\n" + + assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt1,v_ss,v2_ss\n" + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z,,\n" + "2,,,,,,,,hi; there,nice; output\n", getStringFromSheet(resultSheet)); @@ -184,20 +176,20 @@ public void testStructuredDataViaBaseWriters() throws IOException, Exception { // Test field globs rsp.setReturnFields( new SolrReturnFields("id,foo*", req) ); resultSheet = getWSResultForQuery(req, rsp); - assertEquals("ID,Foo Integer,Foo String,Foo Long,foo_b,foo_f,foo_d,foo_dt1\n" + + assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt1\n" + "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n" + "2,,,,,,,\n", getStringFromSheet(resultSheet)); rsp.setReturnFields( new SolrReturnFields("id,*_d*", req) ); resultSheet = getWSResultForQuery(req, rsp); - assertEquals("ID,foo_d,foo_dt1\n" + + assertEquals("id,foo_d,foo_dt1\n" + "1,-1.0E300,2000-01-02T03:04:05Z\n" + "2,,\n", getStringFromSheet(resultSheet)); // Test function queries rsp.setReturnFields( new SolrReturnFields("sum(1,1),id,exists(foo_s1),div(9,1),foo_f", req) ); resultSheet = getWSResultForQuery(req, rsp); - assertEquals("sum(1,1),ID,exists(foo_s1),div(9,1),foo_f\n" + + assertEquals("sum(1,1),id,exists(foo_s1),div(9,1),foo_f\n" + ",1,,,1.414\n" + ",2,,,\n", getStringFromSheet(resultSheet)); @@ -217,7 +209,7 @@ public void testPseudoFields() throws Exception { // Use Pseudo Field SolrQueryRequest req = req("q","id:1", "wt","xlsx", "fl","XXX:id,foo_s"); XSSFSheet resultSheet = getWSResultForQuery(req); - assertEquals("XXX,Foo String\n1,hi\n", getStringFromSheet(resultSheet)); + assertEquals("XXX,foo_s\n1,hi\n", getStringFromSheet(resultSheet)); String txt = getStringFromSheet(getWSResultForQuery(req("q","id:1", "wt","xlsx", "fl","XXX:id,YYY:[docid],FOO:foo_s"))); String[] lines = txt.split("\n"); From be259eb6099c95da926e4d4adf4acb71bb1ac64c Mon Sep 17 00:00:00 2001 From: Tony Moriarty Date: Mon, 5 Sep 2016 17:31:53 +1000 Subject: [PATCH 21/21] fix indentation --- .../test-files/extraction/solr/collection1/conf/solrconfig.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml index f60e703d6290..ed33c6b5cd86 100644 --- a/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml +++ b/solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml @@ -203,7 +203,7 @@ - +