From d0cd13e763e77bfb39c8e1a657e968436122a6a6 Mon Sep 17 00:00:00 2001 From: Alessandro Benedetti Date: Tue, 1 May 2018 13:49:31 +0100 Subject: [PATCH 1/4] [SOLR-12299] More Like This Query Params refactor + tests --- .../KNearestNeighborClassifier.java | 30 +- .../KNearestNeighborDocumentClassifier.java | 27 +- .../lucene/queries/mlt/MoreLikeThis.java | 478 +-------------- .../queries/mlt/MoreLikeThisParameters.java | 548 ++++++++++++++++++ .../lucene/queries/mlt/MoreLikeThisQuery.java | 13 +- .../lucene/queries/mlt/TestMoreLikeThis.java | 78 +-- .../solr/handler/MoreLikeThisHandler.java | 79 +-- .../component/MoreLikeThisComponent.java | 5 +- .../solr/search/mlt/CloudMLTQParser.java | 50 +- .../solr/search/mlt/SimpleMLTQParser.java | 27 +- .../solr/handler/MoreLikeThisHandlerTest.java | 1 - 11 files changed, 707 insertions(+), 629 deletions(-) create mode 100644 lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisParameters.java diff --git a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java index 1bc53b0202cf..b652c682ff82 100644 --- a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java +++ b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java @@ -31,6 +31,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; import org.apache.lucene.queries.mlt.MoreLikeThis; +import org.apache.lucene.queries.mlt.MoreLikeThisParameters; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; @@ -90,8 +91,8 @@ public class KNearestNeighborClassifier implements Classifier { * @param query a {@link Query} to eventually filter the docs used for training the classifier, or {@code null} * if all the indexed docs should be used * @param k the no. of docs to select in the MLT results to find the nearest neighbor - * @param minDocsFreq {@link MoreLikeThis#minDocFreq} parameter - * @param minTermFreq {@link MoreLikeThis#minTermFreq} parameter + * @param minDocsFreq {@link org.apache.lucene.queries.mlt.MoreLikeThisParameters#minDocFreq} parameter + * @param minTermFreq {@link org.apache.lucene.queries.mlt.MoreLikeThisParameters#minTermFreq} parameter * @param classFieldName the name of the field used as the output for the classifier * @param textFieldNames the name of the fields used as the inputs for the classifier, they can contain boosting indication e.g. title^10 */ @@ -100,8 +101,10 @@ public KNearestNeighborClassifier(IndexReader indexReader, Similarity similarity this.textFieldNames = textFieldNames; this.classFieldName = classFieldName; this.mlt = new MoreLikeThis(indexReader); - this.mlt.setAnalyzer(analyzer); - this.mlt.setFieldNames(textFieldNames); + MoreLikeThisParameters mltParameters = mlt.getParameters(); + + mltParameters.setAnalyzer(analyzer); + mltParameters.setFieldNames(textFieldNames); this.indexSearcher = new IndexSearcher(indexReader); if (similarity != null) { this.indexSearcher.setSimilarity(similarity); @@ -109,10 +112,10 @@ public KNearestNeighborClassifier(IndexReader indexReader, Similarity similarity this.indexSearcher.setSimilarity(new BM25Similarity()); } if (minDocsFreq > 0) { - mlt.setMinDocFreq(minDocsFreq); + mltParameters.setMinDocFreq(minDocsFreq); } if (minTermFreq > 0) { - mlt.setMinTermFreq(minTermFreq); + mltParameters.setMinTermFreq(minTermFreq); } this.query = query; this.k = k; @@ -158,19 +161,12 @@ public List> getClasses(String text, int max) thr private TopDocs knnSearch(String text) throws IOException { BooleanQuery.Builder mltQuery = new BooleanQuery.Builder(); + MoreLikeThisParameters mltParameters = mlt.getParameters(); + MoreLikeThisParameters.BoostProperties boostConfiguration = mltParameters.getBoostConfiguration(); + boostConfiguration.setBoost(true); //terms boost actually helps in MLT queries for (String fieldName : textFieldNames) { - String boost = null; - mlt.setBoost(true); //terms boost actually helps in MLT queries - if (fieldName.contains("^")) { - String[] field2boost = fieldName.split("\\^"); - fieldName = field2boost[0]; - boost = field2boost[1]; - } - if (boost != null) { - mlt.setBoostFactor(Float.parseFloat(boost));//if we have a field boost, we add it - } + boostConfiguration.addFieldWithBoost(fieldName); mltQuery.add(new BooleanClause(mlt.like(fieldName, new StringReader(text)), BooleanClause.Occur.SHOULD)); - mlt.setBoostFactor(1);// restore neutral boost for next field } Query classFieldQuery = new WildcardQuery(new Term(classFieldName, "*")); mltQuery.add(new BooleanClause(classFieldQuery, BooleanClause.Occur.MUST)); diff --git a/lucene/classification/src/java/org/apache/lucene/classification/document/KNearestNeighborDocumentClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/document/KNearestNeighborDocumentClassifier.java index 39684ee25e74..d0dc6bf05de8 100644 --- a/lucene/classification/src/java/org/apache/lucene/classification/document/KNearestNeighborDocumentClassifier.java +++ b/lucene/classification/src/java/org/apache/lucene/classification/document/KNearestNeighborDocumentClassifier.java @@ -29,6 +29,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.mlt.MoreLikeThisParameters; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.IndexSearcher; @@ -60,8 +61,8 @@ public class KNearestNeighborDocumentClassifier extends KNearestNeighborClassifi * @param query a {@link org.apache.lucene.search.Query} to eventually filter the docs used for training the classifier, or {@code null} * if all the indexed docs should be used * @param k the no. of docs to select in the MLT results to find the nearest neighbor - * @param minDocsFreq {@link org.apache.lucene.queries.mlt.MoreLikeThis#minDocFreq} parameter - * @param minTermFreq {@link org.apache.lucene.queries.mlt.MoreLikeThis#minTermFreq} parameter + * @param minDocsFreq {@link org.apache.lucene.queries.mlt.MoreLikeThisParameters#minDocFreq} parameter + * @param minTermFreq {@link org.apache.lucene.queries.mlt.MoreLikeThisParameters#minTermFreq} parameter * @param classFieldName the name of the field used as the output for the classifier * @param field2analyzer map with key a field name and the related {org.apache.lucene.analysis.Analyzer} * @param textFieldNames the name of the fields used as the inputs for the classifier, they can contain boosting indication e.g. title^10 @@ -103,24 +104,18 @@ public List> getClasses(Document document, int ma */ private TopDocs knnSearch(Document document) throws IOException { BooleanQuery.Builder mltQuery = new BooleanQuery.Builder(); - - for (String fieldName : textFieldNames) { - String boost = null; - if (fieldName.contains("^")) { - String[] field2boost = fieldName.split("\\^"); - fieldName = field2boost[0]; - boost = field2boost[1]; - } + MoreLikeThisParameters mltParameters = mlt.getParameters(); + MoreLikeThisParameters.BoostProperties boostConfiguration = mltParameters.getBoostConfiguration(); + boostConfiguration.setBoost(true); + for (String fieldNameWithBoost : textFieldNames) { + boostConfiguration.addFieldWithBoost(fieldNameWithBoost); + } + for (String fieldName : mltParameters.getFieldNames()) { String[] fieldValues = document.getValues(fieldName); - mlt.setBoost(true); // we want always to use the boost coming from TF * IDF of the term - if (boost != null) { - mlt.setBoostFactor(Float.parseFloat(boost)); // this is an additional multiplicative boost coming from the field boost - } - mlt.setAnalyzer(field2analyzer.get(fieldName)); + mltParameters.setAnalyzer(field2analyzer.get(fieldName)); for (String fieldContent : fieldValues) { mltQuery.add(new BooleanClause(mlt.like(fieldName, new StringReader(fieldContent)), BooleanClause.Occur.SHOULD)); } - mlt.setBoostFactor(1);// restore neutral boost for next field } Query classFieldQuery = new WildcardQuery(new Term(classFieldName, "*")); mltQuery.add(new BooleanClause(classFieldQuery, BooleanClause.Occur.MUST)); diff --git a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java index 9afa68708a6f..0a024526ee51 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java @@ -23,7 +23,6 @@ import java.util.Collection; import java.util.HashMap; import java.util.Map; -import java.util.Set; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; @@ -32,7 +31,6 @@ import org.apache.lucene.index.Fields; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; @@ -109,23 +107,7 @@ *
*

More Advanced Usage

*

- * You may want to use {@link #setFieldNames setFieldNames(...)} so you can examine - * multiple fields (e.g. body and title) for similarity. - *

- * Depending on the size of your index and the size and makeup of your documents you - * may want to call the other set methods to control how the similarity queries are - * generated: - *

    - *
  • {@link #setMinTermFreq setMinTermFreq(...)} - *
  • {@link #setMinDocFreq setMinDocFreq(...)} - *
  • {@link #setMaxDocFreq setMaxDocFreq(...)} - *
  • {@link #setMaxDocFreqPct setMaxDocFreqPct(...)} - *
  • {@link #setMinWordLen setMinWordLen(...)} - *
  • {@link #setMaxWordLen setMaxWordLen(...)} - *
  • {@link #setMaxQueryTerms setMaxQueryTerms(...)} - *
  • {@link #setMaxNumTokensParsed setMaxNumTokensParsed(...)} - *
  • {@link #setStopWords setStopWord(...)} - *
+ * You may want to see the parameters class *
*
* 
@@ -139,141 +121,10 @@
  *
*/ public final class MoreLikeThis { - - /** - * Default maximum number of tokens to parse in each example doc field that is not stored with TermVector support. - * - * @see #getMaxNumTokensParsed - */ - public static final int DEFAULT_MAX_NUM_TOKENS_PARSED = 5000; - - /** - * Ignore terms with less than this frequency in the source doc. - * - * @see #getMinTermFreq - * @see #setMinTermFreq - */ - public static final int DEFAULT_MIN_TERM_FREQ = 2; - - /** - * Ignore words which do not occur in at least this many docs. - * - * @see #getMinDocFreq - * @see #setMinDocFreq - */ - public static final int DEFAULT_MIN_DOC_FREQ = 5; - - /** - * Ignore words which occur in more than this many docs. - * - * @see #getMaxDocFreq - * @see #setMaxDocFreq - * @see #setMaxDocFreqPct - */ - public static final int DEFAULT_MAX_DOC_FREQ = Integer.MAX_VALUE; - - /** - * Boost terms in query based on score. - * - * @see #isBoost - * @see #setBoost - */ - public static final boolean DEFAULT_BOOST = false; - - /** - * Default field names. Null is used to specify that the field names should be looked - * up at runtime from the provided reader. - */ - public static final String[] DEFAULT_FIELD_NAMES = new String[]{"contents"}; - - /** - * Ignore words less than this length or if 0 then this has no effect. - * - * @see #getMinWordLen - * @see #setMinWordLen - */ - public static final int DEFAULT_MIN_WORD_LENGTH = 0; - - /** - * Ignore words greater than this length or if 0 then this has no effect. - * - * @see #getMaxWordLen - * @see #setMaxWordLen - */ - public static final int DEFAULT_MAX_WORD_LENGTH = 0; - - /** - * Default set of stopwords. - * If null means to allow stop words. - * - * @see #setStopWords - * @see #getStopWords - */ - public static final Set DEFAULT_STOP_WORDS = null; - - /** - * Current set of stop words. - */ - private Set stopWords = DEFAULT_STOP_WORDS; - - /** - * Return a Query with no more than this many terms. - * - * @see BooleanQuery#getMaxClauseCount - * @see #getMaxQueryTerms - * @see #setMaxQueryTerms - */ - public static final int DEFAULT_MAX_QUERY_TERMS = 25; - - /** - * Analyzer that will be used to parse the doc. - */ - private Analyzer analyzer = null; - - /** - * Ignore words less frequent that this. - */ - private int minTermFreq = DEFAULT_MIN_TERM_FREQ; - - /** - * Ignore words which do not occur in at least this many docs. - */ - private int minDocFreq = DEFAULT_MIN_DOC_FREQ; - /** - * Ignore words which occur in more than this many docs. + * All tunable parameters that regulates the query building */ - private int maxDocFreq = DEFAULT_MAX_DOC_FREQ; - - /** - * Should we apply a boost to the Query based on the scores? - */ - private boolean boost = DEFAULT_BOOST; - - /** - * Field name we'll analyze. - */ - private String[] fieldNames = DEFAULT_FIELD_NAMES; - - /** - * The maximum number of tokens to parse in each example doc field that is not stored with TermVector support - */ - private int maxNumTokensParsed = DEFAULT_MAX_NUM_TOKENS_PARSED; - - /** - * Ignore words if less than this len. - */ - private int minWordLen = DEFAULT_MIN_WORD_LENGTH; - - /** - * Ignore words if greater than this len. - */ - private int maxWordLen = DEFAULT_MAX_WORD_LENGTH; - - /** - * Don't return a query longer than this. - */ - private int maxQueryTerms = DEFAULT_MAX_QUERY_TERMS; + private MoreLikeThisParameters parameters; /** * For idf() calculations. @@ -285,30 +136,6 @@ public final class MoreLikeThis { */ private final IndexReader ir; - /** - * Boost factor to use when boosting the terms - */ - private float boostFactor = 1; - - /** - * Returns the boost factor used when boosting terms - * - * @return the boost factor used when boosting terms - * @see #setBoostFactor(float) - */ - public float getBoostFactor() { - return boostFactor; - } - - /** - * Sets the boost factor to use when boosting terms - * - * @see #getBoostFactor() - */ - public void setBoostFactor(float boostFactor) { - this.boostFactor = boostFactor; - } - /** * Constructor requiring an IndexReader. */ @@ -317,10 +144,14 @@ public MoreLikeThis(IndexReader ir) { } public MoreLikeThis(IndexReader ir, TFIDFSimilarity sim) { + this.parameters = new MoreLikeThisParameters(); this.ir = ir; this.similarity = sim; } + public MoreLikeThisParameters getParameters() { + return parameters; + } public TFIDFSimilarity getSimilarity() { return similarity; @@ -330,242 +161,6 @@ public void setSimilarity(TFIDFSimilarity similarity) { this.similarity = similarity; } - /** - * Returns an analyzer that will be used to parse source doc with. The default analyzer - * is not set. - * - * @return the analyzer that will be used to parse source doc with. - */ - public Analyzer getAnalyzer() { - return analyzer; - } - - /** - * Sets the analyzer to use. An analyzer is not required for generating a query with the - * {@link #like(int)} method, all other 'like' methods require an analyzer. - * - * @param analyzer the analyzer to use to tokenize text. - */ - public void setAnalyzer(Analyzer analyzer) { - this.analyzer = analyzer; - } - - /** - * Returns the frequency below which terms will be ignored in the source doc. The default - * frequency is the {@link #DEFAULT_MIN_TERM_FREQ}. - * - * @return the frequency below which terms will be ignored in the source doc. - */ - public int getMinTermFreq() { - return minTermFreq; - } - - /** - * Sets the frequency below which terms will be ignored in the source doc. - * - * @param minTermFreq the frequency below which terms will be ignored in the source doc. - */ - public void setMinTermFreq(int minTermFreq) { - this.minTermFreq = minTermFreq; - } - - /** - * Returns the frequency at which words will be ignored which do not occur in at least this - * many docs. The default frequency is {@link #DEFAULT_MIN_DOC_FREQ}. - * - * @return the frequency at which words will be ignored which do not occur in at least this - * many docs. - */ - public int getMinDocFreq() { - return minDocFreq; - } - - /** - * Sets the frequency at which words will be ignored which do not occur in at least this - * many docs. - * - * @param minDocFreq the frequency at which words will be ignored which do not occur in at - * least this many docs. - */ - public void setMinDocFreq(int minDocFreq) { - this.minDocFreq = minDocFreq; - } - - /** - * Returns the maximum frequency in which words may still appear. - * Words that appear in more than this many docs will be ignored. The default frequency is - * {@link #DEFAULT_MAX_DOC_FREQ}. - * - * @return get the maximum frequency at which words are still allowed, - * words which occur in more docs than this are ignored. - */ - public int getMaxDocFreq() { - return maxDocFreq; - } - - /** - * Set the maximum frequency in which words may still appear. Words that appear - * in more than this many docs will be ignored. - * - * @param maxFreq the maximum count of documents that a term may appear - * in to be still considered relevant - */ - public void setMaxDocFreq(int maxFreq) { - this.maxDocFreq = maxFreq; - } - - /** - * Set the maximum percentage in which words may still appear. Words that appear - * in more than this many percent of all docs will be ignored. - * - * @param maxPercentage the maximum percentage of documents (0-100) that a term may appear - * in to be still considered relevant - */ - public void setMaxDocFreqPct(int maxPercentage) { - this.maxDocFreq = maxPercentage * ir.numDocs() / 100; - } - - - /** - * Returns whether to boost terms in query based on "score" or not. The default is - * {@link #DEFAULT_BOOST}. - * - * @return whether to boost terms in query based on "score" or not. - * @see #setBoost - */ - public boolean isBoost() { - return boost; - } - - /** - * Sets whether to boost terms in query based on "score" or not. - * - * @param boost true to boost terms in query based on "score", false otherwise. - * @see #isBoost - */ - public void setBoost(boolean boost) { - this.boost = boost; - } - - /** - * Returns the field names that will be used when generating the 'More Like This' query. - * The default field names that will be used is {@link #DEFAULT_FIELD_NAMES}. - * - * @return the field names that will be used when generating the 'More Like This' query. - */ - public String[] getFieldNames() { - return fieldNames; - } - - /** - * Sets the field names that will be used when generating the 'More Like This' query. - * Set this to null for the field names to be determined at runtime from the IndexReader - * provided in the constructor. - * - * @param fieldNames the field names that will be used when generating the 'More Like This' - * query. - */ - public void setFieldNames(String[] fieldNames) { - this.fieldNames = fieldNames; - } - - /** - * Returns the minimum word length below which words will be ignored. Set this to 0 for no - * minimum word length. The default is {@link #DEFAULT_MIN_WORD_LENGTH}. - * - * @return the minimum word length below which words will be ignored. - */ - public int getMinWordLen() { - return minWordLen; - } - - /** - * Sets the minimum word length below which words will be ignored. - * - * @param minWordLen the minimum word length below which words will be ignored. - */ - public void setMinWordLen(int minWordLen) { - this.minWordLen = minWordLen; - } - - /** - * Returns the maximum word length above which words will be ignored. Set this to 0 for no - * maximum word length. The default is {@link #DEFAULT_MAX_WORD_LENGTH}. - * - * @return the maximum word length above which words will be ignored. - */ - public int getMaxWordLen() { - return maxWordLen; - } - - /** - * Sets the maximum word length above which words will be ignored. - * - * @param maxWordLen the maximum word length above which words will be ignored. - */ - public void setMaxWordLen(int maxWordLen) { - this.maxWordLen = maxWordLen; - } - - /** - * Set the set of stopwords. - * Any word in this set is considered "uninteresting" and ignored. - * Even if your Analyzer allows stopwords, you might want to tell the MoreLikeThis code to ignore them, as - * for the purposes of document similarity it seems reasonable to assume that "a stop word is never interesting". - * - * @param stopWords set of stopwords, if null it means to allow stop words - * @see #getStopWords - */ - public void setStopWords(Set stopWords) { - this.stopWords = stopWords; - } - - /** - * Get the current stop words being used. - * - * @see #setStopWords - */ - public Set getStopWords() { - return stopWords; - } - - - /** - * Returns the maximum number of query terms that will be included in any generated query. - * The default is {@link #DEFAULT_MAX_QUERY_TERMS}. - * - * @return the maximum number of query terms that will be included in any generated query. - */ - public int getMaxQueryTerms() { - return maxQueryTerms; - } - - /** - * Sets the maximum number of query terms that will be included in any generated query. - * - * @param maxQueryTerms the maximum number of query terms that will be included in any - * generated query. - */ - public void setMaxQueryTerms(int maxQueryTerms) { - this.maxQueryTerms = maxQueryTerms; - } - - /** - * @return The maximum number of tokens to parse in each example doc field that is not stored with TermVector support - * @see #DEFAULT_MAX_NUM_TOKENS_PARSED - */ - public int getMaxNumTokensParsed() { - return maxNumTokensParsed; - } - - /** - * @param i The maximum number of tokens to parse in each example doc field that is not stored with TermVector support - */ - public void setMaxNumTokensParsed(int i) { - maxNumTokensParsed = i; - } - - /** * Return a query that will return docs like the passed lucene document ID. * @@ -573,12 +168,6 @@ public void setMaxNumTokensParsed(int i) { * @return a query that will return docs like the passed lucene document ID. */ public Query like(int docNum) throws IOException { - if (fieldNames == null) { - // gather list of valid fields from lucene - Collection fields = MultiFields.getIndexedFields(ir); - fieldNames = fields.toArray(new String[fields.size()]); - } - return createQuery(retrieveTerms(docNum)); } @@ -588,11 +177,6 @@ public Query like(int docNum) throws IOException { * @return More Like This query for the passed document. */ public Query like(Map> filteredDocument) throws IOException { - if (fieldNames == null) { - // gather list of valid fields from lucene - Collection fields = MultiFields.getIndexedFields(ir); - fieldNames = fields.toArray(new String[fields.size()]); - } return createQuery(retrieveTerms(filteredDocument)); } @@ -615,18 +199,20 @@ public Query like(String fieldName, Reader... readers) throws IOException { */ private Query createQuery(PriorityQueue q) { BooleanQuery.Builder query = new BooleanQuery.Builder(); + MoreLikeThisParameters.BoostProperties boostConfiguration = parameters.getBoostConfiguration(); ScoreTerm scoreTerm; float bestScore = -1; while ((scoreTerm = q.pop()) != null) { Query tq = new TermQuery(new Term(scoreTerm.topField, scoreTerm.word)); - if (boost) { + if (boostConfiguration.isBoostByTermScore()) { if (bestScore == -1) { bestScore = (scoreTerm.score); } float myScore = (scoreTerm.score); - tq = new BoostQuery(tq, boostFactor * myScore / bestScore); + float fieldBoost = boostConfiguration.getFieldBoost(scoreTerm.topField); + tq = new BoostQuery(tq, fieldBoost * myScore / bestScore); } try { @@ -647,8 +233,13 @@ private Query createQuery(PriorityQueue q) { private PriorityQueue createQueue(Map> perFieldTermFrequencies) throws IOException { // have collected all words in doc and their freqs int numDocs = ir.numDocs(); + int maxQueryTerms = parameters.getMaxQueryTerms(); + int minTermFreq = parameters.getMinTermFreq(); + int minDocFreq = parameters.getMinDocFreq(); + int maxDocFreq = parameters.getMaxDocFreq(); final int limit = Math.min(maxQueryTerms, this.getTermsCount(perFieldTermFrequencies)); FreqQ queue = new FreqQ(limit); // will order words by score + for (Map.Entry> entry : perFieldTermFrequencies.entrySet()) { Map perWordTermFrequencies = entry.getValue(); String fieldName = entry.getKey(); @@ -701,27 +292,6 @@ private int getTermsCount(Map> perFieldTermFrequencies) return totalTermsCount; } - /** - * Describe the parameters that control how the "more like this" query is formed. - */ - public String describeParams() { - StringBuilder sb = new StringBuilder(); - sb.append("\t").append("maxQueryTerms : ").append(maxQueryTerms).append("\n"); - sb.append("\t").append("minWordLen : ").append(minWordLen).append("\n"); - sb.append("\t").append("maxWordLen : ").append(maxWordLen).append("\n"); - sb.append("\t").append("fieldNames : "); - String delim = ""; - for (String fieldName : fieldNames) { - sb.append(delim).append(fieldName); - delim = ", "; - } - sb.append("\n"); - sb.append("\t").append("boost : ").append(boost).append("\n"); - sb.append("\t").append("minTermFreq : ").append(minTermFreq).append("\n"); - sb.append("\t").append("minDocFreq : ").append(minDocFreq).append("\n"); - return sb.toString(); - } - /** * Find words for a more-like-this query former. * @@ -729,7 +299,7 @@ public String describeParams() { */ private PriorityQueue retrieveTerms(int docNum) throws IOException { Map> field2termFreqMap = new HashMap<>(); - for (String fieldName : fieldNames) { + for (String fieldName : parameters.getFieldNamesOrInit(ir)) { final Fields vectors = ir.getTermVectors(docNum); final Terms vector; if (vectors != null) { @@ -760,7 +330,7 @@ private PriorityQueue retrieveTerms(int docNum) throws IOException { private PriorityQueue retrieveTerms(Map> field2fieldValues) throws IOException { Map> field2termFreqMap = new HashMap<>(); - for (String fieldName : fieldNames) { + for (String fieldName : parameters.getFieldNamesOrInit(ir)) { for (String field : field2fieldValues.keySet()) { Collection fieldValues = field2fieldValues.get(field); if(fieldValues == null) @@ -815,6 +385,7 @@ private void addTermFrequencies(Map> field2termFreqMap, */ private void addTermFrequencies(Reader r, Map> perFieldTermFrequencies, String fieldName) throws IOException { + Analyzer analyzer = parameters.getAnalyzer(); if (analyzer == null) { throw new UnsupportedOperationException("To use MoreLikeThis without " + "term vectors, you must provide an Analyzer"); @@ -828,7 +399,7 @@ private void addTermFrequencies(Reader r, Map> perField while (ts.incrementToken()) { String word = termAtt.toString(); tokenCount++; - if (tokenCount > maxNumTokensParsed) { + if (tokenCount > parameters.getMaxNumTokensParsed()) { break; } if (isNoiseWord(word)) { @@ -856,13 +427,13 @@ private void addTermFrequencies(Reader r, Map> perField */ private boolean isNoiseWord(String term) { int len = term.length(); - if (minWordLen > 0 && len < minWordLen) { + if (parameters.getMinWordLen() > 0 && len < parameters.getMinWordLen()) { return true; } - if (maxWordLen > 0 && len > maxWordLen) { + if (parameters.getMaxWordLen() > 0 && len > parameters.getMaxWordLen()) { return true; } - return stopWords != null && stopWords.contains(term); + return parameters.getStopWords() != null && parameters.getStopWords().contains(term); } @@ -898,6 +469,7 @@ private PriorityQueue retrieveTerms(Reader r, String fieldName) throw * @see #retrieveInterestingTerms(java.io.Reader, String) */ public String[] retrieveInterestingTerms(int docNum) throws IOException { + int maxQueryTerms = parameters.getMaxQueryTerms(); ArrayList al = new ArrayList<>(maxQueryTerms); PriorityQueue pq = retrieveTerms(docNum); ScoreTerm scoreTerm; @@ -918,9 +490,9 @@ public String[] retrieveInterestingTerms(int docNum) throws IOException { * @param fieldName field passed to analyzer to use when analyzing the content * @return the most interesting words in the document * @see #retrieveTerms(java.io.Reader, String) - * @see #setMaxQueryTerms */ public String[] retrieveInterestingTerms(Reader r, String fieldName) throws IOException { + int maxQueryTerms = parameters.getMaxQueryTerms(); ArrayList al = new ArrayList<>(maxQueryTerms); PriorityQueue pq = retrieveTerms(r, fieldName); ScoreTerm scoreTerm; diff --git a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisParameters.java b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisParameters.java new file mode 100644 index 000000000000..34e19ed47972 --- /dev/null +++ b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisParameters.java @@ -0,0 +1,548 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.queries.mlt; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.search.BooleanQuery; + +/** + * This class models all the parameters that affect how to generate a More Like This Query: + *

+ *
+ *

More Advanced Usage

+ *

+ * You may want to use {@link #setFieldNames setFieldNames(...)} so you can examine + * multiple fields (e.g. body and title) for similarity. + *

+ * Depending on the size of your index and the size and makeup of your documents you + * may want to call the other set methods to control how the similarity queries are + * generated: + *

    + *
  • {@link #setMinTermFreq setMinTermFreq(...)} + *
  • {@link #setMinDocFreq setMinDocFreq(...)} + *
  • {@link #setMaxDocFreq setMaxDocFreq(...)} + *
  • {@link #setMaxDocFreqPct setMaxDocFreqPct(...)} + *
  • {@link #setMinWordLen setMinWordLen(...)} + *
  • {@link #setMaxWordLen setMaxWordLen(...)} + *
  • {@link #setMaxQueryTerms setMaxQueryTerms(...)} + *
  • {@link #setMaxNumTokensParsed setMaxNumTokensParsed(...)} + *
  • {@link #setStopWords setStopWord(...)} + *
+ *
+ *
+ */ +public final class MoreLikeThisParameters { + /** + * Default maximum number of tokens to parse in each example doc field that is not stored with TermVector support. + * + * @see #getMaxNumTokensParsed + */ + public static final int DEFAULT_MAX_NUM_TOKENS_PARSED = 5000; + + /** + * Ignore terms with less than this frequency in the source doc. + * + * @see #getMinTermFreq + * @see #setMinTermFreq + */ + public static final int DEFAULT_MIN_TERM_FREQ = 2; + + /** + * Ignore words which do not occur in at least this many docs. + * + * @see #getMinDocFreq + * @see #setMinDocFreq + */ + public static final int DEFAULT_MIN_DOC_FREQ = 5; + + /** + * Ignore words which occur in more than this many docs. + * + * @see #getMaxDocFreq + * @see #setMaxDocFreq + * @see #setMaxDocFreqPct + */ + public static final int DEFAULT_MAX_DOC_FREQ = Integer.MAX_VALUE; + + /** + * Default field names. Null is used to specify that the field names should be looked + * up at runtime from the provided reader. + */ + public static final String[] DEFAULT_FIELD_NAMES = new String[]{"contents"}; + + /** + * Ignore words less than this length or if 0 then this has no effect. + * + * @see #getMinWordLen + * @see #setMinWordLen + */ + public static final int DEFAULT_MIN_WORD_LENGTH = 0; + + /** + * Ignore words greater than this length or if 0 then this has no effect. + * + * @see #getMaxWordLen + * @see #setMaxWordLen + */ + public static final int DEFAULT_MAX_WORD_LENGTH = 0; + + /** + * Default set of stopwords. + * If null means to allow stop words. + * + * @see #setStopWords + * @see #getStopWords + */ + public static final Set DEFAULT_STOP_WORDS = null; + + /** + * Return a Query with no more than this many terms. + * + * @see BooleanQuery#getMaxClauseCount + * @see #getMaxQueryTerms + * @see #setMaxQueryTerms + */ + public static final int DEFAULT_MAX_QUERY_TERMS = 25; + + /** + * Analyzer that will be used to parse the doc. + * This analyzer will be used for all the fields in the document. + */ + private Analyzer analyzer = null; + + /** + * Advanced : + * Pass a specific Analyzer per field + */ + private Map fieldToAnalyzer; + + /** + * Ignore words less frequent that this. + */ + private int minTermFreq = DEFAULT_MIN_TERM_FREQ; + + /** + * Ignore words which do not occur in at least this many docs. + */ + private int minDocFreq = DEFAULT_MIN_DOC_FREQ; + + /** + * Ignore words which occur in more than this many docs. + */ + private int maxDocFreq = DEFAULT_MAX_DOC_FREQ; + + /** + * Field name we'll analyze. + */ + private String[] fieldNames = DEFAULT_FIELD_NAMES; + + /** + * The maximum number of tokens to parse in each example doc field that is not stored with TermVector support + */ + private int maxNumTokensParsed = DEFAULT_MAX_NUM_TOKENS_PARSED; + + /** + * Ignore words if less than this len. + */ + private int minWordLen = DEFAULT_MIN_WORD_LENGTH; + + /** + * Ignore words if greater than this len. + */ + private int maxWordLen = DEFAULT_MAX_WORD_LENGTH; + + /** + * Don't return a query longer than this. + */ + private int maxQueryTerms = DEFAULT_MAX_QUERY_TERMS; + + /** + * Current set of stop words. + */ + private Set stopWords = DEFAULT_STOP_WORDS; + + /** + * The boost configuration will be used to manage how : + * - terms are boosted in the MLT query + * - fields are boosted in the MLT query + */ + private BoostProperties boostConfiguration = new BoostProperties(); + + /** + * Returns the boost configurations that regulate how terms and fields + * are boosted in a More Like This query + * + * @return boost properties that will be used when building the query + */ + public BoostProperties getBoostConfiguration() { + return boostConfiguration; + } + + /** + * Returns an analyzer that will be used to parse source doc with. The default analyzer + * is not set. + * + * @return the analyzer that will be used to parse source doc with. + */ + public Analyzer getAnalyzer() { + return analyzer; + } + + /** + * Sets the analyzer to use. An analyzer is not required for generating a query + * when using {@link MoreLikeThis} like(int docId) and term Vector is available + * for the fields we are interested in using for similarity. + * method, all other 'like' methods require an analyzer. + * + * @param analyzer the analyzer to use to tokenize text. + */ + public void setAnalyzer(Analyzer analyzer) { + this.analyzer = analyzer; + } + + public Map getFieldToAnalyzer() { + return fieldToAnalyzer; + } + + public void setFieldToAnalyzer(Map fieldToAnalyzer) { + this.fieldToAnalyzer = fieldToAnalyzer; + } + + /** + * Returns the frequency below which terms will be ignored in the source doc. The default + * frequency is the {@link #DEFAULT_MIN_TERM_FREQ}. + * + * @return the frequency below which terms will be ignored in the source doc. + */ + public int getMinTermFreq() { + return minTermFreq; + } + + /** + * Sets the frequency below which terms will be ignored in the source doc. + * + * @param minTermFreq the frequency below which terms will be ignored in the source doc. + */ + public void setMinTermFreq(int minTermFreq) { + this.minTermFreq = minTermFreq; + } + + /** + * Returns the frequency at which words will be ignored which do not occur in at least this + * many docs. The default frequency is {@link #DEFAULT_MIN_DOC_FREQ}. + * + * @return the frequency at which words will be ignored which do not occur in at least this + * many docs. + */ + public int getMinDocFreq() { + return minDocFreq; + } + + /** + * Sets the frequency at which words will be ignored which do not occur in at least this + * many docs. + * + * @param minDocFreq the frequency at which words will be ignored which do not occur in at + * least this many docs. + */ + public void setMinDocFreq(int minDocFreq) { + this.minDocFreq = minDocFreq; + } + + /** + * Returns the maximum frequency in which words may still appear. + * Words that appear in more than this many docs will be ignored. The default frequency is + * {@link #DEFAULT_MAX_DOC_FREQ}. + * + * @return get the maximum frequency at which words are still allowed, + * words which occur in more docs than this are ignored. + */ + public int getMaxDocFreq() { + return maxDocFreq; + } + + /** + * Set the maximum frequency in which words may still appear. Words that appear + * in more than this many docs will be ignored. + * + * @param maxFreq the maximum count of documents that a term may appear + * in to be still considered relevant + */ + public void setMaxDocFreq(int maxFreq) { + this.maxDocFreq = maxFreq; + } + + /** + * Set the maximum percentage in which words may still appear. Words that appear + * in more than this many percent of all docs will be ignored. + * + * @param maxPercentage the maximum percentage of documents (0-100) that a term may appear + * in to be still considered relevant + */ + public void setMaxDocFreqPct(IndexReader ir, int maxPercentage) { + this.maxDocFreq = maxPercentage * ir.numDocs() / 100; + } + + /** + * Returns the field names that will be used when generating the 'More Like This' query. + * If current field names are null, fetch them from the index reader. + * The default field names that will be used is {@link #DEFAULT_FIELD_NAMES}. + * + * @return the field names that will be used when generating the 'More Like This' query. + */ + public String[] getFieldNamesOrInit(IndexReader ir) { + if (fieldNames == null) { + // gather list of valid fields from lucene + Collection fields = MultiFields.getIndexedFields(ir); + fieldNames = fields.toArray(new String[fields.size()]); + } + return fieldNames; + } + + /** + * Returns the field names that will be used when generating the 'More Like This' query. + * The default field names that will be used is {@link #DEFAULT_FIELD_NAMES}. + * + * @return the field names that will be used when generating the 'More Like This' query. + */ + public String[] getFieldNames() { + return fieldNames; + } + + /** + * Sets the field names that will be used when generating the 'More Like This' query. + * Set this to null for the field names to be determined at runtime from the IndexReader + * provided in the constructor. + * + * @param fieldNames the field names that will be used when generating the 'More Like This' + * query. + */ + public void setFieldNames(String[] fieldNames) { + this.fieldNames = Arrays.stream(fieldNames).map(fieldName -> fieldName.split("\\^")[0]).toArray(String[]::new); + } + + /** + * Returns the minimum term length below which words will be ignored. Set this to 0 for no + * minimum term length. The default is {@link #DEFAULT_MIN_WORD_LENGTH}. + * + * @return the minimum term length below which words will be ignored. + */ + public int getMinWordLen() { + return minWordLen; + } + + /** + * Sets the minimum term length below which words will be ignored. + * + * @param minWordLen the minimum term length below which words will be ignored. + */ + public void setMinWordLen(int minWordLen) { + this.minWordLen = minWordLen; + } + + /** + * Returns the maximum term length above which words will be ignored. Set this to 0 for no + * maximum term length. The default is {@link #DEFAULT_MAX_WORD_LENGTH}. + * + * @return the maximum term length above which words will be ignored. + */ + public int getMaxWordLen() { + return maxWordLen; + } + + /** + * Sets the maximum term length above which words will be ignored. + * + * @param maxWordLen the maximum term length above which words will be ignored. + */ + public void setMaxWordLen(int maxWordLen) { + this.maxWordLen = maxWordLen; + } + + /** + * Set the set of stopwords. + * Any term in this set is considered "uninteresting" and ignored. + * Even if your Analyzer allows stopwords, you might want to tell the MoreLikeThis code to ignore them, as + * for the purposes of document similarity it seems reasonable to assume that "a stop term is never interesting". + * + * @param stopWords set of stopwords, if null it means to allow stop words + * @see #getStopWords + */ + public void setStopWords(Set stopWords) { + this.stopWords = stopWords; + } + + /** + * Get the current stop words being used. + * + * @see #setStopWords + */ + public Set getStopWords() { + return stopWords; + } + + /** + * Returns the maximum number of query terms that will be included in any generated query. + * The default is {@link #DEFAULT_MAX_QUERY_TERMS}. + * + * @return the maximum number of query terms that will be included in any generated query. + */ + public int getMaxQueryTerms() { + return maxQueryTerms; + } + + /** + * Sets the maximum number of query terms that will be included in any generated query. + * + * @param maxQueryTerms the maximum number of query terms that will be included in any + * generated query. + */ + public void setMaxQueryTerms(int maxQueryTerms) { + this.maxQueryTerms = maxQueryTerms; + } + + /** + * @return The maximum number of tokens to parse in each example doc field that is not stored with TermVector support + * @see #DEFAULT_MAX_NUM_TOKENS_PARSED + */ + public int getMaxNumTokensParsed() { + return maxNumTokensParsed; + } + + /** + * @param i The maximum number of tokens to parse in each example doc field that is not stored with TermVector support + */ + public void setMaxNumTokensParsed(int i) { + maxNumTokensParsed = i; + } + + /** + * Describe the parameters that control how the "more like this" query is formed. + */ + public String describeParams() { + StringBuilder sb = new StringBuilder(); + sb.append("\t").append("maxQueryTerms : ").append(this.getMaxQueryTerms()).append("\n"); + sb.append("\t").append("minWordLen : ").append(this.getMinWordLen()).append("\n"); + sb.append("\t").append("maxWordLen : ").append(this.getMaxWordLen()).append("\n"); + sb.append("\t").append("fieldNames : "); + String delim = ""; + for (String fieldName : fieldNames) { + sb.append(delim).append(fieldName); + delim = ", "; + } + sb.append("\n"); + sb.append("\t").append("boost : ").append(this.getBoostConfiguration().isBoostByTermScore()).append("\n"); + sb.append("\t").append("minTermFreq : ").append(this.getMinTermFreq()).append("\n"); + sb.append("\t").append("minDocFreq : ").append(this.getMinDocFreq()).append("\n"); + return sb.toString(); + } + + public class BoostProperties { + /** + * By default the query time boost factor is not applied + */ + public static final boolean DEFAULT_BOOST = false; + + /** + * If enabled a query time boost factor will applied to each query term. + * This boostFactor is the term score ( calculated by the similarity function). + * More the term is considered interesting, stronger the query time boost + */ + private boolean boostByTermScore = DEFAULT_BOOST; + + /** + * This is an additional multiplicative factor that may affect how strongly + * the query terms will be boosted. + * If a query time boost factor > 1 is specified, each query term boost + * is equal the term score multiplied by this factor. + */ + private float boostFactor = 1.0f; + + /** + * This is an alternative to the generic boost factor. + * This map allows to boost each field differently. + */ + private Map fieldToBoostFactor = new HashMap<>(); + + public BoostProperties() { + } + + public void addFieldWithBoost(String boostedField) { + String fieldName; + String boost; + if (boostedField.contains("^")) { + String[] field2boost = boostedField.split("\\^"); + fieldName = field2boost[0]; + boost = field2boost[1]; + if (boost != null) { + fieldToBoostFactor.put(fieldName, Float.parseFloat(boost)); + } + } else { + fieldToBoostFactor.put(boostedField, boostFactor); + } + } + + public void setBoostFactor(float boostFactor) { + this.boostFactor = boostFactor; + } + + public float getFieldBoost(String fieldName) { + float queryTimeBoost = boostFactor; + if (fieldToBoostFactor != null) { + Float currentFieldQueryTimeBoost = fieldToBoostFactor.get(fieldName); + if (currentFieldQueryTimeBoost != null) { + queryTimeBoost = currentFieldQueryTimeBoost; + } + } + return queryTimeBoost; + } + + public void setFieldToBoostFactor(Map fieldToBoostFactor) { + this.fieldToBoostFactor = fieldToBoostFactor; + } + + /** + * Returns whether a query time boost for each term based on its score is enabled or not. + * The default is false. + * + * @return whether to boostByTermScore terms in query based on "score" or not. + * @see #setBoost + */ + public boolean isBoostByTermScore() { + return boostByTermScore; + } + + /** + * Sets whether to set a query time boost for each term based on its score or not. + * + * @param boostEnabled true to boost each term based on its score, false otherwise. + * @see #isBoostByTermScore + */ + public void setBoost(boolean boostEnabled) { + this.boostByTermScore = boostEnabled; + } + } + +} diff --git a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java index 9f3310c7a2c8..c29636c612ce 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisQuery.java @@ -58,15 +58,16 @@ public MoreLikeThisQuery(String likeText, String[] moreLikeFields, Analyzer anal @Override public Query rewrite(IndexReader reader) throws IOException { MoreLikeThis mlt = new MoreLikeThis(reader); + MoreLikeThisParameters mltParameters = mlt.getParameters(); - mlt.setFieldNames(moreLikeFields); - mlt.setAnalyzer(analyzer); - mlt.setMinTermFreq(minTermFrequency); + mltParameters.setFieldNames(moreLikeFields); + mltParameters.setAnalyzer(analyzer); + mltParameters.setMinTermFreq(minTermFrequency); if (minDocFreq >= 0) { - mlt.setMinDocFreq(minDocFreq); + mltParameters.setMinDocFreq(minDocFreq); } - mlt.setMaxQueryTerms(maxQueryTerms); - mlt.setStopWords(stopWords); + mltParameters.setMaxQueryTerms(maxQueryTerms); + mltParameters.setStopWords(stopWords); BooleanQuery bq = (BooleanQuery) mlt.like(fieldName, new StringReader(likeText)); BooleanQuery.Builder newBq = new BooleanQuery.Builder(); for (BooleanClause clause : bq) { diff --git a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java index 32a610bf8a93..343a021d1fa8 100644 --- a/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java +++ b/lucene/queries/src/test/org/apache/lucene/queries/mlt/TestMoreLikeThis.java @@ -94,18 +94,19 @@ public void testBoostFactor() throws Throwable { Map originalValues = getOriginalValues(); MoreLikeThis mlt = new MoreLikeThis(reader); + MoreLikeThisParameters mltParameters = mlt.getParameters(); Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); - mlt.setAnalyzer(analyzer); - mlt.setMinDocFreq(1); - mlt.setMinTermFreq(1); - mlt.setMinWordLen(1); - mlt.setFieldNames(new String[] {"text"}); - mlt.setBoost(true); + mltParameters.setAnalyzer(analyzer); + mltParameters.setMinDocFreq(1); + mltParameters.setMinTermFreq(1); + mltParameters.setMinWordLen(1); + mltParameters.setFieldNames(new String[] {"text"}); + mltParameters.getBoostConfiguration().setBoost(true); // this mean that every term boost factor will be multiplied by this // number float boostFactor = 5; - mlt.setBoostFactor(boostFactor); + mltParameters.getBoostConfiguration().setBoostFactor(boostFactor); BooleanQuery query = (BooleanQuery) mlt.like("text", new StringReader( "lucene release")); @@ -131,13 +132,14 @@ public void testBoostFactor() throws Throwable { private Map getOriginalValues() throws IOException { Map originalValues = new HashMap<>(); MoreLikeThis mlt = new MoreLikeThis(reader); + MoreLikeThisParameters mltParameters = mlt.getParameters(); Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); - mlt.setAnalyzer(analyzer); - mlt.setMinDocFreq(1); - mlt.setMinTermFreq(1); - mlt.setMinWordLen(1); - mlt.setFieldNames(new String[] {"text"}); - mlt.setBoost(true); + mltParameters.setAnalyzer(analyzer); + mltParameters.setMinDocFreq(1); + mltParameters.setMinTermFreq(1); + mltParameters.setMinWordLen(1); + mltParameters.setFieldNames(new String[] {"text"}); + mltParameters.getBoostConfiguration().setBoost(true); BooleanQuery query = (BooleanQuery) mlt.like("text", new StringReader( "lucene release")); Collection clauses = query.clauses(); @@ -154,12 +156,13 @@ private Map getOriginalValues() throws IOException { // LUCENE-3326 public void testMultiFields() throws Exception { MoreLikeThis mlt = new MoreLikeThis(reader); + MoreLikeThisParameters mltParameters = mlt.getParameters(); Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); - mlt.setAnalyzer(analyzer); - mlt.setMinDocFreq(1); - mlt.setMinTermFreq(1); - mlt.setMinWordLen(1); - mlt.setFieldNames(new String[] {"text", "foobar"}); + mltParameters.setAnalyzer(analyzer); + mltParameters.setMinDocFreq(1); + mltParameters.setMinTermFreq(1); + mltParameters.setMinWordLen(1); + mltParameters.setFieldNames(new String[] {"text", "foobar"}); mlt.like("foobar", new StringReader("this is a test")); analyzer.close(); } @@ -167,12 +170,13 @@ public void testMultiFields() throws Exception { // LUCENE-5725 public void testMultiValues() throws Exception { MoreLikeThis mlt = new MoreLikeThis(reader); + MoreLikeThisParameters mltParameters = mlt.getParameters(); Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false); - mlt.setAnalyzer(analyzer); - mlt.setMinDocFreq(1); - mlt.setMinTermFreq(1); - mlt.setMinWordLen(1); - mlt.setFieldNames(new String[] {"text"}); + mltParameters.setAnalyzer(analyzer); + mltParameters.setMinDocFreq(1); + mltParameters.setMinTermFreq(1); + mltParameters.setMinWordLen(1); + mltParameters.setFieldNames(new String[] {"text"}); BooleanQuery query = (BooleanQuery) mlt.like("text", new StringReader("lucene"), new StringReader("lucene release"), @@ -209,13 +213,14 @@ public void testTopN() throws Exception { // setup MLT query MoreLikeThis mlt = new MoreLikeThis(reader); + MoreLikeThisParameters mltParameters = mlt.getParameters(); Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false); - mlt.setAnalyzer(analyzer); - mlt.setMaxQueryTerms(topN); - mlt.setMinDocFreq(1); - mlt.setMinTermFreq(1); - mlt.setMinWordLen(1); - mlt.setFieldNames(new String[]{"text"}); + mltParameters.setAnalyzer(analyzer); + mltParameters.setMaxQueryTerms(topN); + mltParameters.setMinDocFreq(1); + mltParameters.setMinTermFreq(1); + mltParameters.setMinWordLen(1); + mltParameters.setFieldNames(new String[]{"text"}); // perform MLT query String likeText = ""; @@ -294,13 +299,14 @@ public void testMultiFieldShouldReturnPerFieldBooleanQuery() throws Exception { // setup MLT query MoreLikeThis mlt = new MoreLikeThis(reader); - - mlt.setAnalyzer(analyzer); - mlt.setMaxQueryTerms(maxQueryTerms); - mlt.setMinDocFreq(1); - mlt.setMinTermFreq(1); - mlt.setMinWordLen(1); - mlt.setFieldNames(new String[]{FOR_SALE, NOT_FOR_SALE}); + MoreLikeThisParameters mltParameters = mlt.getParameters(); + + mltParameters.setAnalyzer(analyzer); + mltParameters.setMaxQueryTerms(maxQueryTerms); + mltParameters.setMinDocFreq(1); + mltParameters.setMinTermFreq(1); + mltParameters.setMinWordLen(1); + mltParameters.setFieldNames(new String[]{FOR_SALE, NOT_FOR_SALE}); // perform MLT query BooleanQuery query = (BooleanQuery) mlt.like(inputDocId); diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index 62f1016bbaf1..c8e7913216bb 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -31,6 +31,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.queries.mlt.MoreLikeThis; +import org.apache.lucene.queries.mlt.MoreLikeThisParameters; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.BoostQuery; @@ -134,12 +135,12 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw SolrIndexSearcher searcher = req.getSearcher(); - MoreLikeThisHelper mlt = new MoreLikeThisHelper(params, searcher); + MoreLikeThisHelper mltHelper = new MoreLikeThisHelper(params, searcher); // Hold on to the interesting terms if relevant TermStyle termStyle = TermStyle.get(params.get(MoreLikeThisParams.INTERESTING_TERMS)); List interesting = (termStyle == TermStyle.NONE) - ? null : new ArrayList<>(mlt.mlt.getMaxQueryTerms()); + ? null : new ArrayList<>(mltHelper.mlt.getParameters().getMaxQueryTerms()); DocListAndSet mltDocs = null; @@ -167,7 +168,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw // Find documents MoreLikeThis - either with a reader or a query // -------------------------------------------------------------------------------- if (reader != null) { - mltDocs = mlt.getMoreLikeThis(reader, start, rows, filters, + mltDocs = mltHelper.getMoreLikeThis(reader, start, rows, filters, interesting, flags); } else if (q != null) { // Matching options @@ -186,7 +187,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw if (iterator.hasNext()) { // do a MoreLikeThis query for each document in results int id = iterator.nextDoc(); - mltDocs = mlt.getMoreLikeThis(id, start, rows, filters, interesting, + mltDocs = mltHelper.getMoreLikeThis(id, start, rows, filters, interesting, flags); } } else { @@ -252,7 +253,7 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw // TODO resolve duplicated code with DebugComponent. Perhaps it should be added to doStandardDebug? if (dbg == true) { try { - NamedList dbgInfo = SolrPluginUtils.doStandardDebug(req, q, mlt.getRawMLTQuery(), mltDocs.docList, dbgQuery, dbgResults); + NamedList dbgInfo = SolrPluginUtils.doStandardDebug(req, q, mltHelper.getBoostedMLTQuery(), mltDocs.docList, dbgQuery, dbgResults); if (null != dbgInfo) { if (null != filters) { dbgInfo.add("filter_queries", req.getParams().getParams(CommonParams.FQ)); @@ -293,7 +294,6 @@ public static class MoreLikeThisHelper final IndexReader reader; final SchemaField uniqueKeyField; final boolean needDocSet; - Map boostFields; public MoreLikeThisHelper( SolrParams params, SolrIndexSearcher searcher ) { @@ -322,37 +322,37 @@ public MoreLikeThisHelper( SolrParams params, SolrIndexSearcher searcher ) } this.mlt = new MoreLikeThis( reader ); // TODO -- after LUCENE-896, we can use , searcher.getSimilarity() ); - mlt.setFieldNames(fields); - mlt.setAnalyzer( searcher.getSchema().getIndexAnalyzer() ); + MoreLikeThisParameters mltParameters = mlt.getParameters(); + MoreLikeThisParameters.BoostProperties boostConfiguration = mltParameters.getBoostConfiguration(); + + mltParameters.setFieldNames(fields); + mltParameters.setAnalyzer( searcher.getSchema().getIndexAnalyzer() ); // configurable params + + mltParameters.setMinTermFreq( params.getInt(MoreLikeThisParams.MIN_TERM_FREQ, MoreLikeThisParameters.DEFAULT_MIN_TERM_FREQ)); + mltParameters.setMinDocFreq( params.getInt(MoreLikeThisParams.MIN_DOC_FREQ, MoreLikeThisParameters.DEFAULT_MIN_DOC_FREQ)); + mltParameters.setMaxDocFreq( params.getInt(MoreLikeThisParams.MAX_DOC_FREQ, MoreLikeThisParameters.DEFAULT_MAX_DOC_FREQ)); + mltParameters.setMinWordLen( params.getInt(MoreLikeThisParams.MIN_WORD_LEN, MoreLikeThisParameters.DEFAULT_MIN_WORD_LENGTH)); + mltParameters.setMaxWordLen( params.getInt(MoreLikeThisParams.MAX_WORD_LEN, MoreLikeThisParameters.DEFAULT_MAX_WORD_LENGTH)); + mltParameters.setMaxQueryTerms( params.getInt(MoreLikeThisParams.MAX_QUERY_TERMS, MoreLikeThisParameters.DEFAULT_MAX_QUERY_TERMS)); + mltParameters.setMaxNumTokensParsed(params.getInt(MoreLikeThisParams.MAX_NUM_TOKENS_PARSED, MoreLikeThisParameters.DEFAULT_MAX_NUM_TOKENS_PARSED)); - mlt.setMinTermFreq( params.getInt(MoreLikeThisParams.MIN_TERM_FREQ, MoreLikeThis.DEFAULT_MIN_TERM_FREQ)); - mlt.setMinDocFreq( params.getInt(MoreLikeThisParams.MIN_DOC_FREQ, MoreLikeThis.DEFAULT_MIN_DOC_FREQ)); - mlt.setMaxDocFreq( params.getInt(MoreLikeThisParams.MAX_DOC_FREQ, MoreLikeThis.DEFAULT_MAX_DOC_FREQ)); - mlt.setMinWordLen( params.getInt(MoreLikeThisParams.MIN_WORD_LEN, MoreLikeThis.DEFAULT_MIN_WORD_LENGTH)); - mlt.setMaxWordLen( params.getInt(MoreLikeThisParams.MAX_WORD_LEN, MoreLikeThis.DEFAULT_MAX_WORD_LENGTH)); - mlt.setMaxQueryTerms( params.getInt(MoreLikeThisParams.MAX_QUERY_TERMS, MoreLikeThis.DEFAULT_MAX_QUERY_TERMS)); - mlt.setMaxNumTokensParsed(params.getInt(MoreLikeThisParams.MAX_NUM_TOKENS_PARSED, MoreLikeThis.DEFAULT_MAX_NUM_TOKENS_PARSED)); - mlt.setBoost( params.getBool(MoreLikeThisParams.BOOST, false ) ); + boostConfiguration.setBoost( params.getBool(MoreLikeThisParams.BOOST, false ) ); // There is no default for maxDocFreqPct. Also, it's a bit oddly expressed as an integer value // (percentage of the collection's documents count). We keep Lucene's convention here. if (params.getInt(MoreLikeThisParams.MAX_DOC_FREQ_PCT) != null) { - mlt.setMaxDocFreqPct(params.getInt(MoreLikeThisParams.MAX_DOC_FREQ_PCT)); + mltParameters.setMaxDocFreqPct(reader,params.getInt(MoreLikeThisParams.MAX_DOC_FREQ_PCT)); } - boostFields = SolrPluginUtils.parseFieldBoosts(params.getParams(MoreLikeThisParams.QF)); + String[] fieldsWithBoost = params.getParams(MoreLikeThisParams.QF); + boostConfiguration.setFieldToBoostFactor(SolrPluginUtils.parseFieldBoosts(fieldsWithBoost)); } - private Query rawMLTQuery; private Query boostedMLTQuery; private BooleanQuery realMLTQuery; - public Query getRawMLTQuery(){ - return rawMLTQuery; - } - public Query getBoostedMLTQuery(){ return boostedMLTQuery; } @@ -361,35 +361,12 @@ public Query getRealMLTQuery(){ return realMLTQuery; } - private Query getBoostedQuery(Query mltquery) { - BooleanQuery boostedQuery = (BooleanQuery)mltquery; - if (boostFields.size() > 0) { - BooleanQuery.Builder newQ = new BooleanQuery.Builder(); - newQ.setMinimumNumberShouldMatch(boostedQuery.getMinimumNumberShouldMatch()); - for (BooleanClause clause : boostedQuery) { - Query q = clause.getQuery(); - float originalBoost = 1f; - if (q instanceof BoostQuery) { - BoostQuery bq = (BoostQuery) q; - q = bq.getQuery(); - originalBoost = bq.getBoost(); - } - Float fieldBoost = boostFields.get(((TermQuery) q).getTerm().field()); - q = ((fieldBoost != null) ? new BoostQuery(q, fieldBoost * originalBoost) : clause.getQuery()); - newQ.add(q, clause.getOccur()); - } - boostedQuery = newQ.build(); - } - return boostedQuery; - } - public DocListAndSet getMoreLikeThis( int id, int start, int rows, List filters, List terms, int flags ) throws IOException { Document doc = reader.document(id); - rawMLTQuery = mlt.like(id); - boostedMLTQuery = getBoostedQuery( rawMLTQuery ); + boostedMLTQuery = mlt.like(id); if( terms != null ) { - fillInterestingTermsFromMLTQuery( rawMLTQuery, terms ); + fillInterestingTermsFromMLTQuery( boostedMLTQuery, terms ); } // exclude current document from results @@ -412,8 +389,7 @@ public DocListAndSet getMoreLikeThis( int id, int start, int rows, List f public DocListAndSet getMoreLikeThis( Reader reader, int start, int rows, List filters, List terms, int flags ) throws IOException { // analyzing with the first field: previous (stupid) behavior - rawMLTQuery = mlt.like(mlt.getFieldNames()[0], reader); - boostedMLTQuery = getBoostedQuery( rawMLTQuery ); + boostedMLTQuery = mlt.like(mlt.getParameters().getFieldNames()[0], reader); if( terms != null ) { fillInterestingTermsFromMLTQuery( boostedMLTQuery, terms ); } @@ -439,12 +415,9 @@ public NamedList getMoreLikeTheseQuery(DocList docs) if (mltquery.clauses().size() == 0) { return result; } - mltquery = (BooleanQuery) getBoostedQuery(mltquery); - // exclude current document from results BooleanQuery.Builder mltQuery = new BooleanQuery.Builder(); mltQuery.add(mltquery, BooleanClause.Occur.MUST); - mltQuery.add( new TermQuery(new Term(uniqueKeyField.getName(), uniqueId)), BooleanClause.Occur.MUST_NOT); result.add(uniqueId, mltQuery.build()); diff --git a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java index fd9d37d4aad7..f06cb6be9fbe 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java @@ -381,6 +381,10 @@ NamedList getMoreLikeThese(ResponseBuilder rb, while (iterator.hasNext()) { int id = iterator.nextDoc(); int rows = p.getInt(MoreLikeThisParams.DOC_COUNT, 5); + boolean interestingTermsExtraction = p.getBool(MoreLikeThisParams.INTERESTING_TERMS, false); + if(interestingTermsExtraction){ + + } DocListAndSet sim = mltHelper.getMoreLikeThis(id, 0, rows, null, null, flags); String name = schema.printableUniqueKey(searcher.doc(id)); @@ -388,7 +392,6 @@ NamedList getMoreLikeThese(ResponseBuilder rb, if (dbg != null) { SimpleOrderedMap docDbg = new SimpleOrderedMap<>(); - docDbg.add("rawMLTQuery", mltHelper.getRawMLTQuery().toString()); docDbg .add("boostedMLTQuery", mltHelper.getBoostedMLTQuery().toString()); docDbg.add("realMLTQuery", mltHelper.getRealMLTQuery().toString()); diff --git a/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java b/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java index 7669db8634a2..ae30d34dde91 100644 --- a/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java +++ b/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java @@ -25,6 +25,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.mlt.MoreLikeThisParameters; import org.apache.solr.legacy.LegacyNumericUtils; import org.apache.lucene.queries.mlt.MoreLikeThis; import org.apache.lucene.search.BooleanClause; @@ -73,19 +74,21 @@ public Query parse() { String[] qf = localParams.getParams("qf"); Map boostFields = new HashMap<>(); MoreLikeThis mlt = new MoreLikeThis(req.getSearcher().getIndexReader()); + MoreLikeThisParameters mltParameters = mlt.getParameters(); + MoreLikeThisParameters.BoostProperties boostConfiguration = mltParameters.getBoostConfiguration(); - mlt.setMinTermFreq(localParams.getInt("mintf", MoreLikeThis.DEFAULT_MIN_TERM_FREQ)); - mlt.setMinDocFreq(localParams.getInt("mindf", 0)); - mlt.setMinWordLen(localParams.getInt("minwl", MoreLikeThis.DEFAULT_MIN_WORD_LENGTH)); - mlt.setMaxWordLen(localParams.getInt("maxwl", MoreLikeThis.DEFAULT_MAX_WORD_LENGTH)); - mlt.setMaxQueryTerms(localParams.getInt("maxqt", MoreLikeThis.DEFAULT_MAX_QUERY_TERMS)); - mlt.setMaxNumTokensParsed(localParams.getInt("maxntp", MoreLikeThis.DEFAULT_MAX_NUM_TOKENS_PARSED)); - mlt.setMaxDocFreq(localParams.getInt("maxdf", MoreLikeThis.DEFAULT_MAX_DOC_FREQ)); + mltParameters.setMinTermFreq(localParams.getInt("mintf", MoreLikeThisParameters.DEFAULT_MIN_TERM_FREQ)); + mltParameters.setMinDocFreq(localParams.getInt("mindf", 0)); + mltParameters.setMinWordLen(localParams.getInt("minwl", MoreLikeThisParameters.DEFAULT_MIN_WORD_LENGTH)); + mltParameters.setMaxWordLen(localParams.getInt("maxwl", MoreLikeThisParameters.DEFAULT_MAX_WORD_LENGTH)); + mltParameters.setMaxQueryTerms(localParams.getInt("maxqt", MoreLikeThisParameters.DEFAULT_MAX_QUERY_TERMS)); + mltParameters.setMaxNumTokensParsed(localParams.getInt("maxntp", MoreLikeThisParameters.DEFAULT_MAX_NUM_TOKENS_PARSED)); + mltParameters.setMaxDocFreq(localParams.getInt("maxdf", MoreLikeThisParameters.DEFAULT_MAX_DOC_FREQ)); - Boolean boost = localParams.getBool("boost", MoreLikeThis.DEFAULT_BOOST); - mlt.setBoost(boost); + Boolean boost = localParams.getBool("boost",MoreLikeThisParameters.BoostProperties.DEFAULT_BOOST); + boostConfiguration.setBoost(boost); - mlt.setAnalyzer(req.getSchema().getIndexAnalyzer()); + mltParameters.setAnalyzer(req.getSchema().getIndexAnalyzer()); Map> filteredDocument = new HashMap<>(); String[] fieldNames; @@ -104,6 +107,7 @@ public Query parse() { } // Parse field names and boosts from the fields boostFields = SolrPluginUtils.parseFieldBoosts(fields.toArray(new String[0])); + boostConfiguration.setFieldToBoostFactor(boostFields); fieldNames = boostFields.keySet().toArray(new String[0]); } else { ArrayList fields = new ArrayList(); @@ -124,7 +128,7 @@ public Query parse() { "MoreLikeThis requires at least one similarity field: qf" ); } - mlt.setFieldNames(fieldNames); + mltParameters.setFieldNames(fieldNames); for (String field : fieldNames) { Collection fieldValues = doc.getFieldValues(field); if (fieldValues != null) { @@ -142,29 +146,7 @@ public Query parse() { } try { - Query rawMLTQuery = mlt.like(filteredDocument); - BooleanQuery boostedMLTQuery = (BooleanQuery) rawMLTQuery; - - if (boost && boostFields.size() > 0) { - BooleanQuery.Builder newQ = new BooleanQuery.Builder(); - newQ.setMinimumNumberShouldMatch(boostedMLTQuery.getMinimumNumberShouldMatch()); - - for (BooleanClause clause : boostedMLTQuery) { - Query q = clause.getQuery(); - float originalBoost = 1f; - if (q instanceof BoostQuery) { - BoostQuery bq = (BoostQuery) q; - q = bq.getQuery(); - originalBoost = bq.getBoost(); - } - Float fieldBoost = boostFields.get(((TermQuery) q).getTerm().field()); - q = ((fieldBoost != null) ? new BoostQuery(q, fieldBoost * originalBoost) : clause.getQuery()); - newQ.add(q, clause.getOccur()); - } - - boostedMLTQuery = QueryUtils.build(newQ, this); - } - + Query boostedMLTQuery = mlt.like(filteredDocument); // exclude current document from results BooleanQuery.Builder realMLTQuery = new BooleanQuery.Builder(); realMLTQuery.add(boostedMLTQuery, BooleanClause.Occur.MUST); diff --git a/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java b/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java index 4a3400b70f06..1b45ed4e7357 100644 --- a/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java +++ b/solr/core/src/java/org/apache/solr/search/mlt/SimpleMLTQParser.java @@ -16,6 +16,7 @@ */ package org.apache.solr.search.mlt; import org.apache.lucene.index.Term; +import org.apache.lucene.queries.mlt.MoreLikeThisParameters; import org.apache.solr.legacy.LegacyNumericUtils; import org.apache.lucene.queries.mlt.MoreLikeThis; import org.apache.lucene.search.BooleanClause; @@ -69,16 +70,18 @@ public Query parse() { "document with id [" + uniqueValue + "]"); ScoreDoc[] scoreDocs = td.scoreDocs; MoreLikeThis mlt = new MoreLikeThis(req.getSearcher().getIndexReader()); - - mlt.setMinTermFreq(localParams.getInt("mintf", MoreLikeThis.DEFAULT_MIN_TERM_FREQ)); - mlt.setMinDocFreq(localParams.getInt("mindf", MoreLikeThis.DEFAULT_MIN_DOC_FREQ)); - mlt.setMinWordLen(localParams.getInt("minwl", MoreLikeThis.DEFAULT_MIN_WORD_LENGTH)); - mlt.setMaxWordLen(localParams.getInt("maxwl", MoreLikeThis.DEFAULT_MAX_WORD_LENGTH)); - mlt.setMaxQueryTerms(localParams.getInt("maxqt", MoreLikeThis.DEFAULT_MAX_QUERY_TERMS)); - mlt.setMaxNumTokensParsed(localParams.getInt("maxntp", MoreLikeThis.DEFAULT_MAX_NUM_TOKENS_PARSED)); - mlt.setMaxDocFreq(localParams.getInt("maxdf", MoreLikeThis.DEFAULT_MAX_DOC_FREQ)); - Boolean boost = localParams.getBool("boost", false); - mlt.setBoost(boost); + MoreLikeThisParameters mltParameters = mlt.getParameters(); + MoreLikeThisParameters.BoostProperties boostConfiguration = mltParameters.getBoostConfiguration(); + + mltParameters.setMinTermFreq(localParams.getInt("mintf", MoreLikeThisParameters.DEFAULT_MIN_TERM_FREQ)); + mltParameters.setMinDocFreq(localParams.getInt("mindf", MoreLikeThisParameters.DEFAULT_MIN_DOC_FREQ)); + mltParameters.setMinWordLen(localParams.getInt("minwl", MoreLikeThisParameters.DEFAULT_MIN_WORD_LENGTH)); + mltParameters.setMaxWordLen(localParams.getInt("maxwl", MoreLikeThisParameters.DEFAULT_MAX_WORD_LENGTH)); + mltParameters.setMaxQueryTerms(localParams.getInt("maxqt", MoreLikeThisParameters.DEFAULT_MAX_QUERY_TERMS)); + mltParameters.setMaxNumTokensParsed(localParams.getInt("maxntp", MoreLikeThisParameters.DEFAULT_MAX_NUM_TOKENS_PARSED)); + mltParameters.setMaxDocFreq(localParams.getInt("maxdf", MoreLikeThisParameters.DEFAULT_MAX_DOC_FREQ)); + Boolean boost = localParams.getBool("boost",MoreLikeThisParameters.BoostProperties.DEFAULT_BOOST); + boostConfiguration.setBoost(boost); String[] fieldNames; @@ -112,8 +115,8 @@ public Query parse() { "MoreLikeThis requires at least one similarity field: qf" ); } - mlt.setFieldNames(fieldNames); - mlt.setAnalyzer(req.getSchema().getIndexAnalyzer()); + mltParameters.setFieldNames(fieldNames); + mltParameters.setAnalyzer(req.getSchema().getIndexAnalyzer()); Query rawMLTQuery = mlt.like(scoreDocs[0].doc); BooleanQuery boostedMLTQuery = (BooleanQuery) rawMLTQuery; diff --git a/solr/core/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java index aa63ce325761..d514ca44f671 100644 --- a/solr/core/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/MoreLikeThisHandlerTest.java @@ -103,7 +103,6 @@ public void testInterface() throws Exception params.set(CommonParams.DEBUG_QUERY, "true"); mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); assertQ("morelike this - harrison ford",mltreq - ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='rawMLTQuery']" ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='boostedMLTQuery']" ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='realMLTQuery']" ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/lst[@name='explain']/str[@name='45']" From e944e83b137527d5128a56be0253d25f4db9395f Mon Sep 17 00:00:00 2001 From: Alessandro Benedetti Date: Wed, 2 May 2018 17:39:53 +0100 Subject: [PATCH 2/4] [SOLR-12304] More Like This component interesting term fix +tests --- .../solr/handler/MoreLikeThisHandler.java | 2 +- .../component/MoreLikeThisComponent.java | 44 ++- .../component/MoreLikeThisComponentTest.java | 286 ++++++++++++++++++ 3 files changed, 321 insertions(+), 11 deletions(-) create mode 100644 solr/core/src/test/org/apache/solr/handler/component/MoreLikeThisComponentTest.java diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index 62f1016bbaf1..ace4ff3c90e7 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -389,7 +389,7 @@ public DocListAndSet getMoreLikeThis( int id, int start, int rows, List f rawMLTQuery = mlt.like(id); boostedMLTQuery = getBoostedQuery( rawMLTQuery ); if( terms != null ) { - fillInterestingTermsFromMLTQuery( rawMLTQuery, terms ); + fillInterestingTermsFromMLTQuery( boostedMLTQuery, terms ); } // exclude current document from results diff --git a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java index fd9d37d4aad7..c4afc1d9e22f 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java @@ -191,11 +191,6 @@ public void finishStage(ResponseBuilder rb) { log.info("MLT: results added for key: " + key + " documents: " + shardDocList.toString()); -// if (log.isDebugEnabled()) { -// for (SolrDocument doc : shardDocList) { -// doc.addField("shard", "=" + r.getShard()); -// } -// } SolrDocumentList mergedDocList = tempResults.get(key); if (mergedDocList == null) { @@ -370,21 +365,31 @@ NamedList getMoreLikeThese(ResponseBuilder rb, IndexSchema schema = searcher.getSchema(); MoreLikeThisHandler.MoreLikeThisHelper mltHelper = new MoreLikeThisHandler.MoreLikeThisHelper( p, searcher); - NamedList mlt = new SimpleOrderedMap<>(); + NamedList mltResponse = new SimpleOrderedMap<>(); DocIterator iterator = docs.iterator(); SimpleOrderedMap dbg = null; if (rb.isDebug()) { dbg = new SimpleOrderedMap<>(); } + + SimpleOrderedMap interestingTermsResponse = null; + MoreLikeThisParams.TermStyle termStyle = MoreLikeThisParams.TermStyle.get(p.get(MoreLikeThisParams.INTERESTING_TERMS)); + List interestingTerms = (termStyle == MoreLikeThisParams.TermStyle.NONE) + ? null : new ArrayList<>(mltHelper.getMoreLikeThis().getMaxQueryTerms()); + + if (interestingTerms!=null) { + interestingTermsResponse = new SimpleOrderedMap<>(); + } while (iterator.hasNext()) { int id = iterator.nextDoc(); int rows = p.getInt(MoreLikeThisParams.DOC_COUNT, 5); - DocListAndSet sim = mltHelper.getMoreLikeThis(id, 0, rows, null, null, + + DocListAndSet sim = mltHelper.getMoreLikeThis(id, 0, rows, null, interestingTerms, flags); String name = schema.printableUniqueKey(searcher.doc(id)); - mlt.add(name, sim.docList); + mltResponse.add(name, sim.docList); if (dbg != null) { SimpleOrderedMap docDbg = new SimpleOrderedMap<>(); @@ -403,13 +408,32 @@ NamedList getMoreLikeThese(ResponseBuilder rb, docDbg.add("explain", explains); dbg.add(name, docDbg); } + + if (interestingTermsResponse != null) { + if (termStyle == MoreLikeThisParams.TermStyle.DETAILS) { + NamedList interestingTermsWithScore = new NamedList<>(); + for (MoreLikeThisHandler.InterestingTerm interestingTerm : interestingTerms) { + interestingTermsWithScore.add(interestingTerm.term.toString(), interestingTerm.boost); + } + interestingTermsResponse.add(name, interestingTermsWithScore); + } else { + List interestingTermsString = new ArrayList<>(interestingTerms.size()); + for(MoreLikeThisHandler.InterestingTerm interestingTerm : interestingTerms){ + interestingTermsString.add(interestingTerm.term.toString()); + } + interestingTermsResponse.add(name, interestingTermsString); + } + } } - // add debug information if (dbg != null) { rb.addDebugInfo("moreLikeThis", dbg); } - return mlt; + // add Interesting Terms + if (interestingTermsResponse != null) { + rb.rsp.add("interestingTerms", interestingTermsResponse); + } + return mltResponse; } // /////////////////////////////////////////// diff --git a/solr/core/src/test/org/apache/solr/handler/component/MoreLikeThisComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/MoreLikeThisComponentTest.java new file mode 100644 index 000000000000..764d26078362 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/component/MoreLikeThisComponentTest.java @@ -0,0 +1,286 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.component; + +import org.apache.lucene.util.LuceneTestCase.Slow; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.MoreLikeThisParams; +import org.apache.solr.core.SolrCore; +import org.apache.solr.request.LocalSolrQueryRequest; +import org.apache.solr.request.SolrQueryRequest; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Test for MoreLikeThisComponent + * + * + * @see MoreLikeThisComponent + */ +@Slow +public class MoreLikeThisComponentTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void moreLikeThisBeforeClass() throws Exception { + initCore("solrconfig.xml", "schema.xml"); + assertU(adoc("id","42","name","Tom Cruise","subword","Top Gun","subword","Risky Business","subword","The Color of Money","subword","Minority Report","subword", "Days of Thunder","subword", "Eyes Wide Shut","subword", "Far and Away", "foo_ti","10")); + assertU(adoc("id","43","name","Tom Hanks","subword","The Green Mile","subword","Forest Gump","subword","Philadelphia Story","subword","Big","subword","Cast Away", "foo_ti","10")); + assertU(adoc("id","44","name","Harrison Ford","subword","Star Wars","subword","Indiana Jones","subword","Patriot Games","subword","Regarding Henry")); + assertU(adoc("id","45","name","George Harrison","subword","Yellow Submarine","subword","Help","subword","Magical Mystery Tour","subword","Sgt. Peppers Lonley Hearts Club Band")); + assertU(adoc("id","46","name","Nicole Kidman","subword","Batman","subword","Days of Thunder","subword","Eyes Wide Shut","subword","Far and Away")); + assertU(commit()); + } + + private void initCommonMoreLikeThisParams(ModifiableSolrParams params) { + params.set(MoreLikeThisParams.MLT, "true"); + params.set(MoreLikeThisParams.SIMILARITY_FIELDS, "name,subword"); + params.set(MoreLikeThisParams.MIN_TERM_FREQ,"1"); + params.set(MoreLikeThisParams.MIN_DOC_FREQ,"1"); + params.set("indent","true"); + } + + @Test + public void testMLT_baseParams_shouldReturnSimilarDocuments() + { + SolrCore core = h.getCore(); + ModifiableSolrParams params = new ModifiableSolrParams(); + + initCommonMoreLikeThisParams(params); + + params.set(CommonParams.Q, "id:42"); + SolrQueryRequest mltreq = new LocalSolrQueryRequest( core, params); + assertQ("morelikethis - tom cruise",mltreq + ,"//result/doc[1]/str[@name='id'][.='46']" + ,"//result/doc[2]/str[@name='id'][.='43']"); + + params.set(CommonParams.Q, "id:44"); + mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); + assertQ("morelike this - harrison ford",mltreq + ,"//result/doc[1]/str[@name='id'][.='45']"); + mltreq.close(); + } + + @Test + public void testMLT_baseParamsInterestingTermsDetails_shouldReturnSimilarDocumentsAndInterestingTermsDetails() + { + SolrCore core = h.getCore(); + ModifiableSolrParams params = new ModifiableSolrParams(); + + initCommonMoreLikeThisParams(params); + params.set(MoreLikeThisParams.INTERESTING_TERMS, "details"); + + params.set(CommonParams.Q, "id:42"); + SolrQueryRequest mltreq = new LocalSolrQueryRequest( core, params); + assertQ("morelikethis - tom cruise",mltreq + ,"//result/doc[1]/str[@name='id'][.='46']" + ,"//result/doc[2]/str[@name='id'][.='43']", + "//lst[@name='interestingTerms']/lst[1][count(*)>0]", + "//lst[@name='interestingTerms']/lst[1]/float[.=1.0]"); + mltreq.close(); + } + + @Test + public void testMLT_baseParamsInterestingTermsList_shouldReturnSimilarDocumentsAndInterestingTermsList() + { + SolrCore core = h.getCore(); + ModifiableSolrParams params = new ModifiableSolrParams(); + + initCommonMoreLikeThisParams(params); + params.set(MoreLikeThisParams.INTERESTING_TERMS, "list"); + + params.set(CommonParams.Q, "id:42"); + SolrQueryRequest mltreq = new LocalSolrQueryRequest( core, params); + assertQ("morelikethis - tom cruise",mltreq + ,"//result/doc[1]/str[@name='id'][.='46']" + ,"//result/doc[2]/str[@name='id'][.='43']", + "//lst[@name='interestingTerms']/arr[@name='42'][count(*)>0]", + "//lst[@name='interestingTerms']/arr[@name='42']/str[.='name:Cruise']"); + mltreq.close(); + } + + @Test + public void testMLT_boostEnabled_shouldReturnSimilarDocumentsConsideringBoost() + { + SolrCore core = h.getCore(); + ModifiableSolrParams params = new ModifiableSolrParams(); + + initCommonMoreLikeThisParams(params); + params.set(MoreLikeThisParams.BOOST, "true"); + + params.set(CommonParams.Q, "id:42"); + SolrQueryRequest mltreq = new LocalSolrQueryRequest( core, params); + assertQ("morelikethis - tom cruise",mltreq + ,"//result/doc[1]/str[@name='id'][.='46']" + ,"//result/doc[2]/str[@name='id'][.='43']"); + + params.set(CommonParams.Q, "id:42"); + params.set(MoreLikeThisParams.QF,"name^5.0 subword^0.1"); + mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); + assertQ("morelikethis with weights",mltreq + ,"//result/doc[1]/str[@name='id'][.='43']" + ,"//result/doc[2]/str[@name='id'][.='46']"); + + mltreq.close(); + } + + @Test + public void testMLT_boostEnabledInterestingTermsDetails_shouldReturnSimilarDocumentsConsideringBoostAndInterestingTermsDetails() + { + SolrCore core = h.getCore(); + ModifiableSolrParams params = new ModifiableSolrParams(); + + initCommonMoreLikeThisParams(params); + params.set(MoreLikeThisParams.BOOST, "true"); + params.set(MoreLikeThisParams.INTERESTING_TERMS, "details"); + + params.set(CommonParams.Q, "id:42"); + SolrQueryRequest mltreq = new LocalSolrQueryRequest( core, params); + assertQ("morelikethis - tom cruise",mltreq + ,"//result/doc[1]/str[@name='id'][.='46']" + ,"//result/doc[2]/str[@name='id'][.='43']", + "//lst[@name='interestingTerms']/lst[1][count(*)>0]", + "//lst[@name='interestingTerms']/lst[1]/float[.>1.0]"); + + params.set(MoreLikeThisParams.QF,"name^5.0 subword^0.1"); + mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); + assertQ("morelikethis with weights",mltreq + ,"//result/doc[1]/str[@name='id'][.='43']" + ,"//result/doc[2]/str[@name='id'][.='46']", + "//lst[@name='interestingTerms']/lst[1][count(*)>0]", + "//lst[@name='interestingTerms']/lst[1]/float[.>5.0]"); + + mltreq.close(); + } + + @Test + public void testMLT_boostEnabledInterestingTermsList_shouldReturnSimilarDocumentsConsideringBoostAndInterestingTermsList() + { + SolrCore core = h.getCore(); + ModifiableSolrParams params = new ModifiableSolrParams(); + + initCommonMoreLikeThisParams(params); + params.set(MoreLikeThisParams.BOOST, "true"); + params.set(MoreLikeThisParams.INTERESTING_TERMS, "list"); + + params.set(CommonParams.Q, "id:42"); + SolrQueryRequest mltreq = new LocalSolrQueryRequest( core, params); + assertQ("morelikethis - tom cruise",mltreq + ,"//result/doc[1]/str[@name='id'][.='46']" + ,"//result/doc[2]/str[@name='id'][.='43']", + "//lst[@name='interestingTerms']/arr[@name='42'][count(*)>0]", + "//lst[@name='interestingTerms']/arr[@name='42']/str[.='name:Cruise']"); + + params.set(MoreLikeThisParams.QF,"name^5.0 subword^0.1"); + mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); + assertQ("morelikethis with weights",mltreq + ,"//result/doc[1]/str[@name='id'][.='43']" + ,"//result/doc[2]/str[@name='id'][.='46']", + "//lst[@name='interestingTerms']/arr[@name='42'][count(*)>0]", + "//lst[@name='interestingTerms']/arr[@name='42']/str[.='name:Cruise']"); + + mltreq.close(); + } + + @Test + public void testMLT_debugEnabled_shouldReturnSimilarDocumentsWithDebug() + { + ModifiableSolrParams params = new ModifiableSolrParams(); + + initCommonMoreLikeThisParams(params); + params.set(MoreLikeThisParams.BOOST, "true"); + + params.set(CommonParams.Q, "id:44"); + params.set(CommonParams.DEBUG_QUERY, "true"); + SolrQueryRequest mltreq = new LocalSolrQueryRequest(h.getCore(), params); + assertQ("morelike this - harrison ford",mltreq + ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='rawMLTQuery']" + ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='boostedMLTQuery']" + ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='realMLTQuery']" + ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/lst[@name='explain']/str[@name='45']" + ); + + params.remove(CommonParams.DEBUG_QUERY); + params.set(CommonParams.Q, "{!field f=id}44"); + mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); + assertQ(mltreq + ,"//result/doc[1]/str[@name='id'][.='45']"); + mltreq.close(); + } + + @Test + public void testMLT_debugEnabledInterestingTermsDetails_shouldReturnSimilarDocumentsWithDebugAndInterestingTermsDetails() + { + ModifiableSolrParams params = new ModifiableSolrParams(); + + initCommonMoreLikeThisParams(params); + params.set(MoreLikeThisParams.BOOST, "true"); + params.set(MoreLikeThisParams.INTERESTING_TERMS, "details"); + + params.set(CommonParams.Q, "id:44"); + params.set(CommonParams.DEBUG_QUERY, "true"); + SolrQueryRequest mltreq = new LocalSolrQueryRequest(h.getCore(), params); + assertQ("morelike this - harrison ford",mltreq + ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='rawMLTQuery']" + ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='boostedMLTQuery']" + ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='realMLTQuery']" + ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/lst[@name='explain']/str[@name='45']", + "//lst[@name='interestingTerms']/lst[1][count(*)>0]", + "//lst[@name='interestingTerms']/lst[1]/float[.>1.0]"); + + params.remove(CommonParams.DEBUG_QUERY); + params.set(CommonParams.Q, "{!field f=id}44"); + mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); + assertQ(mltreq + ,"//result/doc[1]/str[@name='id'][.='45']", + "//lst[@name='interestingTerms']/lst[1][count(*)>0]", + "//lst[@name='interestingTerms']/lst[1]/float[.>1.0]"); + mltreq.close(); + } + + @Test + public void testMLT_debugEnabledInterestingTermsList_shouldReturnSimilarDocumentsWithDebugAndInterestingTermsList() + { + ModifiableSolrParams params = new ModifiableSolrParams(); + + initCommonMoreLikeThisParams(params); + params.set(MoreLikeThisParams.BOOST, "true"); + params.set(MoreLikeThisParams.INTERESTING_TERMS, "list"); + + params.set(CommonParams.Q, "id:44"); + params.set(CommonParams.DEBUG_QUERY, "true"); + + SolrQueryRequest mltreq = new LocalSolrQueryRequest(h.getCore(), params); + assertQ("morelike this - harrison ford",mltreq + ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='rawMLTQuery']" + ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='boostedMLTQuery']" + ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='realMLTQuery']" + ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/lst[@name='explain']/str[@name='45']", + "//lst[@name='interestingTerms']/arr[@name='44'][count(*)>0]", + "//lst[@name='interestingTerms']/arr[@name='44']/str[.='name:Harrison']"); + + params.remove(CommonParams.DEBUG_QUERY); + params.set(CommonParams.Q, "{!field f=id}44"); + mltreq.close(); mltreq = new LocalSolrQueryRequest(h.getCore(), params); + assertQ(mltreq + ,"//result/doc[1]/str[@name='id'][.='45']", + "//lst[@name='interestingTerms']/arr[@name='44'][count(*)>0]", + "//lst[@name='interestingTerms']/arr[@name='44']/str[.='name:Harrison']"); + mltreq.close(); + } +} From 40705fb6e5a25cc18767eb3441f33e4cd016f6af Mon Sep 17 00:00:00 2001 From: Alessandro Benedetti Date: Thu, 3 May 2018 17:44:20 +0100 Subject: [PATCH 3/4] [SOLR-12299] More Like This Parameters + tests --- .../queries/mlt/MoreLikeThisParameters.java | 25 ++-- .../mlt/MoreLikeThisParametersTest.java | 107 ++++++++++++++++++ .../component/MoreLikeThisComponent.java | 2 +- .../component/MoreLikeThisComponentTest.java | 5 +- 4 files changed, 117 insertions(+), 22 deletions(-) create mode 100644 lucene/queries/src/test/org/apache/lucene/queries/mlt/MoreLikeThisParametersTest.java diff --git a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisParameters.java b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisParameters.java index 34e19ed47972..f2a979c7b773 100644 --- a/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisParameters.java +++ b/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThisParameters.java @@ -90,7 +90,7 @@ public final class MoreLikeThisParameters { * Default field names. Null is used to specify that the field names should be looked * up at runtime from the provided reader. */ - public static final String[] DEFAULT_FIELD_NAMES = new String[]{"contents"}; + public static final String[] DEFAULT_FIELD_NAMES = null; /** * Ignore words less than this length or if 0 then this has no effect. @@ -132,12 +132,6 @@ public final class MoreLikeThisParameters { */ private Analyzer analyzer = null; - /** - * Advanced : - * Pass a specific Analyzer per field - */ - private Map fieldToAnalyzer; - /** * Ignore words less frequent that this. */ @@ -222,14 +216,6 @@ public void setAnalyzer(Analyzer analyzer) { this.analyzer = analyzer; } - public Map getFieldToAnalyzer() { - return fieldToAnalyzer; - } - - public void setFieldToAnalyzer(Map fieldToAnalyzer) { - this.fieldToAnalyzer = fieldToAnalyzer; - } - /** * Returns the frequency below which terms will be ignored in the source doc. The default * frequency is the {@link #DEFAULT_MIN_TERM_FREQ}. @@ -464,10 +450,15 @@ public class BoostProperties { * By default the query time boost factor is not applied */ public static final boolean DEFAULT_BOOST = false; + + /** + * By default the query time boost factor is equal to 1.0 + */ + private static final float DEFAULT_BOOST_FACTOR = 1.0f; /** * If enabled a query time boost factor will applied to each query term. - * This boostFactor is the term score ( calculated by the similarity function). + * This boost factor is the term score ( calculated by the similarity function). * More the term is considered interesting, stronger the query time boost */ private boolean boostByTermScore = DEFAULT_BOOST; @@ -478,7 +469,7 @@ public class BoostProperties { * If a query time boost factor > 1 is specified, each query term boost * is equal the term score multiplied by this factor. */ - private float boostFactor = 1.0f; + private float boostFactor = DEFAULT_BOOST_FACTOR; /** * This is an alternative to the generic boost factor. diff --git a/lucene/queries/src/test/org/apache/lucene/queries/mlt/MoreLikeThisParametersTest.java b/lucene/queries/src/test/org/apache/lucene/queries/mlt/MoreLikeThisParametersTest.java new file mode 100644 index 000000000000..866ff23e3a4c --- /dev/null +++ b/lucene/queries/src/test/org/apache/lucene/queries/mlt/MoreLikeThisParametersTest.java @@ -0,0 +1,107 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.queries.mlt; + +import java.io.IOException; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; + +import static org.hamcrest.core.Is.is; + +public class MoreLikeThisParametersTest extends LuceneTestCase { + private MoreLikeThisParameters toTest = new MoreLikeThisParameters(); + + public void testMLTParameters_setFieldNames_shouldRemoveBoosts() { + String[] fieldNames = new String[]{"field1", "field2^5.0", "field3^1.0", "field4"}; + + toTest.setFieldNames(fieldNames); + + assertThat(toTest.getFieldNames(), is(new String[]{"field1", "field2", "field3", "field4"})); + } + + public void testMLTParameters_fieldNamesNotInitialised_shouldFetchThemFromIndex() throws IOException { + Directory directory = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random(), directory); + + Document doc = new Document(); + doc.add(newTextField("field1", "lucene", Field.Store.YES)); + doc.add(newTextField("field2", "more like this", Field.Store.YES)); + writer.addDocument(doc); + + IndexReader reader = writer.getReader(); + writer.close(); + + toTest.getFieldNamesOrInit(reader); + + assertThat(toTest.getFieldNames(), is(new String[]{"field1", "field2"})); + + reader.close(); + directory.close(); + } + + public void testMLTParameters_describeParams_shouldReturnParamsDescriptionString() { + String[] fieldNames = new String[]{"field1", "field2^5.0", "field3^1.0", "field4"}; + toTest.setFieldNames(fieldNames); + + String paramsDescription = toTest.describeParams(); + + assertThat(paramsDescription, + is("\tmaxQueryTerms : 25\n" + + "\tminWordLen : 0\n" + + "\tmaxWordLen : 0\n" + + "\tfieldNames : field1, field2, field3, field4\n" + + "\tboost : false\n" + + "\tminTermFreq : 2\n" + + "\tminDocFreq : 5\n")); + } + + public void testMLTParameters_setBoostFields_shouldParseBoosts() { + String[] fieldNames = new String[]{"field1", "field2^2.0", "field3^3.0", "field4"}; + MoreLikeThisParameters.BoostProperties boostConfiguration = toTest.getBoostConfiguration(); + + for (String fieldWithBoost : fieldNames) { + boostConfiguration.addFieldWithBoost(fieldWithBoost); + } + + assertThat(boostConfiguration.getFieldBoost("field1"), is(1.0F)); + assertThat(boostConfiguration.getFieldBoost("field2"), is(2.0F)); + assertThat(boostConfiguration.getFieldBoost("field3"), is(3.0F)); + assertThat(boostConfiguration.getFieldBoost("field4"), is(1.0F)); + } + + public void testMLTParameters_noBoostConfiguration_shouldReturnDefaultBoost() { + MoreLikeThisParameters.BoostProperties boostConfiguration = toTest.getBoostConfiguration(); + + assertThat(boostConfiguration.getFieldBoost("field1"), is(1.0F)); + assertThat(boostConfiguration.getFieldBoost("field2"), is(1.0F)); + assertThat(boostConfiguration.getFieldBoost("field3"), is(1.0F)); + assertThat(boostConfiguration.getFieldBoost("field4"), is(1.0F)); + + boostConfiguration.setBoostFactor(5.0f); + + assertThat(boostConfiguration.getFieldBoost("field1"), is(5.0F)); + assertThat(boostConfiguration.getFieldBoost("field2"), is(5.0F)); + assertThat(boostConfiguration.getFieldBoost("field3"), is(5.0F)); + assertThat(boostConfiguration.getFieldBoost("field4"), is(5.0F)); + } +} diff --git a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java index 876566a8f32a..daa2dce125a4 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java @@ -376,7 +376,7 @@ NamedList getMoreLikeThese(ResponseBuilder rb, SimpleOrderedMap interestingTermsResponse = null; MoreLikeThisParams.TermStyle termStyle = MoreLikeThisParams.TermStyle.get(p.get(MoreLikeThisParams.INTERESTING_TERMS)); List interestingTerms = (termStyle == MoreLikeThisParams.TermStyle.NONE) - ? null : new ArrayList<>(mltHelper.getMoreLikeThis().getMaxQueryTerms()); + ? null : new ArrayList<>(mltHelper.getMoreLikeThis().getParameters().getMaxQueryTerms()); if (interestingTerms!=null) { interestingTermsResponse = new SimpleOrderedMap<>(); diff --git a/solr/core/src/test/org/apache/solr/handler/component/MoreLikeThisComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/MoreLikeThisComponentTest.java index 764d26078362..326b6b95d8b2 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/MoreLikeThisComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/MoreLikeThisComponentTest.java @@ -209,7 +209,6 @@ public void testMLT_debugEnabled_shouldReturnSimilarDocumentsWithDebug() params.set(CommonParams.DEBUG_QUERY, "true"); SolrQueryRequest mltreq = new LocalSolrQueryRequest(h.getCore(), params); assertQ("morelike this - harrison ford",mltreq - ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='rawMLTQuery']" ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='boostedMLTQuery']" ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='realMLTQuery']" ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/lst[@name='explain']/str[@name='45']" @@ -236,8 +235,7 @@ public void testMLT_debugEnabledInterestingTermsDetails_shouldReturnSimilarDocum params.set(CommonParams.DEBUG_QUERY, "true"); SolrQueryRequest mltreq = new LocalSolrQueryRequest(h.getCore(), params); assertQ("morelike this - harrison ford",mltreq - ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='rawMLTQuery']" - ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='boostedMLTQuery']" + , "//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='boostedMLTQuery']" ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='realMLTQuery']" ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/lst[@name='explain']/str[@name='45']", "//lst[@name='interestingTerms']/lst[1][count(*)>0]", @@ -267,7 +265,6 @@ public void testMLT_debugEnabledInterestingTermsList_shouldReturnSimilarDocument SolrQueryRequest mltreq = new LocalSolrQueryRequest(h.getCore(), params); assertQ("morelike this - harrison ford",mltreq - ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='rawMLTQuery']" ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='boostedMLTQuery']" ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/str[@name='realMLTQuery']" ,"//lst[@name='debug']/lst[@name='moreLikeThis']/lst[@name='44']/lst[@name='explain']/str[@name='45']", From 5d48a7777b13967f014afb935d971b0c5391955c Mon Sep 17 00:00:00 2001 From: Alessandro Benedetti Date: Thu, 3 May 2018 18:14:06 +0100 Subject: [PATCH 4/4] [SOLR-12299] precommit fixes --- .../src/java/org/apache/solr/handler/MoreLikeThisHandler.java | 3 +-- .../src/java/org/apache/solr/search/mlt/CloudMLTQParser.java | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java index c8e7913216bb..e0401306d2fb 100644 --- a/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/MoreLikeThisHandler.java @@ -23,7 +23,6 @@ import java.util.Collection; import java.util.Iterator; import java.util.List; -import java.util.Map; import java.util.regex.Pattern; import org.apache.lucene.document.Document; @@ -41,8 +40,8 @@ import org.apache.solr.common.StringUtils; import org.apache.solr.common.params.CommonParams; import org.apache.solr.common.params.FacetParams; -import org.apache.solr.common.params.MoreLikeThisParams.TermStyle; import org.apache.solr.common.params.MoreLikeThisParams; +import org.apache.solr.common.params.MoreLikeThisParams.TermStyle; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.ContentStream; import org.apache.solr.common.util.NamedList; diff --git a/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java b/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java index ae30d34dde91..af7a2ebd5fd6 100644 --- a/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java +++ b/solr/core/src/java/org/apache/solr/search/mlt/CloudMLTQParser.java @@ -30,7 +30,6 @@ import org.apache.lucene.queries.mlt.MoreLikeThis; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.BoostQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; import org.apache.lucene.util.BytesRefBuilder; @@ -47,7 +46,6 @@ import org.apache.solr.schema.SchemaField; import org.apache.solr.search.QParser; import org.apache.solr.search.QueryParsing; -import org.apache.solr.search.QueryUtils; import org.apache.solr.util.SolrPluginUtils; import static org.apache.solr.common.params.CommonParams.ID;