Skip to content

Commit 9603aa2

Browse files
limingnihaolimingnihao
authored
SOLR-18136: fix multiThreaded=true with rerank & sort (#4164)
When multi-threaded segment-parallel search is enabled (`indexSearcherExecutorThreads > 0` and `multiThreaded=true`) and a query uses both reranking (via `RankQuery` / `ReRankCollector`) and a sort, an `ArrayStoreException` is thrown during the merge phase if some segments have matching documents and others do not.
1 parent 8873cf1 commit 9603aa2

3 files changed

Lines changed: 207 additions & 1 deletion

File tree

‎changelog/unreleased/SOLR-18136-rerank-multithreaded-arraystoreexception.yml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
title: Fix ArrayStoreException when combining rerank with sort under multi-threaded segment-parallel search
2+
type: fixed
3+
authors:
4+
- name: Shiming Li
5+
links:
6+
- name: SOLR-18136
7+
url: https://issues.apache.org/jira/browse/SOLR-18136

‎solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,7 @@ public Object reduce(Collection collectors) throws IOException {
349349
TopDocs mergedTopDocs = null;
350350

351351
if (topDocs.length > 0 && topDocs[0] != null) {
352-
if (topDocs[0] instanceof TopFieldDocs) {
352+
if (Arrays.stream(topDocs).allMatch(td -> td instanceof TopFieldDocs)) {
353353
TopFieldDocs[] topFieldDocs =
354354
Arrays.copyOf(topDocs, topDocs.length, TopFieldDocs[].class);
355355
mergedTopDocs = TopFieldDocs.merge(searcher.weightSort(cmd.getSort()), len, topFieldDocs);

‎solr/core/src/test/org/apache/solr/search/TestMultiThreadedSearcher.java‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,199 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.solr.search;
18+
19+
import java.io.IOException;
20+
import org.apache.lucene.index.Term;
21+
import org.apache.lucene.search.Explanation;
22+
import org.apache.lucene.search.IndexSearcher;
23+
import org.apache.lucene.search.Query;
24+
import org.apache.lucene.search.QueryVisitor;
25+
import org.apache.lucene.search.Rescorer;
26+
import org.apache.lucene.search.ScoreDoc;
27+
import org.apache.lucene.search.ScoreMode;
28+
import org.apache.lucene.search.Sort;
29+
import org.apache.lucene.search.SortField;
30+
import org.apache.lucene.search.TermQuery;
31+
import org.apache.lucene.search.TopDocs;
32+
import org.apache.lucene.search.TopDocsCollector;
33+
import org.apache.lucene.search.Weight;
34+
import org.apache.solr.SolrTestCaseJ4;
35+
import org.apache.solr.core.NodeConfig;
36+
import org.apache.solr.handler.component.MergeStrategy;
37+
import org.apache.solr.index.NoMergePolicyFactory;
38+
import org.apache.solr.update.UpdateShardHandlerConfig;
39+
import org.apache.solr.util.TestHarness;
40+
import org.junit.AfterClass;
41+
import org.junit.BeforeClass;
42+
43+
/** Tests for {@link MultiThreadedSearcher}. */
44+
public class TestMultiThreadedSearcher extends SolrTestCaseJ4 {
45+
46+
@BeforeClass
47+
public static void beforeClass() throws Exception {
48+
systemSetPropertySolrTestsMergePolicyFactory(NoMergePolicyFactory.class.getName());
49+
50+
NodeConfig nodeConfig =
51+
new NodeConfig.NodeConfigBuilder("testNode", TEST_PATH())
52+
.setUseSchemaCache(Boolean.getBoolean("shareSchema"))
53+
.setUpdateShardHandlerConfig(UpdateShardHandlerConfig.TEST_DEFAULT)
54+
.setIndexSearcherExecutorThreads(4)
55+
.build();
56+
createCoreContainer(
57+
nodeConfig,
58+
new TestHarness.TestCoresLocator(
59+
DEFAULT_TEST_CORENAME,
60+
createTempDir("data").toAbsolutePath().toString(),
61+
"solrconfig-minimal.xml",
62+
"schema.xml"));
63+
h.coreName = DEFAULT_TEST_CORENAME;
64+
65+
// Non-matching segments first, matching segment last.
66+
// This ensures different slices see different result counts during parallel search.
67+
for (int seg = 0; seg < 7; seg++) {
68+
for (int i = 0; i < 10; i++) {
69+
assertU(
70+
adoc(
71+
"id", String.valueOf(20000 + seg * 100 + i),
72+
"field1_s", "nomatchterm",
73+
"field4_t", "nomatchterm"));
74+
}
75+
assertU(commit());
76+
}
77+
78+
// Matching segment last
79+
for (int i = 0; i < 10; i++) {
80+
assertU(
81+
adoc(
82+
"id", String.valueOf(10000 + i),
83+
"field1_s", "xyzrareterm",
84+
"field4_t", "xyzrareterm"));
85+
}
86+
assertU(commit());
87+
}
88+
89+
@AfterClass
90+
public static void afterClass() {
91+
System.clearProperty(SYSTEM_PROPERTY_SOLR_TESTS_MERGEPOLICYFACTORY);
92+
}
93+
94+
public void testReRankWithMultiThreadedSearch() throws Exception {
95+
float fixedScore = 5.0f;
96+
h.getCore()
97+
.withSearcher(
98+
searcher -> {
99+
int numSegments = searcher.getTopReaderContext().leaves().size();
100+
assertTrue("Expected > 5 segments, got " + numSegments, numSegments > 5);
101+
assertTrue(
102+
"Expected > 1 slice, got " + searcher.getSlices().length,
103+
searcher.getSlices().length > 1);
104+
105+
final QueryCommand cmd = new QueryCommand();
106+
cmd.setFlags(SolrIndexSearcher.GET_SCORES);
107+
cmd.setLen(10);
108+
cmd.setMultiThreaded(true);
109+
cmd.setSort(
110+
new Sort(SortField.FIELD_SCORE, new SortField("id", SortField.Type.STRING)));
111+
cmd.setQuery(
112+
new SimpleReRankQuery(
113+
new TermQuery(new Term("field1_s", "xyzrareterm")), fixedScore));
114+
115+
final QueryResult qr = searcher.search(cmd);
116+
117+
assertTrue(qr.getDocList().matches() >= 1);
118+
final DocIterator iter = qr.getDocList().iterator();
119+
assertTrue(iter.hasNext());
120+
iter.next();
121+
assertEquals(fixedScore, iter.score(), 0);
122+
return null;
123+
});
124+
}
125+
126+
private static final class SimpleReRankQuery extends RankQuery {
127+
128+
private Query q;
129+
private final float reRankScore;
130+
131+
SimpleReRankQuery(Query q, float reRankScore) {
132+
this.q = q;
133+
this.reRankScore = reRankScore;
134+
}
135+
136+
@Override
137+
public Weight createWeight(IndexSearcher indexSearcher, ScoreMode scoreMode, float boost)
138+
throws IOException {
139+
return q.createWeight(indexSearcher, scoreMode, boost);
140+
}
141+
142+
@Override
143+
public void visit(QueryVisitor visitor) {
144+
q.visit(visitor);
145+
}
146+
147+
@Override
148+
public boolean equals(Object obj) {
149+
return this == obj;
150+
}
151+
152+
@Override
153+
public int hashCode() {
154+
return q.hashCode();
155+
}
156+
157+
@Override
158+
public String toString(String field) {
159+
return q.toString(field);
160+
}
161+
162+
@Override
163+
public TopDocsCollector<? extends ScoreDoc> getTopDocsCollector(
164+
int len, QueryCommand cmd, IndexSearcher searcher) throws IOException {
165+
return new ReRankCollector(
166+
len,
167+
len,
168+
new Rescorer() {
169+
@Override
170+
public TopDocs rescore(IndexSearcher searcher, TopDocs firstPassTopDocs, int topN) {
171+
for (ScoreDoc scoreDoc : firstPassTopDocs.scoreDocs) {
172+
scoreDoc.score = reRankScore;
173+
}
174+
return firstPassTopDocs;
175+
}
176+
177+
@Override
178+
public Explanation explain(
179+
IndexSearcher searcher, Explanation firstPassExplanation, int docID) {
180+
return firstPassExplanation;
181+
}
182+
},
183+
cmd,
184+
searcher,
185+
null);
186+
}
187+
188+
@Override
189+
public MergeStrategy getMergeStrategy() {
190+
return null;
191+
}
192+
193+
@Override
194+
public RankQuery wrap(Query q) {
195+
this.q = q;
196+
return this;
197+
}
198+
}
199+
}

0 commit comments

Comments
 (0)