[global-index-eslib]use thin packaging and fix pom bundle,fix query compatibility and vector metrics, and

paimon-eslib/README.md

@@ -35,6 +35,11 @@ indexed according to their data type and per-field options.
 > `es-index`. The module and its ESLib/Lucene dependencies require Java 11 or newer; the root Maven
 > build intentionally skips `paimon-eslib` when it runs on JDK 8.
 
+`paimon-eslib` is distributed as a thin JAR and does not embed ESLib or Lucene classes. Maven and
+Gradle resolve these dependencies transitively. When installing JARs manually, place `eslib-core`,
+`eslib-simdvec`, and their Lucene 9.12 dependencies in the same runtime classloader as
+`paimon-eslib`.
+
 See the general [Global Index](../docs/docs/multimodal-table/global-index.mdx) documentation for the
 required Data Evolution table properties, coverage/freshness behavior, and shared build options.
 

paimon-eslib/pom.xml

@@ -114,56 +114,11 @@ under the License.
 
     <repositories>
         <repository>
-            <id>eslib-oss</id>
-            <url>https://paimon-es-public-bucket.oss-cn-beijing.aliyuncs.com/maven/</url>
+            <id>eslib-github</id>
+            <url>https://raw.githubusercontent.com/CrownChu/es-paimon-lib-releases/main/repository</url>
             <releases><enabled>true</enabled></releases>
             <snapshots><enabled>false</enabled></snapshots>
         </repository>
     </repositories>
 
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-shade-plugin</artifactId>
-                <executions>
-                    <execution>
-                        <id>shade-eslib</id>
-                        <phase>package</phase>
-                        <goals>
-                            <goal>shade</goal>
-                        </goals>
-                        <configuration>
-                            <artifactSet>
-                                <includes>
-                                    <include>io.github.paimon.eslib:eslib-core</include>
-                                    <include>io.github.paimon.eslib:eslib-simdvec</include>
-                                    <include>org.apache.lucene:*</include>
-                                </includes>
-                            </artifactSet>
-                            <relocations>
-                                <relocation>
-                                    <pattern>org.apache.lucene</pattern>
-                                    <shadedPattern>org.apache.paimon.shade.lucene912</shadedPattern>
-                                </relocation>
-                            </relocations>
-                            <transformers>
-                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
-                            </transformers>
-                            <filters>
-                                <filter>
-                                    <artifact>*:*</artifact>
-                                    <excludes>
-                                        <exclude>module-info.class</exclude>
-                                        <exclude>META-INF/MANIFEST.MF</exclude>
-                                        <exclude>META-INF/versions/**</exclude>
-                                    </excludes>
-                                </filter>
-                            </filters>
-                        </configuration>
-                    </execution>
-                </executions>
-            </plugin>
-        </plugins>
-    </build>
 </project>

paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexReader.java

@@ -350,6 +350,19 @@ ExecutorService queryExecutor() {
         return queryExecutor;
     }
 
+    /** Returns the primary vector field's effective metric in Paimon's metric vocabulary. */
+    String primaryVectorMetric() {
+        if (fields.isEmpty()) {
+            return null;
+        }
+        String physicalField = physicalField(fields.get(0).name());
+        FieldIndexConfig config =
+                physicalField == null ? null : indexOptions.getConfig(physicalField);
+        return config != null && config.indexType() == FieldIndexConfig.IndexType.VECTOR
+                ? ESIndexOptions.toPaimonVectorMetric(config.metric())
+                : null;
+    }
+
     int openStreamCount() {
         synchronized (openProviders) {
             return openProviders.size();
@@ -440,7 +453,7 @@ public CompletableFuture<Optional<ScoredGlobalIndexResult>> visitVectorSearch(
                                                     result.count,
                                                     DEFAULT_SAMPLE_LIMIT));
                         }
-                        return toScoredResult(result, topK);
+                        return toVectorScoredResult(result, topK, config.metric());
                     } catch (IOException e) {
                         throw new RuntimeException("Vector search failed", e);
                     }
@@ -632,9 +645,13 @@ private static FullTextQuerySpec parseSpec(String field, JsonNode q, int depth)
         }
         if (q.has("match")) {
             JsonNode m = requireObject(q.get("match"), "match query");
+            // Older SQL serialization includes a logical "column" routing hint. Accept it for
+            // compatibility, but keep the already resolved physical field authoritative so column
+            // renames continue to work.
             requireOnlyFields(
                     m,
                     "match query",
+                    "column",
                     "query",
                     "terms",
                     "operator",
@@ -650,7 +667,7 @@ private static FullTextQuerySpec parseSpec(String field, JsonNode q, int depth)
                     requireObject(
                             q.has("match_phrase") ? q.get("match_phrase") : q.get("phrase"),
                             "phrase query");
-            requireOnlyFields(p, "phrase query", "query", "terms", "slop");
+            requireOnlyFields(p, "phrase query", "column", "query", "terms", "slop");
             int slop = p.has("slop") ? intValue(p.get("slop"), "slop") : 0;
             return new FullTextQuerySpec.Phrase(field, queryText(p), slop);
         } else if (q.has("boost")) {
@@ -967,6 +984,53 @@ private Optional<ScoredGlobalIndexResult> toScoredResult(SearchResult result, in
         return Optional.of(ScoredGlobalIndexResult.create(bitmap, scoreMap::get));
     }
 
+    private Optional<ScoredGlobalIndexResult> toVectorScoredResult(
+            SearchResult result, int limit, String metric) {
+        if (result == null || result.count == 0) {
+            return Optional.empty();
+        }
+
+        int count = Math.min(result.count, limit);
+        RoaringNavigableMap64 bitmap = new RoaringNavigableMap64();
+        Map<Long, Float> scoreMap = new HashMap<>(count);
+        for (int i = 0; i < count; i++) {
+            long id = result.ids[i];
+            bitmap.add(id);
+            scoreMap.put(id, toPaimonVectorScore(result.scores[i], metric));
+        }
+
+        return Optional.of(ScoredGlobalIndexResult.create(bitmap, scoreMap::get));
+    }
+
+    /** Converts Lucene's metric-specific score into Paimon's exact vector-search score. */
+    static float toPaimonVectorScore(float luceneScore, String metric) {
+        if (!Float.isFinite(luceneScore)) {
+            throw new IllegalArgumentException(
+                    "Non-finite Lucene vector score for metric '" + metric + "': " + luceneScore);
+        }
+        String normalized = metric == null ? "l2" : metric.toLowerCase(Locale.ROOT);
+        switch (normalized) {
+            case "l2":
+            case "euclidean":
+                return luceneScore;
+            case "cosine":
+            case "dot_product":
+            case "dp":
+                return 2.0f * luceneScore - 1.0f;
+            case "inner_product":
+            case "mip":
+            case "maximum_inner_product":
+                if (luceneScore <= 0.0f) {
+                    throw new IllegalArgumentException(
+                            "Lucene maximum-inner-product score must be positive; got: "
+                                    + luceneScore);
+                }
+                return luceneScore < 1.0f ? 1.0f - 1.0f / luceneScore : luceneScore - 1.0f;
+            default:
+                throw new IllegalArgumentException("Unknown ESLib vector metric: " + metric);
+        }
+    }
+
     private void checkNotClosed() throws IOException {
         if (closed) {
             throw new IOException("Reader already closed");

paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexer.java

@@ -21,12 +21,14 @@
 import org.apache.paimon.globalindex.GlobalIndexIOMeta;
 import org.apache.paimon.globalindex.GlobalIndexReader;
 import org.apache.paimon.globalindex.GlobalIndexWriter;
-import org.apache.paimon.globalindex.GlobalIndexer;
+import org.apache.paimon.globalindex.VectorGlobalIndexer;
 import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
 import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
 import org.apache.paimon.options.Options;
 import org.apache.paimon.types.DataField;
 
+import org.elasticsearch.eslib.api.model.FieldIndexConfig;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -37,14 +39,17 @@
  * ES multi-index global indexer using ESLib. Builds Lucene-based indexes supporting vector,
  * fulltext, and scalar fields.
  */
-public class ESIndexGlobalIndexer implements GlobalIndexer {
+public class ESIndexGlobalIndexer implements VectorGlobalIndexer {
 
     private final List<DataField> fields;
     private final ESIndexOptions indexOptions;
+    private final String configuredVectorMetric;
+    private volatile String readerVectorMetric;
 
     public ESIndexGlobalIndexer(List<DataField> fields, Options options) {
         this.fields = Collections.unmodifiableList(new ArrayList<>(fields));
         this.indexOptions = new ESIndexOptions(this.fields, options);
+        this.configuredVectorMetric = primaryVectorMetric(this.fields, this.indexOptions);
     }
 
     @Override
@@ -57,6 +62,50 @@ public GlobalIndexReader createReader(
             GlobalIndexFileReader fileReader,
             List<GlobalIndexIOMeta> files,
             ExecutorService executor) {
-        return new ESIndexGlobalIndexReader(fileReader, files, fields, indexOptions, executor);
+        ESIndexGlobalIndexReader reader =
+                new ESIndexGlobalIndexReader(fileReader, files, fields, indexOptions, executor);
+        try {
+            registerReaderVectorMetric(reader.primaryVectorMetric());
+            return reader;
+        } catch (RuntimeException e) {
+            try {
+                reader.close();
+            } catch (IOException closeFailure) {
+                e.addSuppressed(closeFailure);
+            }
+            throw e;
+        }
+    }
+
+    @Override
+    public String metric() {
+        String metric = readerVectorMetric;
+        return metric == null ? configuredVectorMetric : metric;
+    }
+
+    private synchronized void registerReaderVectorMetric(String metric) {
+        if (metric == null) {
+            return;
+        }
+        if (readerVectorMetric == null) {
+            readerVectorMetric = metric;
+        } else if (!readerVectorMetric.equals(metric)) {
+            throw new IllegalArgumentException(
+                    "Cannot combine es-index shards with different vector metrics: "
+                            + readerVectorMetric
+                            + " and "
+                            + metric
+                            + ".");
+        }
+    }
+
+    private static String primaryVectorMetric(List<DataField> fields, ESIndexOptions indexOptions) {
+        if (fields.isEmpty()) {
+            return null;
+        }
+        FieldIndexConfig config = indexOptions.getConfig(fields.get(0).name());
+        return config != null && config.indexType() == FieldIndexConfig.IndexType.VECTOR
+                ? ESIndexOptions.toPaimonVectorMetric(config.metric())
+                : null;
     }
 }

paimon-eslib/src/main/java/org/apache/paimon/eslib/index/ESIndexOptions.java

@@ -159,6 +159,26 @@ public FieldIndexConfig getConfig(String fieldName) {
         return fieldConfigs.get(fieldName);
     }
 
+    /** Maps an ESLib/Lucene vector metric name to Paimon's exact-search metric vocabulary. */
+    static String toPaimonVectorMetric(String metric) {
+        String normalized = metric == null ? "l2" : metric.toLowerCase(Locale.ROOT);
+        switch (normalized) {
+            case "l2":
+            case "euclidean":
+                return "l2";
+            case "cosine":
+                return "cosine";
+            case "dot_product":
+            case "dp":
+            case "inner_product":
+            case "mip":
+            case "maximum_inner_product":
+                return "inner_product";
+            default:
+                throw new IllegalArgumentException("Unknown ESLib vector metric: " + metric);
+        }
+    }
+
     /** Returns the keyword multi-field sub-field name for {@code fieldName} if one exists. */
     public String keywordSubField(String fieldName) {
         String subField = fieldName + KEYWORD_SUBFIELD_SUFFIX;

paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexFullTextQueryParserTest.java

@@ -27,6 +27,32 @@
 
 class ESIndexFullTextQueryParserTest {
 
+    @Test
+    void acceptsPersistedColumnRoutingHints() {
+        FullTextQuerySpec.Match match =
+                (FullTextQuerySpec.Match)
+                        ESIndexGlobalIndexReader.parseSpec(
+                                "physical_text",
+                                "{\"match\":{\"column\":\"logical_text\",\"terms\":\"paimon\"}}");
+        assertThat(match.field()).isEqualTo("physical_text");
+        assertThat(match.text()).isEqualTo("paimon");
+
+        FullTextQuerySpec.Phrase phrase =
+                (FullTextQuerySpec.Phrase)
+                        ESIndexGlobalIndexReader.parseSpec(
+                                "physical_text",
+                                "{\"match_phrase\":{\"column\":\"logical_text\",\"terms\":\"apache paimon\",\"slop\":1}}");
+        assertThat(phrase.field()).isEqualTo("physical_text");
+        assertThat(phrase.text()).isEqualTo("apache paimon");
+        assertThat(phrase.slop()).isEqualTo(1);
+
+        assertThat(
+                        ESIndexGlobalIndexReader.parseSpec(
+                                "physical_text",
+                                "{\"boolean\":{\"queries\":[[\"Must\",{\"match\":{\"column\":\"logical_text\",\"terms\":\"paimon\"}}],[\"Should\",{\"phrase\":{\"column\":\"logical_text\",\"terms\":\"apache paimon\"}}]]}}"))
+                .isNotNull();
+    }
+
     @Test
     void parsesCaseInsensitiveBooleanOperatorsAndRejectsTypos() {
         FullTextQuerySpec.Match and =

paimon-eslib/src/test/java/org/apache/paimon/eslib/index/ESIndexGlobalIndexE2ETest.java

@@ -1636,6 +1636,40 @@ private static GlobalIndexIOMeta ioMeta(java.nio.file.Path archiveDir, ResultEnt
                 entry.meta());
     }
 
+    @Test
+    void vectorSearchReturnsPaimonCosineScoreScale(@TempDir java.nio.file.Path tmp)
+            throws Exception {
+        List<DataField> fields =
+                List.of(new DataField(0, "embedding", DataTypes.ARRAY(DataTypes.FLOAT())));
+        Map<String, String> optionMap = new HashMap<>();
+        optionMap.put("global-index.es-index.fields.embedding.dimension", "2");
+        optionMap.put("global-index.es-index.fields.embedding.metric", "cosine");
+        ESIndexOptions options = new ESIndexOptions(fields, Options.fromMap(optionMap));
+
+        java.nio.file.Path archiveDir = tmp.resolve("cosine-scores");
+        Files.createDirectories(archiveDir);
+        ESIndexGlobalIndexWriter writer =
+                new ESIndexGlobalIndexWriter(new LocalDirWriter(archiveDir), fields, options);
+        writer.write(new float[] {1.0f, 0.0f}, 0L);
+        writer.write(new float[] {0.0f, 1.0f}, 1L);
+        ResultEntry entry = writer.finish().get(0);
+
+        ESIndexGlobalIndexReader reader =
+                new ESIndexGlobalIndexReader(
+                        new LocalFileReader(), List.of(ioMeta(archiveDir, entry)), fields, options);
+        try {
+            ScoredGlobalIndexResult result =
+                    reader.visitVectorSearch(
+                                    new VectorSearch(new float[] {1.0f, 0.0f}, 2, "embedding"))
+                            .join()
+                            .orElseThrow(AssertionError::new);
+            assertEquals(1.0f, result.scoreGetter().score(0L), 0.000001f);
+            assertEquals(0.0f, result.scoreGetter().score(1L), 0.000001f);
+        } finally {
+            reader.close();
+        }
+    }
+
     /**
      * Verifies the DiskBBQ vector codec write path through {@link ESIndexGlobalIndexWriter}.
      *