SOLR-17990: Add FAISS integration module for Solr

.gitignore

@@ -14,9 +14,12 @@
 
 # Ignore Gradle build output directory
 build
+bin
 out
 cluster
 
 .gatling
 
 logs
+
+faiss/test-config

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "faiss/solr"]
+	path = faiss/solr
+	url = https://github.com/apache/solr.git
+	branch = branch_10_0

faiss/README.md

@@ -0,0 +1,79 @@
+# FAISS Module for Solr
+
+FAISS integration for Apache Solr. Backported from Lucene 11.0 to work with Lucene 10.3.1.
+
+## Requirements
+
+- Java 21+
+- FAISS native library (`libfaiss_c.so` version 1.11.0)
+- Solr 9.2.0+
+
+## Building
+
+```bash
+./gradlew :faiss:jar
+```
+
+JAR is created at `faiss/build/libs/solr-faiss.jar`.
+
+## Installation
+
+Copy the JAR to Solr's lib directory:
+
+```bash
+cp faiss/build/libs/solr-faiss.jar $SOLR_HOME/server/solr-webapp/webapp/WEB-INF/lib/
+```
+
+Set `LD_LIBRARY_PATH` to include the FAISS library directory.
+
+## Configuration
+
+### solrconfig.xml
+
+```xml
+<codecFactory name="FaissCodecFactory" class="org.apache.solr.faiss.FaissCodecFactory">
+  <str name="faissDescription">IDMap,HNSW16</str>
+  <str name="faissIndexParams">efConstruction=100</str>
+</codecFactory>
+```
+
+### schema.xml
+
+```xml
+<fieldType name="faiss_vector" 
+           class="solr.DenseVectorField" 
+           vectorDimension="128" 
+           knnAlgorithm="faiss" 
+           similarityFunction="dot_product"/>
+
+<field name="vector_field" type="faiss_vector" indexed="true" stored="true"/>
+```
+
+Note: FAISS supports `dot_product` and `euclidean`, but not `cosine`.
+
+## Testing
+
+Tests require the Solr branch_10_0 submodule. Clone with `--recursive` or run:
+
+```bash
+git submodule update --init --recursive
+./gradlew :faiss:test
+```
+
+Without the submodule, tests use published Solr 9.2.0 artifacts and may fail due to Lucene version mismatch.
+
+## Git Submodule
+
+This module uses a git submodule pointing to Solr branch_10_0 for tests. Only a reference (~200 bytes) is stored, not the actual Solr code.
+
+To clone with submodule:
+```bash
+git clone <repo-url> --recursive
+```
+
+## Troubleshooting
+
+- **ClassNotFoundException** - Ensure JAR is in Solr's classpath
+- **UnsatisfiedLinkError** - Set `LD_LIBRARY_PATH` to include `libfaiss_c.so`
+- **LinkageError** - Requires exact FAISS version 1.11.0
+- **Tests fail** - Initialize submodule: `git submodule update --init --recursive`

faiss/build.gradle

@@ -0,0 +1,189 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+plugins {
+    id 'java'
+    id 'java-library'
+}
+
+description = 'FAISS plugin for Solr'
+
+repositories {
+    mavenCentral()
+}
+
+configurations {
+    provided
+}
+
+sourceSets {
+    main { compileClasspath += configurations.provided }
+}
+
+def solrSubmodulePath = new File(projectDir, 'solr')
+def solrSubmoduleExists = solrSubmodulePath.exists() && solrSubmodulePath.isDirectory()
+
+dependencies {
+    implementation "org.apache.lucene:lucene-core:10.3.1"
+    implementation "org.apache.lucene:lucene-codecs:10.3.1"
+    provided "org.apache.solr:solr-core:${solrVersion}"
+    implementation "org.slf4j:slf4j-api:2.0.5"
+
+    testImplementation "org.apache.lucene:lucene-test-framework:10.3.1"
+    testImplementation "org.apache.lucene:lucene-backward-codecs:10.3.1"
+    testImplementation "org.apache.lucene:lucene-sandbox:10.3.1"
+    testImplementation "commons-io:commons-io:2.20.0"
+    testImplementation "junit:junit:4.13.2"
+    testImplementation "org.mockito:mockito-inline:5.2.0"
+    testImplementation "org.hamcrest:hamcrest:3.0"
+
+    if (solrSubmoduleExists) {
+        testImplementation fileTree(dir: "${solrSubmodulePath}/solr/test-framework/build/libs", include: 'solr-test-framework-*.jar', excludes: ['*sources*.jar', '*javadoc*.jar'])
+        testImplementation fileTree(dir: "${solrSubmodulePath}/solr/core/build/libs", include: 'solr-core-*.jar', excludes: ['*sources*.jar', '*javadoc*.jar'])
+        testImplementation fileTree(dir: "${solrSubmodulePath}/solr/solrj/build/libs", include: 'solr-solrj-*.jar', excludes: ['*sources*.jar', '*javadoc*.jar'])
+        testImplementation fileTree(dir: "${solrSubmodulePath}/solr/api/build/libs", include: 'solr-api-*.jar', excludes: ['*sources*.jar', '*javadoc*.jar'])
+        testImplementation fileTree(dir: "${solrSubmodulePath}/solr/solrj-zookeeper/build/libs", include: 'solr-solrj-zookeeper-*.jar', excludes: ['*sources*.jar', '*javadoc*.jar'])
+
+        testImplementation("org.apache.zookeeper:zookeeper:3.9.4") {
+            exclude group: "org.apache.yetus", module: "audience-annotations"
+        }
+        testImplementation("org.apache.zookeeper:zookeeper-jute:3.9.4") {
+            exclude group: "org.apache.yetus", module: "audience-annotations"
+        }
+        testImplementation("org.apache.curator:curator-client:5.9.0") {
+            exclude group: 'org.apache.zookeeper', module: 'zookeeper'
+        }
+        testImplementation("org.apache.curator:curator-framework:5.9.0") {
+            exclude group: 'org.apache.zookeeper', module: 'zookeeper'
+        }
+        testImplementation("org.apache.curator:curator-test:5.9.0") {
+            exclude group: 'org.apache.zookeeper', module: 'zookeeper'
+            exclude group: 'com.google.guava', module: 'guava'
+            exclude group: 'io.dropwizard.metrics', module: 'metrics-core'
+        }
+        testImplementation "jakarta.servlet:jakarta.servlet-api:6.0.0"
+        testImplementation "org.eclipse.jetty:jetty-server:12.0.27"
+        testImplementation "org.eclipse.jetty:jetty-session:12.0.27"
+        testImplementation "org.eclipse.jetty.ee10:jetty-ee10-servlet:12.0.27"
+        testImplementation "org.eclipse.jetty:jetty-util:12.0.27"
+        testImplementation "org.eclipse.jetty:jetty-client:12.0.27"
+        testImplementation "org.eclipse.jetty:jetty-alpn-server:12.0.27"
+        testImplementation "org.eclipse.jetty:jetty-alpn-java-server:12.0.27"
+        testImplementation "org.eclipse.jetty:jetty-rewrite:12.0.27"
+        testImplementation "org.eclipse.jetty.http2:jetty-http2-server:12.0.27"
+        testImplementation "org.eclipse.jetty.http2:jetty-http2-common:12.0.27"
+        testImplementation "org.apache.logging.log4j:log4j-api:2.21.0"
+        testImplementation "org.apache.logging.log4j:log4j-core:2.21.0"
+        testImplementation "io.dropwizard.metrics:metrics-core:4.2.26"
+        testImplementation "io.dropwizard.metrics:metrics-jetty12-ee10:4.2.26"
+        testImplementation "commons-cli:commons-cli:1.10.0"
+        testImplementation "org.apache.httpcomponents:httpclient:4.5.14"
+        testImplementation "org.apache.httpcomponents:httpcore:4.4.16"
+        testImplementation "org.apache.httpcomponents:httpmime:4.5.14"
+        testImplementation "io.opentelemetry:opentelemetry-api:1.53.0"
+        testImplementation("io.opentelemetry:opentelemetry-exporter-prometheus:1.50.0-alpha") {
+            transitive = false
+        }
+        testImplementation "io.prometheus:prometheus-metrics-model:1.1.0"
+        testImplementation("io.opentelemetry:opentelemetry-sdk:1.53.0") {
+            exclude group: "io.opentelemetry", module: "opentelemetry-sdk-logs"
+        }
+        testImplementation("io.prometheus:prometheus-metrics-exposition-formats:1.1.0") {
+            exclude group: "io.prometheus", module: "prometheus-metrics-shaded-protobuf"
+            exclude group: "io.prometheus", module: "prometheus-metrics-config"
+        }
+        testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:2.8.3"
+    } else {
+        testImplementation group: 'org.apache.solr', name: 'solr-core', version: "${solrVersion}", {
+            exclude group: "org.eclipse.jetty", module: "jetty-http"
+            exclude group: "org.eclipse.jetty", module: "jetty-server"
+            exclude group: "org.eclipse.jetty", module: "jetty-servlet"
+        }
+        testImplementation group: 'org.apache.solr', name: 'solr-test-framework', version: "${solrVersion}"
+        testImplementation group: 'org.apache.solr', name: 'solr-solrj', version: "${solrVersion}"
+        testImplementation group: 'org.apache.solr', name: 'solr-solrj-zookeeper', version: "${solrVersion}"
+    }
+}
+
+java {
+    sourceCompatibility = JavaVersion.VERSION_21
+    targetCompatibility = JavaVersion.VERSION_21
+}
+
+def currentJavaVersion = JavaVersion.current()
+if (currentJavaVersion == JavaVersion.VERSION_21) {
+    tasks.withType(JavaCompile).configureEach {
+        options.compilerArgs += ['--enable-preview']
+    }
+}
+
+jar {
+    archiveBaseName.set('solr-faiss')
+}
+
+task buildSolrSubmodule(type: Exec) {
+    description = 'Builds Solr submodule to generate test framework JARs'
+    group = 'build'
+    onlyIf { solrSubmoduleExists }
+    workingDir = solrSubmodulePath
+    commandLine = ['sh', '-c', './gradlew :solr:test-framework:jar :solr:core:jar :solr:solrj:jar :solr:api:jar :solr:solrj-zookeeper:jar --no-daemon']
+    doFirst {
+        logger.lifecycle("Building Solr submodule at ${solrSubmodulePath.absolutePath}")
+    }
+}
+
+test.doFirst {
+    if (!solrSubmoduleExists) {
+        def separator = "=".multiply(70)
+        logger.warn("")
+        logger.warn(separator)
+        logger.warn("WARNING: Solr submodule not found at ${solrSubmodulePath.absolutePath}")
+        logger.warn("")
+        logger.warn("Tests will use published Solr 9.2.0 artifacts (Lucene 9.4.2)")
+        logger.warn("which may cause test failures due to Lucene version mismatch.")
+        logger.warn("")
+        logger.warn("To fix this, initialize the submodule:")
+        logger.warn("  git submodule update --init --recursive")
+        logger.warn("")
+        logger.warn("Or clone the repository with:")
+        logger.warn("  git clone <repo-url> --recursive")
+        logger.warn("")
+        logger.warn("Production code will still compile and work without the submodule.")
+        logger.warn(separator)
+        logger.warn("")
+    }
+}
+
+test.dependsOn buildSolrSubmodule
+compileTestJava.dependsOn buildSolrSubmodule
+
+test {
+    jvmArgs '-Djava.security.egd=file:/dev/./urandom'
+    if (currentJavaVersion == JavaVersion.VERSION_21) {
+        jvmArgs '--enable-preview'
+    }
+    if (System.getenv('LD_LIBRARY_PATH') != null) {
+        jvmArgs "-Djava.library.path=${System.getenv('LD_LIBRARY_PATH')}"
+    } else if (System.getenv('CONDA_PREFIX') != null) {
+        def condaLibPath = "${System.getenv('CONDA_PREFIX')}/lib"
+        jvmArgs "-Djava.library.path=${condaLibPath}"
+    }
+    if (solrSubmoduleExists) {
+        jvmArgs "-Dtests.src.home=${solrSubmodulePath.absolutePath}"
+    }
+    testClassesDirs = sourceSets.test.output.classesDirs
+    enabled = true
+}