From a3f9d2fea0104fb7360129321c8d21f19813a1db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Feb 2023 09:19:51 +0100 Subject: [PATCH 01/16] Enable renovatebot --- .github/renovate.json | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 .github/renovate.json diff --git a/.github/renovate.json b/.github/renovate.json new file mode 100644 index 000000000000..a00983762ada --- /dev/null +++ b/.github/renovate.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "dependencyDashboard": false, + "enabledManagers": ["gradle"], + "ignorePaths": ["gradle/**", "solr/**", "buildSrc/**", "dev-docs/solr-missing-doclet/**"], + "postUpgradeTasks": { + "commands": ["./gradlew updateLicenses"], + "fileFilters": ["solr/licenses/*.sha1"], + "executionMode": "branch" + }, + "packageRules": [ + { + "matchDatasources": ["maven"], + "matchPackageNames": ["commons-collections:commons-collections", "commons-io:commons-io", "commons-lang:commons-lang"], + "versioning": "regex:^(?\\d{1,4})\\.(?\\d+)(\\.(?\\d+))?$" + }, + { + "matchPackagePrefixes": ["software.amazon.awssdk"], + "extends": ["schedule:monthly"] + }, + { + "matchPackageNames": ["solr:modules", "HH:mm"], + "enabled": false + } + ], + "schedule": [ + "every sunday" + ], + "prConcurrentLimit": 5 +} From ec1873c6a1ee029889bae3d4dc4058310c2fe6fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Feb 2023 13:03:31 +0100 Subject: [PATCH 02/16] Add PR label `dependencies` for each PR opened. --- .github/renovate.json | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/renovate.json b/.github/renovate.json index a00983762ada..c14bbfef292b 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -26,5 +26,6 @@ "schedule": [ "every sunday" ], + "labels": ["dependencies"], "prConcurrentLimit": 5 } From 27fe6b4ea038d8b5dc70a2e6cedb0456f5ef6818 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Feb 2023 10:59:00 +0100 Subject: [PATCH 03/16] SOLR-16660 Deprecate analytics component (#1361) --- solr/CHANGES.txt | 2 ++ .../org/apache/solr/handler/component/AnalyticsComponent.java | 4 ++++ .../pages/requesthandlers-searchcomponents.adoc | 2 +- .../query-guide/pages/analytics-expression-sources.adoc | 2 ++ .../query-guide/pages/analytics-mapping-functions.adoc | 2 ++ .../query-guide/pages/analytics-reduction-functions.adoc | 2 ++ solr/solr-ref-guide/modules/query-guide/pages/analytics.adoc | 2 ++ .../modules/upgrade-notes/pages/major-changes-in-solr-9.adoc | 1 + 8 files changed, 16 insertions(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 71363745cee7..051438bbdbc3 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -299,6 +299,8 @@ Other Changes * SOLR-16573: Introduce EmbeddedSolrServerTestRule, a JUnit 4 TestRule for Solr testing (Joshua Ouma, David Smiley) +* SOLR-16660: Deprecate analytics component (janhoy) + * SOLR-15733: Separate out a solrj-streaming module (Joel Bernstein, Kevin Risden) * SOLR-15928: Dim 'Add Collection' and 'Create Alias' buttons when user lacks proper permissions (janhoy) diff --git a/solr/modules/analytics/src/java/org/apache/solr/handler/component/AnalyticsComponent.java b/solr/modules/analytics/src/java/org/apache/solr/handler/component/AnalyticsComponent.java index 17b197216f42..c93ebf51de36 100644 --- a/solr/modules/analytics/src/java/org/apache/solr/handler/component/AnalyticsComponent.java +++ b/solr/modules/analytics/src/java/org/apache/solr/handler/component/AnalyticsComponent.java @@ -26,14 +26,18 @@ import org.apache.solr.analytics.util.OldAnalyticsParams; import org.apache.solr.analytics.util.OldAnalyticsRequestConverter; import org.apache.solr.common.util.NamedList; +import org.apache.solr.logging.DeprecationLog; import org.apache.solr.response.SolrQueryResponse; /** Computes analytics requests. */ +@Deprecated(since = "9.2") public class AnalyticsComponent extends SearchComponent { public static final String COMPONENT_NAME = "analytics"; @Override public void init(NamedList args) { + DeprecationLog.log( + "analytics", "The Analytics component is deprecated. See upgrade notes for more."); AnalyticsRequestParser.init(); } diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/requesthandlers-searchcomponents.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/requesthandlers-searchcomponents.adoc index 8a3144c6c615..08bf0d27f312 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/requesthandlers-searchcomponents.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/requesthandlers-searchcomponents.adoc @@ -242,7 +242,7 @@ They are called in the order listed. Apart from default components, Solr ships with a number of additional - very useful - components. They do need to defined and referenced in `solrconfig.xml` to be actually used. -* `AnalyticsComponent`, described in the section xref:query-guide:analytics.adoc[]. +* `AnalyticsComponent`, described in the section xref:query-guide:analytics.adoc[] (Deprecated). * `ClusteringComponent`, described in the section xref:query-guide:result-clustering.adoc[]. * `PhrasesIdentificationComponent`, used to identify & score "phrases" found in the input string, based on shingles in indexed fields, described in the {solr-javadocs}/core/org/apache/solr/handler/component/PhrasesIdentificationComponent.html[PhrasesIdentificationComponent] javadocs. * `QueryElevationComponent`, described in the section xref:query-guide:query-elevation-component.adoc[]. diff --git a/solr/solr-ref-guide/modules/query-guide/pages/analytics-expression-sources.adoc b/solr/solr-ref-guide/modules/query-guide/pages/analytics-expression-sources.adoc index 267762b567a0..7b535d558e9e 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/analytics-expression-sources.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/analytics-expression-sources.adoc @@ -18,6 +18,8 @@ Expression sources are the source of the data being aggregated in xref:analytics.adoc#expressions[analytics expressions]. +WARNING: The analytics component is deprecated. We recommend looking into similar functionality found in xref:json-facet-api.adoc[JSON Facet API]. Please notify the project if there's functionality you need that isn't currently covered by JSON facets. + These sources can be either Solr fields indexed with docValues, or constants. == Supported Field Types diff --git a/solr/solr-ref-guide/modules/query-guide/pages/analytics-mapping-functions.adoc b/solr/solr-ref-guide/modules/query-guide/pages/analytics-mapping-functions.adoc index b1a2e7b62ec7..8c0c867aeb16 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/analytics-mapping-functions.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/analytics-mapping-functions.adoc @@ -18,6 +18,8 @@ Mapping functions map values for each Solr Document or Reduction. +WARNING: The analytics component is deprecated. We recommend looking into similar functionality found in xref:json-facet-api.adoc[JSON Facet API]. Please notify the project if there's functionality you need that isn't currently covered by JSON facets. + Below is a list of all mapping functions provided by the Analytics Component. These mappings can be chained together to implement more complex functionality. diff --git a/solr/solr-ref-guide/modules/query-guide/pages/analytics-reduction-functions.adoc b/solr/solr-ref-guide/modules/query-guide/pages/analytics-reduction-functions.adoc index 25255deabf5f..946cbd4bea5e 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/analytics-reduction-functions.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/analytics-reduction-functions.adoc @@ -21,6 +21,8 @@ Reduction functions reduce the values of xref:analytics-expression-sources.adoc[ and/or unreduced xref:analytics-mapping-functions.adoc[mapping functions] for every Solr Document to a single value. +WARNING: The analytics component is deprecated. We recommend looking into similar functionality found in xref:json-facet-api.adoc[JSON Facet API]. Please notify the project if there's functionality you need that isn't currently covered by JSON facets. + Below is a list of all reduction functions provided by the Analytics Component. These can be combined using mapping functions to implement more complex functionality. diff --git a/solr/solr-ref-guide/modules/query-guide/pages/analytics.adoc b/solr/solr-ref-guide/modules/query-guide/pages/analytics.adoc index c2f0fedf646a..5369bed88071 100644 --- a/solr/solr-ref-guide/modules/query-guide/pages/analytics.adoc +++ b/solr/solr-ref-guide/modules/query-guide/pages/analytics.adoc @@ -20,6 +20,8 @@ The Analytics Component allows users to calculate complex statistical aggregations over result sets. +WARNING: The analytics component is deprecated. We recommend looking into similar functionality found in xref:json-facet-api.adoc[JSON Facet API]. Please notify the project if there's functionality you need that isn't currently covered by JSON facets. + The component enables interacting with data in a variety of ways, both through a diverse set of analytics functions as well as powerful faceting functionality. The standard facets are supported within the analytics component with additions that leverage its analytical capabilities. diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc index a9983fc1a851..b8c9970357f0 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc @@ -67,6 +67,7 @@ Due to changes in Lucene 9, that isn't possible any more. == Solr 9.2 === Upgrade to Jetty 10.x * Solr upgraded to Jetty 10.x from 9.x due to Jetty 9.x is now end of life. Jetty 10.x has a Java 11 minimum and matches Solr 9 minimum Java version. Jetty logging has been replaced with slf4j again matching Solr. See https://webtide.com/jetty-10-and-11-have-arrived/ for additional Jetty 10.x highlights. +* The xref:query-guide:analytics.adoc[Analytics Component] has been deprecated. Consider using xref:query-guide:json-facet-api.adoc[JSON Facet API] as a substitute. Please notify the project if there's functionality you need that isn't currently covered by JSON facets. === Jetty Configuration * Solr no longer duplicates certain Jetty "server" library dependencies between `server/lib` and `WEB-INF/lib` (jetty-util, jetty-io, etc.). From 6726f79d09f1e5c21f9305801d85531ab7bb765d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Feb 2023 11:03:26 +0100 Subject: [PATCH 04/16] SOLR-15959 Deprecate loading of solr.xml from ZK (#1362) --- solr/CHANGES.txt | 2 ++ solr/core/src/java/org/apache/solr/core/NodeConfig.java | 4 ++++ .../configuration-guide/pages/configuring-solr-xml.adoc | 3 +++ .../deployment-guide/pages/upgrading-a-solr-cluster.adoc | 2 +- .../deployment-guide/pages/zookeeper-file-management.adoc | 6 +++--- .../modules/getting-started/pages/tutorial-solrcloud.adoc | 2 -- .../upgrade-notes/pages/major-changes-in-solr-9.adoc | 1 + 7 files changed, 14 insertions(+), 6 deletions(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 051438bbdbc3..f132187e6c5e 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -296,6 +296,8 @@ Other Changes * SOLR-15955: Upgrade to Jetty 10.x (Mark Miller, Kevin Risden) +* SOLR-15959: Deprecate loading of solr.xml from Zookeeper (janhoy) + * SOLR-16573: Introduce EmbeddedSolrServerTestRule, a JUnit 4 TestRule for Solr testing (Joshua Ouma, David Smiley) diff --git a/solr/core/src/java/org/apache/solr/core/NodeConfig.java b/solr/core/src/java/org/apache/solr/core/NodeConfig.java index 398596dc2fd6..6eb0a77978c5 100644 --- a/solr/core/src/java/org/apache/solr/core/NodeConfig.java +++ b/solr/core/src/java/org/apache/solr/core/NodeConfig.java @@ -37,6 +37,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.logging.DeprecationLog; import org.apache.solr.logging.LogWatcherConfig; import org.apache.solr.servlet.SolrDispatchFilter; import org.apache.solr.update.UpdateShardHandlerConfig; @@ -218,6 +219,9 @@ public static NodeConfig loadNodeConfig(Path solrHome, Properties nodeProperties .build()) { if (zkClient.exists("/solr.xml", true)) { log.info("solr.xml found in ZooKeeper. Loading..."); + DeprecationLog.log( + "solrxml-zookeeper", + "Loading solr.xml from zookeeper is deprecated. See reference guide for details."); byte[] data = zkClient.getData("/solr.xml", null, null, true); return SolrXmlConfig.fromInputStream( solrHome, new ByteArrayInputStream(data), nodeProperties, true); diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/configuring-solr-xml.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/configuring-solr-xml.adoc index 39d98f5d5417..5de1e855031b 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/configuring-solr-xml.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/configuring-solr-xml.adoc @@ -25,6 +25,9 @@ For details on how to configure `core.properties`, see the section xref:core-dis You can find `solr.xml` in your `$SOLR_HOME` directory (usually `server/solr` or `/var/solr/data`) or optionally in ZooKeeper when using SolrCloud. If `$SOLR_HOME/solr.xml` is not found, Solr will use the default `solr.xml` file. +WARNING: Loading `solr.xml` from Zookeeper is deprecated, and will not be supported in a future version. +Being the node config of Solr, this file must be available at early startup and also be allowed to differ between nodes. + The default `solr.xml` file is found in `$SOLR_TIP/server/solr/solr.xml` and looks like this: [source,xml] diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/upgrading-a-solr-cluster.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/upgrading-a-solr-cluster.adoc index 2fb214ee4aaa..c3d582193424 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/upgrading-a-solr-cluster.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/upgrading-a-solr-cluster.adoc @@ -39,7 +39,7 @@ You'll be doing a rolling restart of your cluster (each node, one-by-one), but w * `ZK_HOST`: The ZooKeeper connection string your current SolrCloud nodes use to connect to ZooKeeper; this value will be the same for all nodes in the cluster. * `SOLR_HOST`: The hostname each Solr node used to register with ZooKeeper when joining the SolrCloud cluster; this value will be used to set the *host* Java system property when starting the new Solr process. * `SOLR_PORT`: The port each Solr node is listening on, such as 8983. -* `SOLR_HOME`: The absolute path to the Solr home directory for each Solr node; this directory must contain a `solr.xml` file. +* `SOLR_HOME`: The absolute path to the Solr home directory for each Solr node. This value will be passed to the new Solr process using the `solr.solr.home` system property, see: xref:configuration-guide:configuring-solr-xml.adoc[]. + If you are upgrading from an installation of Solr 5.x or later, these values can typically be found in either `/var/solr/solr.in.sh` or `/etc/default/solr.in.sh`. diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/zookeeper-file-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/zookeeper-file-management.adoc index fa5ee7c38c0b..a636ad6aaccb 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/zookeeper-file-management.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/zookeeper-file-management.adoc @@ -85,13 +85,13 @@ Please see xref:taking-solr-to-production.adoc#zookeeper-chroot[ZooKeeper chroot There are certain configuration files containing cluster wide configuration. Since some of these are crucial for the cluster to function properly, you may need to upload such files to ZooKeeper before starting your Solr cluster for the first time. -Examples of such configuration files (not exhaustive) are `solr.xml`, `security.json` and `clusterprops.json`. +Examples of such configuration files (not exhaustive) are `security.json` and `clusterprops.json`. -If you for example would like to keep your `solr.xml` in ZooKeeper to avoid having to copy it to every node's `solr_home` directory, you can push it to ZooKeeper with the bin/solr utility (Unix example): +If you for example would like to enable authentication, you can push your `security.json` file to ZooKeeper with the bin/solr utility (Unix example): [source,bash] ---- -bin/solr zk cp file:local/file/path/to/solr.xml zk:/solr.xml -z localhost:2181 +bin/solr zk cp file:local/file/path/to/security.json zk:/security.json -z localhost:2181 ---- NOTE: If you have defined `ZK_HOST` in `solr.in.sh`/`solr.in.cmd` (see xref:zookeeper-ensemble.adoc#updating-solr-include-files,Updating Solr Include Files>>) you can omit `-z ` from the above command. diff --git a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-solrcloud.adoc b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-solrcloud.adoc index 83f9c2ab6673..7a68b1ead628 100644 --- a/solr/solr-ref-guide/modules/getting-started/pages/tutorial-solrcloud.adoc +++ b/solr/solr-ref-guide/modules/getting-started/pages/tutorial-solrcloud.adoc @@ -200,14 +200,12 @@ bin/solr start -cloud -s solr.home/solr -p -z ---- Notice that the above requires you to create a Solr home directory. -You either need to copy `solr.xml` to the `solr_home` directory, or keep in centrally in ZooKeeper `/solr.xml`. Example (with directory structure) that adds a node to an example started with "bin/solr -e cloud": [source,bash] ---- mkdir -p example/cloud/node3/solr -cp server/solr/solr.xml example/cloud/node3/solr bin/solr start -cloud -s example/cloud/node3/solr -p 8987 -z localhost:9983 ---- diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc index b8c9970357f0..0fe8d6e7cc76 100644 --- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc +++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc @@ -67,6 +67,7 @@ Due to changes in Lucene 9, that isn't possible any more. == Solr 9.2 === Upgrade to Jetty 10.x * Solr upgraded to Jetty 10.x from 9.x due to Jetty 9.x is now end of life. Jetty 10.x has a Java 11 minimum and matches Solr 9 minimum Java version. Jetty logging has been replaced with slf4j again matching Solr. See https://webtide.com/jetty-10-and-11-have-arrived/ for additional Jetty 10.x highlights. +* Loading solr.xml from Zookeeper is deprecated. See xref:configuration-guide:configuring-solr-xml.adoc[Configuring solr.xml]. * The xref:query-guide:analytics.adoc[Analytics Component] has been deprecated. Consider using xref:query-guide:json-facet-api.adoc[JSON Facet API] as a substitute. Please notify the project if there's functionality you need that isn't currently covered by JSON facets. === Jetty Configuration From 62c2bab5f2ed1be00a7d1d7ace222a9a423aa082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Feb 2023 11:34:50 +0100 Subject: [PATCH 05/16] SOLR-16650, SOLR-16532: OTEL tracer additional tags (#1342) --- .../opentelemetry/OtelTracerConfigurator.java | 23 +++++++++++++++++++ .../OtelTracerConfiguratorTest.java | 8 +++++++ .../pages/distributed-tracing.adoc | 7 ++++++ 3 files changed, 38 insertions(+) diff --git a/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java b/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java index b6797df3313d..6b876c7c1b63 100644 --- a/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java +++ b/solr/modules/opentelemetry/src/java/org/apache/solr/opentelemetry/OtelTracerConfigurator.java @@ -21,6 +21,7 @@ import io.opentelemetry.sdk.autoconfigure.AutoConfiguredOpenTelemetrySdk; import io.opentracing.Tracer; import java.lang.invoke.MethodHandles; +import java.util.Arrays; import java.util.HashMap; import java.util.Locale; import java.util.Map; @@ -46,6 +47,7 @@ public Tracer getTracer() { setDefaultIfNotConfigured("OTEL_TRACES_EXPORTER", "otlp"); setDefaultIfNotConfigured("OTEL_EXPORTER_OTLP_PROTOCOL", "grpc"); setDefaultIfNotConfigured("OTEL_TRACES_SAMPLER", "parentbased_always_on"); + addOtelResourceAttributes(Map.of("host.name", System.getProperty("host"))); final String currentConfig = getCurrentOtelConfigAsString(); log.info("OpenTelemetry tracer enabled with configuration: {}", currentConfig); @@ -66,6 +68,27 @@ public Tracer getTracer() { return new ClosableTracerShim(shim, otelSdk.getSdkTracerProvider()); } + /** + * Add explicit tags statically to all traces, independent of request. Attributes with same name + * supplied in ENV or SysProp will take precedence over attributes added in code. + */ + void addOtelResourceAttributes(Map attrsToAdd) { + String commaSepAttrs = getEnvOrSysprop("OTEL_RESOURCE_ATTRIBUTES"); + Map attrs = new HashMap<>(attrsToAdd); + if (commaSepAttrs != null) { + attrs.putAll( + Arrays.stream(commaSepAttrs.split(",")) + .filter(e -> e.contains("=")) + .map(e -> e.strip().split("=")) + .collect(Collectors.toMap(kv -> kv[0], kv -> kv[1]))); + } + System.setProperty( + envNameToSyspropName("OTEL_RESOURCE_ATTRIBUTES"), + attrs.entrySet().stream() + .map(e -> e.getKey() + "=" + e.getValue()) + .collect(Collectors.joining(","))); + } + /** Prepares a string with all configuration K/V pairs sorted and semicolon separated */ String getCurrentOtelConfigAsString() { return getCurrentOtelConfig().entrySet().stream() diff --git a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.java b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.java index 4de4eca64ce6..54b89d1d5210 100644 --- a/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.java +++ b/solr/modules/opentelemetry/src/test/org/apache/solr/opentelemetry/OtelTracerConfiguratorTest.java @@ -18,6 +18,7 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; import io.opentracing.util.GlobalTracer; +import java.util.List; import java.util.Map; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.cloud.MiniSolrCloudCluster; @@ -93,6 +94,8 @@ public void testSetDefaultIfNotConfigured() { @Test public void testInjected() throws Exception { + System.setProperty("otel.resource.attributes", "foo=bar,ILLEGAL-LACKS-VALUE,"); + System.setProperty("host", "my.solr.host"); // Make sure the batch exporter times out before our thread lingering time of 10s instance.setDefaultIfNotConfigured("OTEL_BSP_SCHEDULE_DELAY", "1000"); instance.setDefaultIfNotConfigured("OTEL_BSP_EXPORT_TIMEOUT", "2000"); @@ -105,8 +108,13 @@ public void testInjected() throws Exception { assertTrue( "Tracer shim not registered with GlobalTracer", GlobalTracer.get().toString().contains("ClosableTracerShim")); + assertEquals( + List.of("host.name=my.solr.host", "foo=bar"), + List.of(System.getProperty("otel.resource.attributes").split(","))); } finally { cluster.shutdown(); + System.clearProperty("otel.resource.attributes"); + System.clearProperty("host"); } } } diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adoc index 2dd013ff1d73..e51d7fbed150 100644 --- a/solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adoc +++ b/solr/solr-ref-guide/modules/deployment-guide/pages/distributed-tracing.adoc @@ -88,6 +88,13 @@ An equivalent configuration using system properties would be: SOLR_OPTS=-Dotel.exporter.otlp.endpoint=my-remote-collector:4317 -Dotel.traces.sampler=parentbased_traceidratio -Dotel.traces.sampler.arg=0.1 ---- +To add custom tags to the trace, use `OTEL_RESOURCE_ATTRIBUTES`: + +[source,bash] +---- +OTEL_RESOURCE_ATTRIBUTES="application=OnlineBanking,exampleKey=exampleValue" +---- + Consult the https://opentelemetry.io/docs/reference/specification/sdk-environment-variables/[OTEL documentation] for details on all the configuration options. This version of Solr uses https://github.com/open-telemetry/opentelemetry-java/tree/v{dep-version-opentelemetry}[OpenTelemetry SDK v{dep-version-opentelemetry}]. From e2733825bf64c0a51736cd712a91de2d8ff045db Mon Sep 17 00:00:00 2001 From: Kevin Risden Date: Wed, 22 Feb 2023 09:18:07 -0500 Subject: [PATCH 06/16] SOLR-16664: TestCoordinatorRole fails docs is null (#1363) * Add assertion with more details on failure --- .../solr/search/TestCoordinatorRole.java | 138 ++++++++---------- 1 file changed, 63 insertions(+), 75 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java b/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java index 8ea9781e98fc..888a05c63764 100644 --- a/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java +++ b/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java @@ -32,9 +32,10 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Consumer; +import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.impl.HttpSolrClient; +import org.apache.solr.client.solrj.impl.Http2SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.UpdateRequest; @@ -47,10 +48,9 @@ import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.SolrNamedThreadFactory; -import org.apache.solr.common.util.Utils; import org.apache.solr.core.NodeRoles; import org.apache.solr.embedded.JettySolrRunner; import org.apache.solr.servlet.CoordinatorHttpSolrCall; @@ -60,8 +60,6 @@ public class TestCoordinatorRole extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final long DEFAULT_TOLERANCE = 500; - public void testSimple() throws Exception { MiniSolrCloudCluster cluster = configureCluster(4).addConfig("conf", configset("cloud-minimal")).configure(); @@ -84,7 +82,7 @@ public void testSimple() throws Exception { assertEquals(10, rsp.getResults().getNumFound()); System.setProperty(NodeRoles.NODE_ROLES_PROP, "coordinator:on"); - JettySolrRunner coordinatorJetty = null; + final JettySolrRunner coordinatorJetty; try { coordinatorJetty = cluster.startJettySolrRunner(); } finally { @@ -153,17 +151,17 @@ public void testNRTRestart() throws Exception { } assertNotNull(nrtJetty); assertNotNull(pullJetty); - try (HttpSolrClient client = (HttpSolrClient) pullJetty.newClient()) { + try (SolrClient client = pullJetty.newClient()) { client.add(COLL, sid); client.commit(COLL); assertEquals( nrtCore, getHostCoreName( - COLL, qaJettyBase, client, p -> p.add("shards.preference", "replica.type:NRT"))); + COLL, qaJettyBase, p -> p.add(ShardParams.SHARDS_PREFERENCE, "replica.type:NRT"))); assertEquals( pullCore, getHostCoreName( - COLL, qaJettyBase, client, p -> p.add("shards.preference", "replica.type:PULL"))); + COLL, qaJettyBase, p -> p.add(ShardParams.SHARDS_PREFERENCE, "replica.type:PULL"))); // Now , kill NRT jetty JettySolrRunner nrtJettyF = nrtJetty; JettySolrRunner pullJettyF = pullJetty; @@ -178,8 +176,7 @@ public void testNRTRestart() throws Exception { executor.submit( () -> { // we manipulate the jetty instances in a separate thread to more closely mimic - // the behavior we'd - // see irl. + // the behavior we'd see irl. try { Thread.sleep(establishBaselineMs); log.info("stopping NRT jetty ..."); @@ -190,10 +187,8 @@ public void testNRTRestart() throws Exception { nrtJettyF.start(true); log.info("NRT jetty restarted."); // once NRT is back up, we expect PULL to continue serving until the TTL on ZK - // state - // used for query request routing has expired (60s). But here we force a return - // to NRT - // by stopping the PULL replica after a brief delay ... + // state used for query request routing has expired (60s). But here we force a + // return to NRT by stopping the PULL replica after a brief delay ... Thread.sleep(pullServiceTimeMs); log.info("stopping PULL jetty ..."); pullJettyF.stop(); @@ -211,8 +206,7 @@ public void testNRTRestart() throws Exception { getHostCoreName( COLL, qaJettyBase, - client, - p -> p.add("shards.preference", "replica.type:NRT")))) { + p -> p.add(ShardParams.SHARDS_PREFERENCE, "replica.type:NRT")))) { count++; individualRequestStart = new Date().getTime(); } @@ -224,11 +218,9 @@ public void testNRTRestart() throws Exception { establishBaselineMs, now - individualRequestStart); // default tolerance of 500ms below should suffice. Failover to PULL for this case should be - // very fast, - // because our QA-based client already knows both replicas are active, the index is stable, - // so the moment - // the client finds NRT is down it should be able to failover immediately and transparently - // to PULL. + // very fast, because our QA-based client already knows both replicas are active, the index + // is stable, so the moment the client finds NRT is down it should be able to failover + // immediately and transparently to PULL. assertEquals( "when we break out of the NRT query loop, should be b/c routed to PULL", pullCore, @@ -261,14 +253,11 @@ public void testNRTRestart() throws Exception { nrtDowntimeMs, count); // NRT replica is back up, registered as available with Zk, and availability info has been - // pulled down by - // our PULL-replica-based `client`, forwarded indexing command to NRT, index/commit - // completed. All of this - // accounts for the 3000ms tolerance allowed for below. This is not a strict value, and if - // it causes failures - // regularly we should feel free to increase the tolerance; but it's meant to provide a - // stable baseline from - // which to detect regressions. + // pulled down by our PULL-replica-based `client`, forwarded indexing command to NRT, + // index/commit completed. All of this accounts for the 3000ms tolerance allowed for below. + // This is not a strict value, and if it causes failures regularly we should feel free to + // increase the tolerance; but it's meant to provide a stable baseline from which to detect + // regressions. count = 0; start = new Date().getTime(); individualRequestStart = start; @@ -277,10 +266,9 @@ public void testNRTRestart() throws Exception { getHostCoreName( COLL, qaJettyBase, - client, p -> { p.set(CommonParams.Q, "id:345"); - p.add("shards.preference", "replica.type:NRT"); + p.add(ShardParams.SHARDS_PREFERENCE, "replica.type:NRT"); }))) { count++; Thread.sleep(100); @@ -296,7 +284,6 @@ public void testNRTRestart() throws Exception { assertEquals(nrtCore, hostCore); // allow any exceptions to propagate jettyManipulationFuture.get(); - if (true) return; // next phase: just toggle a bunch // TODO: could separate this out into a different test method, but this should suffice for @@ -338,16 +325,16 @@ public void testNRTRestart() throws Exception { start = new Date().getTime(); try { do { - pullCore.equals( - hostCore = - getHostCoreName( - COLL, - qaJettyBase, - client, - p -> { - p.set(CommonParams.Q, "id:345"); - p.add("shards.preference", "replica.type:NRT"); - })); + if (pullCore.equals( + getHostCoreName( + COLL, + qaJettyBase, + p -> { + p.set(CommonParams.Q, "id:345"); + p.add(ShardParams.SHARDS_PREFERENCE, "replica.type:NRT"); + }))) { + done.set(true); + } count++; Thread.sleep(100); } while (!done.get()); @@ -380,44 +367,45 @@ public void testNRTRestart() throws Exception { } } - @SuppressWarnings("rawtypes") - private String getHostCoreName( - String COLL, String qaNode, HttpSolrClient solrClient, Consumer p) + private String getHostCoreName(String COLL, String qaNode, Consumer p) throws Exception { - boolean found = false; - SolrQuery q = new SolrQuery("*:*"); - q.add("fl", "id,desc_s,_core_:[core]").add(OMIT_HEADER, TRUE); + SolrQuery q = + new SolrQuery( + CommonParams.Q, + "*:*", + CommonParams.FL, + "id,desc_s,_core_:[core]", + OMIT_HEADER, + TRUE, + CommonParams.WT, + CommonParams.JAVABIN); p.accept(q); - StringBuilder sb = - new StringBuilder(qaNode).append("/").append(COLL).append("/select?wt=javabin"); - q.forEach(e -> sb.append("&").append(e.getKey()).append("=").append(e.getValue()[0])); SolrDocumentList docs = null; - for (int i = 0; i < 100; i++) { - try { - SimpleOrderedMap rsp = - (SimpleOrderedMap) - Utils.executeGET(solrClient.getHttpClient(), sb.toString(), Utils.JAVABINCONSUMER); - docs = (SolrDocumentList) rsp.get("response"); - if (docs.size() > 0) { - found = true; - break; - } - } catch (SolrException ex) { - // we know we're doing tricky things that might cause transient errors - // TODO: all these query requests go to the QA node -- should QA propagate internal request - // errors - // to the external client (and the external client retry?) or should QA attempt to failover - // transparently - // in the event of an error? - if (i < 5) { - log.info("swallowing transient error", ex); - } else { - log.error("only expect actual _errors_ within a small window (e.g. 500ms)", ex); - fail("initial error time threshold exceeded"); + try (SolrClient solrClient = new Http2SolrClient.Builder(qaNode).build()) { + for (int i = 0; i < 100; i++) { + try { + QueryResponse queryResponse = solrClient.query(COLL, q); + docs = queryResponse.getResults(); + assertNotNull("Docs should not be null. Query response was: " + queryResponse, docs); + if (docs.size() > 0) { + found = true; + break; + } + } catch (SolrException ex) { + // we know we're doing tricky things that might cause transient errors + // TODO: all these query requests go to the QA node -- should QA propagate internal + // request errors to the external client (and the external client retry?) or should QA + // attempt to failover transparently in the event of an error? + if (i < 5) { + log.info("swallowing transient error", ex); + } else { + log.error("only expect actual _errors_ within a small window (e.g. 500ms)", ex); + fail("initial error time threshold exceeded"); + } } + Thread.sleep(100); } - Thread.sleep(100); } assertTrue(found); return (String) docs.get(0).getFieldValue("_core_"); From ea722f139a0d6468449ea11f9f52a5df6ee58eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Feb 2023 15:18:25 +0100 Subject: [PATCH 07/16] Change initial schedule to a few hours every day Add an hourly limit of 5, for slower on-boarding of PRs Add stability requirement of 5 days since release of a version before PR creation --- .github/renovate.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/renovate.json b/.github/renovate.json index c14bbfef292b..071de71105fa 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -24,8 +24,10 @@ } ], "schedule": [ - "every sunday" + "* 8-20 * * *" ], "labels": ["dependencies"], - "prConcurrentLimit": 5 + "prConcurrentLimit": 5, + "prHourlyLimit": 5, + "stabilityDays": 5 } From bf04922c51af4c87b75fb018b02cca6b494ee4ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Wed, 22 Feb 2023 22:23:34 +0100 Subject: [PATCH 08/16] Include only the two build files we care about --- .github/renovate.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/renovate.json b/.github/renovate.json index 071de71105fa..f56bc585eb77 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -2,7 +2,7 @@ "$schema": "https://docs.renovatebot.com/renovate-schema.json", "dependencyDashboard": false, "enabledManagers": ["gradle"], - "ignorePaths": ["gradle/**", "solr/**", "buildSrc/**", "dev-docs/solr-missing-doclet/**"], + "includePaths": ["versions.props", "build.gradle"], "postUpgradeTasks": { "commands": ["./gradlew updateLicenses"], "fileFilters": ["solr/licenses/*.sha1"], From ad4886d31f47f1c5a521e46c2873b6babdaadd38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 23 Feb 2023 10:48:48 +0100 Subject: [PATCH 09/16] Add dev docs explaining the bot --- .github/renovate.json | 1 + dev-docs/dependency-upgrades.adoc | 64 +++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) create mode 100644 dev-docs/dependency-upgrades.adoc diff --git a/.github/renovate.json b/.github/renovate.json index f56bc585eb77..b0b8b9a24c47 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -1,5 +1,6 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "enabled": true, "dependencyDashboard": false, "enabledManagers": ["gradle"], "includePaths": ["versions.props", "build.gradle"], diff --git a/dev-docs/dependency-upgrades.adoc b/dev-docs/dependency-upgrades.adoc new file mode 100644 index 000000000000..0cb8d20185cb --- /dev/null +++ b/dev-docs/dependency-upgrades.adoc @@ -0,0 +1,64 @@ += Dependency upgrades +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +Solr has lots of 3rd party dependencies, defined mainly in `versions.props`. +Keeping them up-to-date is crucial for a number of reasons: + +* minimizing the risk of critical CVE vulnerabilities by staying on a recent and supported version +* avoiding "dependency hell", that can arise from falling too far behind + +Read the `help/dependencies.txt` file for an in-depth explanation of how gradle is deployed in Solr, using +https://github.com/palantir/gradle-consistent-versions[Gradle consistent-versions] plugin. + +== Manual dependency upgrades +In order to upgrade a dependency, you need to run through a number of steps + +1. Identify the available versions from e.g. https://search.maven.org[Maven Central] +2. Update the version in `versions.props` file +3. Run `./gradlew --write-locks` to re-generate `versions.lock`. Note that this may cause a cascading effect where + the locked version of other dependencies also change. +4. Run `./gradlew updateLicenses` to re-generate SHA1 checksums of the new jar files. +5. Once in a while, a new version of a dependency will transitively bring in brand-new dependencies. + You'll need to decide whether to keep or exclude them. See `help/dependencies.txt` for details. + +== Renovate bot Pull Requests +A member of the Solr community operates a Github bot running https://github.com/renovatebot/renovate[Renovate], which +files Pull Requests to Solr with dependency upgrade proposals. The PRs are labeled `dependencies` and include +changes resulting from `--write-locks` and `updateLicenses`. + +Community members and committers can then review, and if manual changes are needed, help bring the PR to completion. +For many dependencies, a changelog is included in the PR text, which may help guide the upgrade decision. + +The scans are run on a schedule. New PRs are filed every Sunday, and only dependency versions that are at least +5 days old are suggested, to guard against bad apples. If a new *major* version of a dependency is available, +that will get its own separate Pull Request, so you can choose. + +If an upgrade is decided, simply merge (and backport) the PR. To skip an upgrade, close the PR. If a PR is left open, +it will be re-used and auto updated whenever a newer patch- or minor version gets available. Thus, one can reduce +churn from frequently-updated dependencies by delaying merge until a few weeks before a new release. One can also +choose to change to a less frequent schedule or disable the bot or by editing `renovate.json` + +=== Configuring renovate.json +While the bot runs on a Github account external to the project, the bot behavior can be tailored by editing +`.github/renovate.json` in this project. See https://docs.renovatebot.com[Renovatebot docs] for available options. + +To disable, change the line `"enabled": true` to `false`. To change the schedule for when new PRs are created, +change the https://docs.renovatebot.com/configuration-options/#schedule[`"schedule"` setting]. + +=== Maintaining the bot itself +If you want to help maintain the bot itself, check out https://github.com/solrbot/renovate-github-action. \ No newline at end of file From b18e2333f531c5df86777fd5233e03d176f9d163 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 23 Feb 2023 14:18:07 +0100 Subject: [PATCH 10/16] Tweak wording. --- dev-docs/dependency-upgrades.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-docs/dependency-upgrades.adoc b/dev-docs/dependency-upgrades.adoc index 0cb8d20185cb..fc78cdb3ebc2 100644 --- a/dev-docs/dependency-upgrades.adoc +++ b/dev-docs/dependency-upgrades.adoc @@ -26,7 +26,7 @@ Read the `help/dependencies.txt` file for an in-depth explanation of how gradle https://github.com/palantir/gradle-consistent-versions[Gradle consistent-versions] plugin. == Manual dependency upgrades -In order to upgrade a dependency, you need to run through a number of steps +In order to upgrade a dependency, you need to run through a number of steps: 1. Identify the available versions from e.g. https://search.maven.org[Maven Central] 2. Update the version in `versions.props` file @@ -38,7 +38,7 @@ In order to upgrade a dependency, you need to run through a number of steps == Renovate bot Pull Requests A member of the Solr community operates a Github bot running https://github.com/renovatebot/renovate[Renovate], which -files Pull Requests to Solr with dependency upgrade proposals. The PRs are labeled `dependencies` and include +files Pull Requests to Solr with dependency upgrade proposals. The PRs are labeled `dependencies` and do include changes resulting from `--write-locks` and `updateLicenses`. Community members and committers can then review, and if manual changes are needed, help bring the PR to completion. @@ -51,7 +51,7 @@ that will get its own separate Pull Request, so you can choose. If an upgrade is decided, simply merge (and backport) the PR. To skip an upgrade, close the PR. If a PR is left open, it will be re-used and auto updated whenever a newer patch- or minor version gets available. Thus, one can reduce churn from frequently-updated dependencies by delaying merge until a few weeks before a new release. One can also -choose to change to a less frequent schedule or disable the bot or by editing `renovate.json` +choose to change to a less frequent schedule or disable the bot, by editing `renovate.json` === Configuring renovate.json While the bot runs on a Github account external to the project, the bot behavior can be tailored by editing From 636e82f9c978d37f556a99cbde4db4c28e022ef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 23 Feb 2023 14:56:41 +0100 Subject: [PATCH 11/16] Adds description to json Group httpcomponents in same PR Check test deps montly instead of weekly --- .github/renovate.json | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/renovate.json b/.github/renovate.json index b0b8b9a24c47..3e0e25e0a4a9 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -1,5 +1,6 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "description": "Runs Renovate with solrbot, see dev-docs/dependency-upgrades.adoc for more", "enabled": true, "dependencyDashboard": false, "enabledManagers": ["gradle"], @@ -11,15 +12,28 @@ }, "packageRules": [ { + "description": "Fix for non-semantic versions for older artifacts", "matchDatasources": ["maven"], "matchPackageNames": ["commons-collections:commons-collections", "commons-io:commons-io", "commons-lang:commons-lang"], "versioning": "regex:^(?\\d{1,4})\\.(?\\d+)(\\.(?\\d+))?$" }, { + "description": "Group these together in same PR", + "matchPackagePatterns": ["org.apache.httpcomponents"], + "groupName": "httpcomponents" + }, + { + "description": "Test-dependencies are checked less often than the shipped deps", + "matchDepTypes": ["test"], + "extends": ["schedule:monthly"] + }, + { + "description": "Noisy, frequently updated dependencies checked less often", "matchPackagePrefixes": ["software.amazon.awssdk"], "extends": ["schedule:monthly"] }, { + "description": "Workaround for https://github.com/renovatebot/renovate/issues/19226", "matchPackageNames": ["solr:modules", "HH:mm"], "enabled": false } From 1696f31d106f69da70d3b0c70360aedd1b71d276 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 23 Feb 2023 16:06:37 +0100 Subject: [PATCH 12/16] Adjust includePaths --- .github/renovate.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/renovate.json b/.github/renovate.json index 3e0e25e0a4a9..1d356993b9df 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -4,7 +4,7 @@ "enabled": true, "dependencyDashboard": false, "enabledManagers": ["gradle"], - "includePaths": ["versions.props", "build.gradle"], + "includePaths": ["versions.*", "build.gradle"], "postUpgradeTasks": { "commands": ["./gradlew updateLicenses"], "fileFilters": ["solr/licenses/*.sha1"], From e5d426bf2107b57b81484a9d4007aa848864861d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Thu, 23 Feb 2023 17:05:01 +0100 Subject: [PATCH 13/16] Rename httpcomponents group --- .github/renovate.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/renovate.json b/.github/renovate.json index 1d356993b9df..6c12d42b9372 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -20,7 +20,7 @@ { "description": "Group these together in same PR", "matchPackagePatterns": ["org.apache.httpcomponents"], - "groupName": "httpcomponents" + "groupName": "org.apache.httpcomponents" }, { "description": "Test-dependencies are checked less often than the shipped deps", From 9d087b8c5c5db7659874d4e5c135a16777cbb4b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 24 Feb 2023 00:03:20 +0100 Subject: [PATCH 14/16] Remove "Maintaining the bot" chapter, as this is already covered in the 3rd party bot github action repo. --- dev-docs/dependency-upgrades.adoc | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/dev-docs/dependency-upgrades.adoc b/dev-docs/dependency-upgrades.adoc index fc78cdb3ebc2..e26e0cb956dc 100644 --- a/dev-docs/dependency-upgrades.adoc +++ b/dev-docs/dependency-upgrades.adoc @@ -39,7 +39,7 @@ In order to upgrade a dependency, you need to run through a number of steps: == Renovate bot Pull Requests A member of the Solr community operates a Github bot running https://github.com/renovatebot/renovate[Renovate], which files Pull Requests to Solr with dependency upgrade proposals. The PRs are labeled `dependencies` and do include -changes resulting from `--write-locks` and `updateLicenses`. +changes resulting from `gradle --write-locks` and `updateLicenses`. Community members and committers can then review, and if manual changes are needed, help bring the PR to completion. For many dependencies, a changelog is included in the PR text, which may help guide the upgrade decision. @@ -54,11 +54,6 @@ churn from frequently-updated dependencies by delaying merge until a few weeks b choose to change to a less frequent schedule or disable the bot, by editing `renovate.json` === Configuring renovate.json -While the bot runs on a Github account external to the project, the bot behavior can be tailored by editing -`.github/renovate.json` in this project. See https://docs.renovatebot.com[Renovatebot docs] for available options. - -To disable, change the line `"enabled": true` to `false`. To change the schedule for when new PRs are created, -change the https://docs.renovatebot.com/configuration-options/#schedule[`"schedule"` setting]. - -=== Maintaining the bot itself -If you want to help maintain the bot itself, check out https://github.com/solrbot/renovate-github-action. \ No newline at end of file +While the bot runs on a https://github.com/solrbot/renovate-github-action[GitHub repo external to the project], +the bot behavior can be tailored by editing `.github/renovate.json` in this project. +See https://docs.renovatebot.com[Renovatebot docs] for available options. \ No newline at end of file From c4d4e5ce4c36086870b28d1f94d9a6bbe4f92fe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 24 Feb 2023 09:29:35 +0100 Subject: [PATCH 15/16] Add solrbot as 'collaborator', i.e. triage role, letting it add the 'dependencies' label to PRs --- .asf.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.asf.yaml b/.asf.yaml index b9381e3f818b..6c0b6773b631 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -18,6 +18,11 @@ github: merge: false rebase: false + # Projects may assign external (non-committer) collaborators the triage role for their repository. + # The triage role allows people to assign, edit, and close issues and pull requests, without giving them write access to the code. + collaborators: + - solrbot + notifications: commits: commits@solr.apache.org issues: issues@solr.apache.org From 1a7d120e2772bb3eb9b3e363853be4c1956ca639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 24 Feb 2023 09:58:43 +0100 Subject: [PATCH 16/16] Remove labels --- .asf.yaml | 5 ----- .github/renovate.json | 1 - 2 files changed, 6 deletions(-) diff --git a/.asf.yaml b/.asf.yaml index 6c0b6773b631..b9381e3f818b 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -18,11 +18,6 @@ github: merge: false rebase: false - # Projects may assign external (non-committer) collaborators the triage role for their repository. - # The triage role allows people to assign, edit, and close issues and pull requests, without giving them write access to the code. - collaborators: - - solrbot - notifications: commits: commits@solr.apache.org issues: issues@solr.apache.org diff --git a/.github/renovate.json b/.github/renovate.json index 6c12d42b9372..15145021d5e2 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -41,7 +41,6 @@ "schedule": [ "* 8-20 * * *" ], - "labels": ["dependencies"], "prConcurrentLimit": 5, "prHourlyLimit": 5, "stabilityDays": 5