diff --git a/bundle/BUILD.bazel b/bundle/BUILD.bazel index 7f21cf219..70880e532 100644 --- a/bundle/BUILD.bazel +++ b/bundle/BUILD.bazel @@ -27,6 +27,5 @@ java_library( java_library( name = "environment_exporter", - visibility = ["//:internal"], exports = ["//bundle/src/main/java/dev/cel/bundle:environment_exporter"], ) diff --git a/bundle/src/main/java/dev/cel/bundle/BUILD.bazel b/bundle/src/main/java/dev/cel/bundle/BUILD.bazel index 0201a5807..4dd81fd9e 100644 --- a/bundle/src/main/java/dev/cel/bundle/BUILD.bazel +++ b/bundle/src/main/java/dev/cel/bundle/BUILD.bazel @@ -126,6 +126,7 @@ java_library( ":environment", "//:auto_value", "//bundle:cel", + "//checker:checker_builder", "//checker:standard_decl", "//common:compiler_common", "//common:options", @@ -133,6 +134,7 @@ java_library( "//common/types:cel_proto_types", "//common/types:cel_types", "//common/types:type_providers", + "//compiler:compiler_builder", "//extensions", "//extensions:extension_library", "//parser:macro", diff --git a/bundle/src/main/java/dev/cel/bundle/CelEnvironment.java b/bundle/src/main/java/dev/cel/bundle/CelEnvironment.java index b54e3ca51..7ec2149a7 100644 --- a/bundle/src/main/java/dev/cel/bundle/CelEnvironment.java +++ b/bundle/src/main/java/dev/cel/bundle/CelEnvironment.java @@ -48,6 +48,7 @@ import dev.cel.compiler.CelCompilerLibrary; import dev.cel.extensions.CelExtensions; import dev.cel.parser.CelStandardMacro; +import dev.cel.runtime.CelRuntime; import dev.cel.runtime.CelRuntimeBuilder; import dev.cel.runtime.CelRuntimeLibrary; import java.util.Arrays; @@ -108,6 +109,9 @@ public abstract class CelEnvironment { /** Standard library subset (which macros, functions to include/exclude) */ public abstract Optional standardLibrarySubset(); + /** Feature flags to enable in the environment. */ + public abstract ImmutableSet features(); + /** Builder for {@link CelEnvironment}. */ @AutoValue.Builder public abstract static class Builder { @@ -159,6 +163,13 @@ public Builder setFunctions(FunctionDecl... functions) { public abstract Builder setStandardLibrarySubset(LibrarySubset stdLibrarySubset); + @CanIgnoreReturnValue + public Builder setFeatures(FeatureFlag... featureFlags) { + return setFeatures(ImmutableSet.copyOf(featureFlags)); + } + + public abstract Builder setFeatures(ImmutableSet macros); + abstract CelEnvironment autoBuild(); @CheckReturnValue @@ -188,18 +199,21 @@ public static Builder newBuilder() { .setDescription("") .setContainer(CelContainer.ofName("")) .setVariables(ImmutableSet.of()) - .setFunctions(ImmutableSet.of()); + .setFunctions(ImmutableSet.of()) + .setFeatures(ImmutableSet.of()); } /** Extends the provided {@link CelCompiler} environment with this configuration. */ public CelCompiler extend(CelCompiler celCompiler, CelOptions celOptions) throws CelEnvironmentException { + celOptions = applyFeatureFlags(celOptions); try { CelTypeProvider celTypeProvider = celCompiler.getTypeProvider(); CelCompilerBuilder compilerBuilder = celCompiler .toCompilerBuilder() .setContainer(container()) + .setOptions(celOptions) .setTypeProvider(celTypeProvider) .addVarDeclarations( variables().stream() @@ -222,19 +236,35 @@ public CelCompiler extend(CelCompiler celCompiler, CelOptions celOptions) /** Extends the provided {@link Cel} environment with this configuration. */ public Cel extend(Cel cel, CelOptions celOptions) throws CelEnvironmentException { + celOptions = applyFeatureFlags(celOptions); try { // Casting is necessary to only extend the compiler here CelCompiler celCompiler = extend((CelCompiler) cel, celOptions); - CelRuntimeBuilder celRuntimeBuilder = cel.toRuntimeBuilder(); - addAllRuntimeExtensions(celRuntimeBuilder, celOptions); + CelRuntime celRuntime = extendRuntime(cel, celOptions); - return CelFactory.combine(celCompiler, celRuntimeBuilder.build()); + return CelFactory.combine(celCompiler, celRuntime); } catch (RuntimeException e) { throw new CelEnvironmentException(e.getMessage(), e); } } + private CelOptions applyFeatureFlags(CelOptions celOptions) { + CelOptions.Builder optionsBuilder = celOptions.toBuilder(); + for (FeatureFlag featureFlag : features()) { + if (featureFlag.name().equals("cel.feature.macro_call_tracking")) { + optionsBuilder.populateMacroCalls(featureFlag.enabled()); + } else if (featureFlag.name().equals("cel.feature.backtick_escape_syntax")) { + optionsBuilder.enableQuotedIdentifierSyntax(featureFlag.enabled()); + } else if (featureFlag.name().equals("cel.feature.cross_type_numeric_comparisons")) { + optionsBuilder.enableHeterogeneousNumericComparisons(featureFlag.enabled()); + } else { + throw new IllegalArgumentException("Unknown feature flag: " + featureFlag.name()); + } + } + return optionsBuilder.build(); + } + private void addAllCompilerExtensions( CelCompilerBuilder celCompilerBuilder, CelOptions celOptions) { // TODO: Add capability to accept user defined exceptions @@ -250,7 +280,9 @@ private void addAllCompilerExtensions( } } - private void addAllRuntimeExtensions(CelRuntimeBuilder celRuntimeBuilder, CelOptions celOptions) { + private CelRuntime extendRuntime(CelRuntime celRuntime, CelOptions celOptions) { + CelRuntimeBuilder celRuntimeBuilder = celRuntime.toRuntimeBuilder(); + celRuntimeBuilder.setOptions(celOptions); // TODO: Add capability to accept user defined exceptions for (ExtensionConfig extensionConfig : extensions()) { CanonicalCelExtension extension = getExtensionOrThrow(extensionConfig.name()); @@ -262,6 +294,7 @@ private void addAllRuntimeExtensions(CelRuntimeBuilder celRuntimeBuilder, CelOpt celRuntimeBuilder.addLibraries(celRuntimeLibrary); } } + return celRuntimeBuilder.build(); } private void applyStandardLibrarySubset(CelCompilerBuilder compilerBuilder) { @@ -625,6 +658,20 @@ public CelType toCelType(CelTypeProvider celTypeProvider) { } } + /** Represents a feature flag that can be enabled in the environment. */ + @AutoValue + public abstract static class FeatureFlag { + /** Normalized name of the feature flag. */ + public abstract String name(); + + /** Whether the feature is enabled or disabled. */ + public abstract boolean enabled(); + + public static FeatureFlag create(String name, boolean enabled) { + return new AutoValue_CelEnvironment_FeatureFlag(name, enabled); + } + } + /** * Represents a configuration for a canonical CEL extension that can be enabled in the * environment. diff --git a/bundle/src/main/java/dev/cel/bundle/CelEnvironmentExporter.java b/bundle/src/main/java/dev/cel/bundle/CelEnvironmentExporter.java index 01410ad0d..1ed113db7 100644 --- a/bundle/src/main/java/dev/cel/bundle/CelEnvironmentExporter.java +++ b/bundle/src/main/java/dev/cel/bundle/CelEnvironmentExporter.java @@ -22,6 +22,7 @@ import dev.cel.expr.Decl.FunctionDecl; import dev.cel.expr.Decl.FunctionDecl.Overload; import com.google.auto.value.AutoValue; +import com.google.common.base.Preconditions; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ListMultimap; @@ -30,6 +31,7 @@ import dev.cel.bundle.CelEnvironment.LibrarySubset; import dev.cel.bundle.CelEnvironment.LibrarySubset.FunctionSelector; import dev.cel.bundle.CelEnvironment.OverloadDecl; +import dev.cel.checker.CelCheckerBuilder; import dev.cel.checker.CelStandardDeclarations.StandardFunction; import dev.cel.checker.CelStandardDeclarations.StandardIdentifier; import dev.cel.common.CelFunctionDecl; @@ -41,6 +43,7 @@ import dev.cel.common.types.CelProtoTypes; import dev.cel.common.types.CelType; import dev.cel.common.types.CelTypes; +import dev.cel.compiler.CelCompiler; import dev.cel.extensions.CelExtensionLibrary; import dev.cel.extensions.CelExtensions; import dev.cel.parser.CelMacro; @@ -161,9 +164,24 @@ public static CelEnvironmentExporter.Builder newBuilder() { * */ public CelEnvironment export(Cel cel) { - CelEnvironment.Builder envBuilder = - CelEnvironment.newBuilder().setContainer(cel.toCheckerBuilder().container()); + return export((CelCompiler) cel); + } + /** + * Exports a {@link CelEnvironment} that describes the configuration of the given {@link + * CelCompiler} instance. + * + *

The exported environment includes: + * + *

    + *
  • Standard library subset: functions and their overloads that are either included or + * excluded from the standard library. + *
  • Extension libraries: names and versions of the extension libraries that are used. + *
  • Custom declarations: functions and variables that are not part of the standard library or + * any of the extension libraries. + *
+ */ + public CelEnvironment export(CelCompiler cel) { // Inventory is a full set of declarations and macros that are found in the configuration of // the supplied CEL instance. // @@ -172,6 +190,14 @@ public CelEnvironment export(Cel cel) { // // Whatever is left will be included in the Environment as custom declarations. + // Checker builder is used to access some parts of the config not exposed in the EnvVisitable + // interface. + CelCheckerBuilder checkerBuilder = cel.toCheckerBuilder(); + + CelEnvironment.Builder envBuilder = + CelEnvironment.newBuilder().setContainer(checkerBuilder.container()); + addOptions(envBuilder, checkerBuilder.options()); + Set inventory = new HashSet<>(); collectInventory(inventory, cel); addExtensionConfigsAndRemoveFromInventory(envBuilder, inventory); @@ -180,11 +206,29 @@ public CelEnvironment export(Cel cel) { return envBuilder.build(); } + private void addOptions(CelEnvironment.Builder envBuilder, CelOptions options) { + // The set of features supported in the exported environment in Go is pretty limited right now. + ImmutableSet.Builder featureFlags = ImmutableSet.builder(); + if (options.enableHeterogeneousNumericComparisons()) { + featureFlags.add( + CelEnvironment.FeatureFlag.create("cel.feature.cross_type_numeric_comparisons", true)); + } + if (options.enableQuotedIdentifierSyntax()) { + featureFlags.add( + CelEnvironment.FeatureFlag.create("cel.feature.backtick_escape_syntax", true)); + } + if (options.populateMacroCalls()) { + featureFlags.add(CelEnvironment.FeatureFlag.create("cel.feature.macro_call_tracking", true)); + } + envBuilder.setFeatures(featureFlags.build()); + } + /** * Collects all function overloads, variable declarations and macros from the given {@link Cel} * instance and stores them in a map. */ - private void collectInventory(Set inventory, Cel cel) { + private void collectInventory(Set inventory, CelCompiler cel) { + Preconditions.checkArgument(cel instanceof EnvVisitable); ((EnvVisitable) cel) .accept( new EnvVisitor() { diff --git a/bundle/src/main/java/dev/cel/bundle/CelEnvironmentYamlParser.java b/bundle/src/main/java/dev/cel/bundle/CelEnvironmentYamlParser.java index 8c19fcfa6..2fa8923f1 100644 --- a/bundle/src/main/java/dev/cel/bundle/CelEnvironmentYamlParser.java +++ b/bundle/src/main/java/dev/cel/bundle/CelEnvironmentYamlParser.java @@ -143,6 +143,51 @@ private CelContainer parseContainer(ParserContext ctx, Node node) { return builder.build(); } + private ImmutableSet parseFeatures( + ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + if (!validateYamlType(node, YamlNodeType.LIST, YamlNodeType.TEXT)) { + ctx.reportError(valueId, "Unsupported features format"); + } + + ImmutableSet.Builder featureFlags = ImmutableSet.builder(); + + SequenceNode featureListNode = (SequenceNode) node; + for (Node featureMapNode : featureListNode.getValue()) { + long featureMapId = ctx.collectMetadata(featureMapNode); + if (!assertYamlType(ctx, featureMapId, featureMapNode, YamlNodeType.MAP)) { + continue; + } + + MappingNode featureMap = (MappingNode) featureMapNode; + String name = ""; + boolean enabled = true; + for (NodeTuple nodeTuple : featureMap.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + Node valueNode = nodeTuple.getValueNode(); + String keyName = ((ScalarNode) keyNode).getValue(); + switch (keyName) { + case "name": + name = newString(ctx, valueNode); + break; + case "enabled": + enabled = newBoolean(ctx, valueNode); + break; + default: + ctx.reportError(keyId, String.format("Unsupported feature tag: %s", keyName)); + break; + } + } + if (name.isEmpty()) { + ctx.reportError(featureMapId, "Missing required attribute(s): name"); + continue; + } + featureFlags.add(CelEnvironment.FeatureFlag.create(name, enabled)); + } + return featureFlags.build(); + } + private ImmutableSet parseAliases(ParserContext ctx, Node node) { ImmutableSet.Builder aliasSetBuilder = ImmutableSet.builder(); long valueId = ctx.collectMetadata(node); @@ -756,6 +801,9 @@ private CelEnvironment.Builder parseConfig(ParserContext ctx, Node node) { case "stdlib": builder.setStandardLibrarySubset(parseLibrarySubset(ctx, valueNode)); break; + case "features": + builder.setFeatures(parseFeatures(ctx, valueNode)); + break; default: ctx.reportError(id, "Unknown config tag: " + fieldName); // continue handling the rest of the nodes diff --git a/bundle/src/main/java/dev/cel/bundle/CelEnvironmentYamlSerializer.java b/bundle/src/main/java/dev/cel/bundle/CelEnvironmentYamlSerializer.java index 81f206b94..2cc229dc9 100644 --- a/bundle/src/main/java/dev/cel/bundle/CelEnvironmentYamlSerializer.java +++ b/bundle/src/main/java/dev/cel/bundle/CelEnvironmentYamlSerializer.java @@ -60,6 +60,7 @@ private CelEnvironmentYamlSerializer() { CelEnvironment.LibrarySubset.OverloadSelector.class, new RepresentOverloadSelector()); this.multiRepresenters.put(CelEnvironment.Alias.class, new RepresentAlias()); this.multiRepresenters.put(CelContainer.class, new RepresentContainer()); + this.multiRepresenters.put(CelEnvironment.FeatureFlag.class, new RepresentFeatureFlag()); } public static String toYaml(CelEnvironment environment) { @@ -94,6 +95,9 @@ public Node representData(Object data) { if (environment.standardLibrarySubset().isPresent()) { configMap.put("stdlib", environment.standardLibrarySubset().get()); } + if (!environment.features().isEmpty()) { + configMap.put("features", environment.features().asList()); + } return represent(configMap.buildOrThrow()); } } @@ -258,4 +262,17 @@ public Node representData(Object data) { return represent(ImmutableMap.of("id", overloadSelector.id())); } } + + private final class RepresentFeatureFlag implements Represent { + + @Override + public Node representData(Object data) { + CelEnvironment.FeatureFlag featureFlag = (CelEnvironment.FeatureFlag) data; + return represent( + ImmutableMap.builder() + .put("name", featureFlag.name()) + .put("enabled", featureFlag.enabled()) + .buildOrThrow()); + } + } } diff --git a/bundle/src/test/java/dev/cel/bundle/CelEnvironmentTest.java b/bundle/src/test/java/dev/cel/bundle/CelEnvironmentTest.java index 6bc84a48f..3386bdaae 100644 --- a/bundle/src/test/java/dev/cel/bundle/CelEnvironmentTest.java +++ b/bundle/src/test/java/dev/cel/bundle/CelEnvironmentTest.java @@ -100,6 +100,49 @@ public void extend_allExtensions() throws Exception { assertThat(result).isTrue(); } + @Test + public void extend_allFeatureFlags() throws Exception { + CelEnvironment environment = + CelEnvironment.newBuilder() + .setFeatures( + CelEnvironment.FeatureFlag.create("cel.feature.macro_call_tracking", true), + CelEnvironment.FeatureFlag.create("cel.feature.backtick_escape_syntax", true), + CelEnvironment.FeatureFlag.create( + "cel.feature.cross_type_numeric_comparisons", true)) + .build(); + + Cel cel = + environment.extend( + CelFactory.standardCelBuilder() + .setStandardMacros(CelStandardMacro.STANDARD_MACROS) + .build(), + CelOptions.DEFAULT); + CelAbstractSyntaxTree ast = + cel.compile("[{'foo.bar': 1}, {'foo.bar': 2}].all(e, e.`foo.bar` < 2.5)").getAst(); + assertThat(ast.getSource().getMacroCalls()).hasSize(1); + boolean result = (boolean) cel.createProgram(ast).eval(); + assertThat(result).isTrue(); + } + + @Test + public void extend_unsupportedFeatureFlag_throws() throws Exception { + CelEnvironment environment = + CelEnvironment.newBuilder() + .setFeatures(CelEnvironment.FeatureFlag.create("unknown.feature", true)) + .build(); + + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> + environment.extend( + CelFactory.standardCelBuilder() + .setStandardMacros(CelStandardMacro.STANDARD_MACROS) + .build(), + CelOptions.DEFAULT)); + assertThat(e).hasMessageThat().contains("Unknown feature flag: unknown.feature"); + } + @Test public void extensionVersion_specific() throws Exception { CelEnvironment environment = diff --git a/bundle/src/test/java/dev/cel/bundle/CelEnvironmentYamlParserTest.java b/bundle/src/test/java/dev/cel/bundle/CelEnvironmentYamlParserTest.java index d69d0517b..98ce55ecc 100644 --- a/bundle/src/test/java/dev/cel/bundle/CelEnvironmentYamlParserTest.java +++ b/bundle/src/test/java/dev/cel/bundle/CelEnvironmentYamlParserTest.java @@ -40,8 +40,8 @@ import dev.cel.common.types.SimpleType; import dev.cel.parser.CelUnparserFactory; import dev.cel.runtime.CelEvaluationListener; -import dev.cel.runtime.CelLateFunctionBindings; import dev.cel.runtime.CelFunctionBinding; +import dev.cel.runtime.CelLateFunctionBindings; import java.io.IOException; import java.net.URL; import java.util.Optional; @@ -81,6 +81,33 @@ public void environment_setBasicProperties() throws Exception { .build()); } + @Test + public void environment_setFeatures() throws Exception { + String yamlConfig = + "name: hello\n" + + "description: empty\n" + + "features:\n" + + " - name: 'cel.feature.macro_call_tracking'\n" + + " enabled: true\n" + + " - name: 'cel.feature.backtick_escape_syntax'\n" + + " enabled: false"; + + CelEnvironment environment = ENVIRONMENT_PARSER.parse(yamlConfig); + + assertThat(environment) + .isEqualTo( + CelEnvironment.newBuilder() + .setSource(environment.source().get()) + .setName("hello") + .setDescription("empty") + .setFeatures( + ImmutableSet.of( + CelEnvironment.FeatureFlag.create("cel.feature.macro_call_tracking", true), + CelEnvironment.FeatureFlag.create( + "cel.feature.backtick_escape_syntax", false))) + .build()); + } + @Test public void environment_setExtensions() throws Exception { String yamlConfig = @@ -672,6 +699,16 @@ private enum EnvironmentParseErrorTestcase { "ERROR: :6:7: Unsupported alias tag: unknown_tag\n" + " | unknown_tag: 'test_value'\n" + " | ......^"), + ILLEGAL_FEATURE_TAG( + "features:\n" + " - name: 'test_feature'\n" + " unknown_tag: 'test_value'\n", + "ERROR: :3:5: Unsupported feature tag: unknown_tag\n" + + " | unknown_tag: 'test_value'\n" + + " | ....^"), + MISSING_FEATURE_NAME( + "features:\n" + " - enabled: true\n", + "ERROR: :2:5: Missing required attribute(s): name\n" + + " | - enabled: true\n" + + " | ....^"), ; private final String yamlConfig; @@ -793,6 +830,7 @@ private enum EnvironmentYamlResourceTestCase { .build()) .setReturnType(TypeDecl.create("bool")) .build()))) + .setFeatures(CelEnvironment.FeatureFlag.create("cel.feature.macro_call_tracking", true)) .build()), LIBRARY_SUBSET_ENV( diff --git a/bundle/src/test/java/dev/cel/bundle/CelEnvironmentYamlSerializerTest.java b/bundle/src/test/java/dev/cel/bundle/CelEnvironmentYamlSerializerTest.java index 7e4be0912..1c56370b2 100644 --- a/bundle/src/test/java/dev/cel/bundle/CelEnvironmentYamlSerializerTest.java +++ b/bundle/src/test/java/dev/cel/bundle/CelEnvironmentYamlSerializerTest.java @@ -126,6 +126,9 @@ public void toYaml_success() throws Exception { FunctionSelector.create( "_+_", ImmutableSet.of("add_bytes", "add_list")))) .build()) + .setFeatures( + CelEnvironment.FeatureFlag.create("cel.feature.macro_call_tracking", true), + CelEnvironment.FeatureFlag.create("cel.feature.backtick_escape_syntax", false)) .build(); String yamlOutput = CelEnvironmentYamlSerializer.toYaml(environment); diff --git a/checker/src/main/java/dev/cel/checker/CelCheckerBuilder.java b/checker/src/main/java/dev/cel/checker/CelCheckerBuilder.java index e19cf5b70..a7d531f88 100644 --- a/checker/src/main/java/dev/cel/checker/CelCheckerBuilder.java +++ b/checker/src/main/java/dev/cel/checker/CelCheckerBuilder.java @@ -35,6 +35,9 @@ public interface CelCheckerBuilder { @CanIgnoreReturnValue CelCheckerBuilder setOptions(CelOptions options); + /** Retrieves the currently configured {@link CelOptions} in the builder. */ + CelOptions options(); + /** * Set the {@link CelContainer} to use as the namespace for resolving CEL expression variables and * functions. diff --git a/checker/src/main/java/dev/cel/checker/CelCheckerLegacyImpl.java b/checker/src/main/java/dev/cel/checker/CelCheckerLegacyImpl.java index df8a82f43..ceab0fa93 100644 --- a/checker/src/main/java/dev/cel/checker/CelCheckerLegacyImpl.java +++ b/checker/src/main/java/dev/cel/checker/CelCheckerLegacyImpl.java @@ -202,6 +202,11 @@ public CelCheckerBuilder setOptions(CelOptions celOptions) { return this; } + @Override + public CelOptions options() { + return this.celOptions; + } + @Override public CelCheckerBuilder setContainer(CelContainer container) { checkNotNull(container); @@ -421,11 +426,6 @@ CelStandardDeclarations standardDeclarations() { return this.standardDeclarations; } - @VisibleForTesting - CelOptions options() { - return this.celOptions; - } - @VisibleForTesting CelTypeProvider celTypeProvider() { return this.celTypeProvider; diff --git a/testing/src/test/resources/environment/dump_env.yaml b/testing/src/test/resources/environment/dump_env.yaml index 6a885ea51..18f96fbcc 100644 --- a/testing/src/test/resources/environment/dump_env.yaml +++ b/testing/src/test/resources/environment/dump_env.yaml @@ -82,3 +82,8 @@ stdlib: overloads: - id: add_bytes - id: add_list +features: +- name: cel.feature.macro_call_tracking + enabled: true +- name: cel.feature.backtick_escape_syntax + enabled: false diff --git a/testing/src/test/resources/environment/extended_env.yaml b/testing/src/test/resources/environment/extended_env.yaml index fbed2b9d5..c420ad4db 100644 --- a/testing/src/test/resources/environment/extended_env.yaml +++ b/testing/src/test/resources/environment/extended_env.yaml @@ -38,6 +38,9 @@ functions: is_type_param: true return: type_name: "bool" +features: + - name: cel.feature.macro_call_tracking + enabled: true # TODO: Add support for below #validators: #- name: cel.validator.duration @@ -46,7 +49,3 @@ functions: #- name: cel.validator.nesting_comprehension_limit # config: # limit: 2 -# TODO: Add support for below -#features: -#- name: cel.feature.macro_call_tracking -# enabled: true \ No newline at end of file