deps: Experiment — adopt canonical CEL - py_proto_library support via rules_python

api/bazel/api_build_system.bzl

@@ -1,6 +1,6 @@
 load("@com_envoyproxy_protoc_gen_validate//bazel:pgv_proto_library.bzl", "pgv_cc_proto_library")
 load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library")
-load("@com_google_protobuf//:protobuf.bzl", _py_proto_library = "py_proto_library")
+load("@rules_python//python:proto.bzl", _py_proto_library = "py_proto_library")
 load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library")
 load("@io_bazel_rules_go//go:def.bzl", "go_test")
 load("@rules_proto//proto:defs.bzl", "proto_library")
@@ -59,53 +59,68 @@ def _py_proto_mapping(dep):
 # https://github.com/bazelbuild/bazel/issues/3935 and/or
 # https://github.com/bazelbuild/bazel/issues/2626 are resolved.
 def _api_py_proto_library(name, srcs = [], deps = []):
-    mapped_deps = [_py_proto_mapping(dep) for dep in deps]
-    mapped_unique_deps = {k: True for k in mapped_deps}.keys()
     _py_proto_library(
         name = name + _PY_PROTO_SUFFIX,
-        srcs = srcs,
-        default_runtime = "@com_google_protobuf//:protobuf_python",
-        protoc = "@com_google_protobuf//:protoc",
-        deps = mapped_unique_deps + [
-            "@com_envoyproxy_protoc_gen_validate//validate:validate_py",
-            "@com_google_googleapis//google/rpc:status_py_proto",
-            "@com_google_googleapis//google/api:annotations_py_proto",
-            "@com_google_googleapis//google/api:http_py_proto",
-            "@com_google_googleapis//google/api:httpbody_py_proto",
+        deps = deps + [
+            "@com_google_googleapis//google/api:http_proto",
         ],
         visibility = ["//visibility:public"],
     )
-
-# This defines googleapis py_proto_library. The repository does not provide its definition and requires
-# overriding it in the consuming project (see https://github.com/grpc/grpc/issues/19255 for more details).
-def py_proto_library(name, deps = [], plugin = None):
-    srcs = [dep[:-6] + ".proto" if dep.endswith("_proto") else dep for dep in deps]
-    proto_deps = []
-
-    # py_proto_library in googleapis specifies *_proto rules in dependencies.
-    # By rewriting *_proto to *.proto above, the dependencies in *_proto rules are not preserved.
-    # As a workaround, manually specify the proto dependencies for the imported python rules.
-    if name == "annotations_py_proto":
-        proto_deps = proto_deps + [":http_py_proto"]
-
-    # checked.proto depends on syntax.proto, we have to add this dependency manually as well.
-    if name == "checked_py_proto":
-        proto_deps = proto_deps + [":syntax_py_proto"]
-
-    # Special handling for expr_proto target
-    if srcs[0] == ":expr_moved.proto":
-        srcs = ["checked.proto", "eval.proto", "explain.proto", "syntax.proto", "value.proto"]
-        proto_deps = proto_deps + ["@com_google_googleapis//google/rpc:status_py_proto"]
-
-    # py_proto_library does not support plugin as an argument yet at gRPC v1.25.0:
-    # https://github.com/grpc/grpc/blob/v1.25.0/bazel/python_rules.bzl#L72.
-    # plugin should also be passed in here when gRPC version is greater than v1.25.x.
+#    mapped_deps = [_py_proto_mapping(dep) for dep in deps]
+#    mapped_unique_deps = {k: True for k in mapped_deps}.keys()
+#    _py_proto_library(
+#        name = name + _PY_PROTO_SUFFIX,
+#        srcs = srcs,
+#        default_runtime = "@com_google_protobuf//:protobuf_python",
+#        protoc = "@com_google_protobuf//:protoc",
+#        deps = mapped_unique_deps + [
+#            "@com_envoyproxy_protoc_gen_validate//validate:validate_py",
+#            "@com_google_googleapis//google/rpc:status_py_proto",
+#            "@com_google_googleapis//google/api:annotations_py_proto",
+#            "@com_google_googleapis//google/api:http_py_proto",
+#            "@com_google_googleapis//google/api:httpbody_py_proto",
+#        ],
+#        visibility = ["//visibility:public"],
+#    )
+
+## This defines googleapis py_proto_library. The repository does not provide its definition and requires
+## overriding it in the consuming project (see https://github.com/grpc/grpc/issues/19255 for more details).
+#def py_proto_library(name, deps = [], plugin = None):
+#    srcs = [dep[:-6] + ".proto" if dep.endswith("_proto") else dep for dep in deps]
+#    proto_deps = []
+#
+#    # py_proto_library in googleapis specifies *_proto rules in dependencies.
+#    # By rewriting *_proto to *.proto above, the dependencies in *_proto rules are not preserved.
+#    # As a workaround, manually specify the proto dependencies for the imported python rules.
+#    if name == "annotations_py_proto":
+#        proto_deps = proto_deps + [":http_py_proto"]
+#
+#    # checked.proto depends on syntax.proto, we have to add this dependency manually as well.
+#    if name == "checked_py_proto":
+#        proto_deps = proto_deps + [":syntax_py_proto"]
+#
+#    # Special handling for expr_proto target
+#    if srcs[0] == ":expr_moved.proto":
+#        srcs = ["checked.proto", "eval.proto", "explain.proto", "syntax.proto", "value.proto"]
+#        proto_deps = proto_deps + ["@com_google_googleapis//google/rpc:status_py_proto"]
+#
+#    # py_proto_library does not support plugin as an argument yet at gRPC v1.25.0:
+#    # https://github.com/grpc/grpc/blob/v1.25.0/bazel/python_rules.bzl#L72.
+#    # plugin should also be passed in here when gRPC version is greater than v1.25.x.
+#    _py_proto_library(
+#        name = name,
+#        srcs = srcs,
+#        default_runtime = "@com_google_protobuf//:protobuf_python",
+#        protoc = "@com_google_protobuf//:protoc",
+#        deps = proto_deps + ["@com_google_protobuf//:protobuf_python"],
+#        visibility = ["//visibility:public"],
+#    )
+
+def cc_proto_library(name, deps = [], **kwargs):
+    native.cc_proto_library(name=name, deps=deps, **kwargs)
     _py_proto_library(
-        name = name,
-        srcs = srcs,
-        default_runtime = "@com_google_protobuf//:protobuf_python",
-        protoc = "@com_google_protobuf//:protoc",
-        deps = proto_deps + ["@com_google_protobuf//:protobuf_python"],
+        name = name.replace('_cc_proto', '') + "_py_pb2",
+        deps = deps,
         visibility = ["//visibility:public"],
     )
 
@@ -157,7 +172,6 @@ def api_cc_py_proto_library(
         deps = [relative_name],
         visibility = ["//visibility:public"],
     )
-    _api_py_proto_library(name, srcs, deps)
 
     # Optionally define gRPC services
     if has_services:
@@ -196,6 +210,8 @@ def api_proto_package(
         has_services = has_services,
     )
 
+    _api_py_proto_library(name = name, srcs = srcs, deps = deps)
+
     compilers = ["@io_bazel_rules_go//proto:go_proto", "@envoy_api//bazel:pgv_plugin_go"]
     if has_services:
         compilers = ["@io_bazel_rules_go//proto:go_grpc", "@envoy_api//bazel:pgv_plugin_go"]

api/bazel/repositories.bzl

@@ -36,6 +36,9 @@ def api_dependencies():
     external_http_archive(
         name = "opencensus_proto",
     )
+    external_http_archive(
+        name = "rules_python",
+    )
     external_http_archive(
         name = "rules_proto",
     )
@@ -56,6 +59,10 @@ def api_dependencies():
         name = "com_github_chrusty_protoc_gen_jsonschema",
     )
 
+    external_http_archive(
+        name = "dev_cel",
+    )
+
     external_http_archive(
         name = "envoy_toolshed",
     )

api/bazel/repository_locations.bzl

@@ -39,11 +39,11 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         project_desc = "xDS API Working Group (xDS-WG)",
         project_url = "https://github.com/cncf/xds",
         # During the UDPA -> xDS migration, we aren't working with releases.
-        version = "e9ce68804cb4e64cab5a52e3c8baf840d4ff87b7",
-        sha256 = "0d33b83f8c6368954e72e7785539f0d272a8aba2f6e2e336ed15fd1514bc9899",
+        version = "expression-new-fields",
+        sha256 = "5144c11e16044df6d9152a3bac56974c7d8c462c88578e33d8f9e03cd03ffce6",
         release_date = "2023-06-07",
-        strip_prefix = "xds-{version}",
-        urls = ["https://github.com/cncf/xds/archive/{version}.tar.gz"],
+        strip_prefix = "xds-cel-expression-new-fields",
+        urls = ["https://github.com/sergiitk/xds/archive/refs/heads/cel-{version}.zip"],
         use_category = ["api"],
         license = "Apache-2.0",
         license_url = "https://github.com/cncf/xds/blob/{version}/LICENSE",
@@ -101,13 +101,26 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         license = "Apache-2.0",
         license_url = "https://github.com/prometheus/client_model/blob/v{version}/LICENSE",
     ),
+    rules_python = dict(
+        project_name = "Bazel Python Rules",
+        project_desc = "Core Python rules -- py_library, py_binary, py_test, py_proto_library",
+        project_url = "https://github.com/bazelbuild/rules_python",
+        version = "0.26.0",
+        sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b",
+        release_date = "2023-10-05",
+        strip_prefix = "rules_python-{version}",
+        urls = ["https://github.com/bazelbuild/rules_python/releases/download/{version}.tar.gz"],
+        use_category = ["api"],
+        license = "Apache-2.0",
+        license_url = "https://github.com/bazelbuild/rules_proto/blob/{version}/LICENSE",
+    ),
     rules_proto = dict(
         project_name = "Protobuf Rules for Bazel",
         project_desc = "Protocol buffer rules for Bazel",
         project_url = "https://github.com/bazelbuild/rules_proto",
-        version = "4.0.0",
-        sha256 = "66bfdf8782796239d3875d37e7de19b1d94301e8972b3cbd2446b332429b4df1",
-        release_date = "2021-09-15",
+        version = "5.3.0-21.7",
+        sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd",
+        release_date = "2022-12-27",
         strip_prefix = "rules_proto-{version}",
         urls = ["https://github.com/bazelbuild/rules_proto/archive/refs/tags/{version}.tar.gz"],
         use_category = ["api"],
@@ -151,6 +164,17 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         use_category = ["build"],
         release_date = "2023-05-30",
     ),
+    dev_cel = dict(
+        project_name = "CEL",
+        project_desc = "Common Expression Language -- specification and binary representation",
+        project_url = "https://github.com/google/cel-spec",
+        strip_prefix = "cel-spec-{version}",
+        sha256 = "3de60ea3a29b6246faf04d206b7ee4633c155042e040086205c83edf713aee29",
+        version = "2657e88023e5e7dfbb62e74de7a7cfd6e8284d7b",
+        urls = ["https://github.com/google/cel-spec/archive/{version}.zip"],
+        use_category = ["api"],
+        release_date = "2023-11-09",
+    ),
     envoy_toolshed = dict(
         project_name = "envoy_toolshed",
         project_desc = "Tooling, libraries, runners and checkers for Envoy proxy's CI",

bazel/protobuf2.patch

@@ -0,0 +1,90 @@
+diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py
+index e7555ee10..a93beb1c5 100644
+--- a/python/google/protobuf/__init__.py
++++ b/python/google/protobuf/__init__.py
+@@ -31,3 +31,10 @@
+ # Copyright 2007 Google Inc. All Rights Reserved.
+
+ __version__ = '4.23.4'
++
++
++if __name__ != '__main__':
++    try:
++        __import__('pkg_resources').declare_namespace(__name__)
++    except ImportError:
++        __path__ = __import__('pkgutil').extend_path(__path__, __name__)
+diff --git a/src/google/protobuf/io/BUILD.bazel b/src/google/protobuf/io/BUILD.bazel
+--- a/src/google/protobuf/io/BUILD.bazel
++++ b/src/google/protobuf/io/BUILD.bazel
+@@ -138,7 +138,7 @@ cc_library(
+         "@com_google_absl//absl/log:absl_log",
+     ] + select({
+         "//build_defs:config_msvc": [],
+-        "//conditions:default": ["@zlib//:zlib"],
++        "//conditions:default": ["//external:zlib"],
+     }),
+ )
+
+diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc
+--- a/src/google/protobuf/port_def.inc	2023-06-27 01:17:34.917105764 +0000
++++ b/src/google/protobuf/port_def.inc	2023-06-27 01:18:12.069060142 +0000
+@@ -1004,7 +1004,7 @@
+ #pragma clang diagnostic ignored "-Wshorten-64-to-32"
+ // Turn on -Wdeprecated-enum-enum-conversion. This deprecation comes in C++20
+ // via http://wg21.link/p1120r0.
+-#pragma clang diagnostic error "-Wdeprecated-enum-enum-conversion"
++// #pragma clang diagnostic error "-Wdeprecated-enum-enum-conversion"
+ // This error has been generally flaky, but we need to disable it specifically
+ // to fix https://github.com/protocolbuffers/protobuf/issues/12313
+ #pragma clang diagnostic ignored "-Wunused-parameter"
+diff --git a/src/google/protobuf/map.h b/src/google/protobuf/map.h
+--- a/src/google/protobuf/map.h	2023-06-30 14:32:34.892148233 +0000
++++ b/src/google/protobuf/map.h	2023-06-30 14:35:37.447992493 +0000
+@@ -883,7 +883,7 @@
+         TreeConvert(b);
+       }
+       ABSL_DCHECK(TableEntryIsTree(b))
+-          << (void*)table_[b] << " " << (uintptr_t)table_[b];
++          << reinterpret_cast<void*>(table_[b]) << " " << static_cast<uintptr_t>(table_[b]);
+       InsertUniqueInTree(b, node);
+       index_of_first_non_null_ = (std::min)(index_of_first_non_null_, b);
+     }
+diff --git a/src/google/protobuf/map_field.h b/src/google/protobuf/map_field.h
+--- a/src/google/protobuf/map_field.h	2023-06-30 17:14:18.934528580 +0000
++++ b/src/google/protobuf/map_field.h	2023-06-30 17:14:52.098500807 +0000
+@@ -345,7 +345,7 @@
+
+  protected:
+   // "protected" stops users from deleting a `MapFieldBase *`
+-  ~MapFieldBase();
++  virtual ~MapFieldBase();
+
+  public:
+   // Returns reference to internal repeated field. Data written using
+diff --git a/src/google/protobuf/compiler/BUILD.bazel b/src/google/protobuf/compiler/BUILD.bazel
+--- a/src/google/protobuf/compiler/BUILD.bazel
++++ b/src/google/protobuf/compiler/BUILD.bazel
+@@ -306,7 +306,7 @@
+     srcs = ["retention.cc"],
+     hdrs = ["retention.h"],
+     include_prefix = "google/protobuf/compiler",
+-    visibility = ["//src/google/protobuf:__subpackages__"],
++    visibility = ["//visibility:public"],
+     deps = [
+         "//src/google/protobuf:protobuf_nowkt",
+         "@com_google_absl//absl/types:span",
+
+diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc
+index 1c6a24945..c27d0bf2a 100644
+--- a/src/google/protobuf/port_def.inc
++++ b/src/google/protobuf/port_def.inc
+@@ -1062,6 +1062,9 @@ static_assert(PROTOBUF_ABSL_MIN(20230125, 3),
+ #pragma warning(disable: 4125)
+ #endif
+
++#if defined(__GNUC__)
++#pragma GCC diagnostic ignored "-Wundef"
++#endif
+ #if PROTOBUF_ENABLE_DEBUG_LOGGING_MAY_LEAK_PII
+ #define PROTOBUF_DEBUG true
+ #else

bazel/repositories.bzl

@@ -3,6 +3,8 @@ load("@envoy_api//bazel:envoy_http_archive.bzl", "envoy_http_archive")
 load("@envoy_api//bazel:external_deps.bzl", "load_repository_locations")
 load(":repository_locations.bzl", "PROTOC_VERSIONS", "REPOSITORY_LOCATIONS_SPEC")
 load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language")
+load("@rules_python//python:repositories.bzl", "py_repositories")
+load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains")
 
 PPC_SKIP_TARGETS = ["envoy.filters.http.lua"]
 
@@ -364,13 +366,18 @@ def envoy_dependencies(skip_targets = []):
     _com_github_wasmtime()
     _com_github_wasm_c_api()
 
+    py_repositories()
+    rules_proto_dependencies()
+    rules_proto_toolchains()
+
     switched_rules_by_language(
         name = "com_google_googleapis_imports",
         cc = True,
         go = True,
         grpc = True,
         rules_override = {
-            "py_proto_library": ["@envoy_api//bazel:api_build_system.bzl", ""],
+            # "py_proto_library": ["@envoy_api//bazel:api_build_system.bzl", ""],
+            "cc_proto_library": ["@envoy_api//bazel:api_build_system.bzl", "",],
         },
     )
     native.bind(
@@ -936,6 +943,12 @@ def _com_google_protobuf():
         patch_args = ["-p1"],
     )
 
+    external_http_archive(
+        "com_github_protocolbuffers_protobuf",
+        patches = ["@envoy//bazel:protobuf2.patch"],
+        patch_args = ["-p1"],
+    )
+
     native.bind(
         name = "protobuf",
         actual = "@com_google_protobuf//:protobuf",

bazel/repository_locations.bzl

@@ -842,6 +842,24 @@ REPOSITORY_LOCATIONS_SPEC = dict(
         license = "Protocol Buffers",
         license_url = "https://github.com/protocolbuffers/protobuf/blob/v{version}/LICENSE",
     ),
+    com_github_protocolbuffers_protobuf = dict(
+        project_name = "Protocol Buffers",
+        project_desc = "Language-neutral, platform-neutral extensible mechanism for serializing structured data",
+        project_url = "https://developers.google.com/protocol-buffers",
+        version = PROTOBUF_VERSION,
+        # When upgrading the protobuf library, please re-run
+        # test/common/json:gen_excluded_unicodes to recompute the ranges
+        # excluded from differential fuzzing that are populated in
+        # test/common/json/json_sanitizer_test_util.cc.
+        sha256 = "a700a49470d301f1190a487a923b5095bf60f08f4ae4cac9f5f7c36883d17971",
+        strip_prefix = "protobuf-{version}",
+        urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v{version}/protobuf-{version}.tar.gz"],
+        use_category = ["dataplane_core", "controlplane"],
+        release_date = "2023-07-06",
+        cpe = "cpe:2.3:a:google:protobuf:*",
+        license = "Protocol Buffers",
+        license_url = "https://github.com/protocolbuffers/protobuf/blob/v{version}/LICENSE",
+    ),
     grpc_httpjson_transcoding = dict(
         project_name = "grpc-httpjson-transcoding",
         project_desc = "Library that supports transcoding so that HTTP/JSON can be converted to gRPC",

tools/protoprint/BUILD

@@ -32,7 +32,7 @@ envoy_pytool_binary(
         "@com_envoyproxy_protoc_gen_validate//validate:validate_py",
         "@com_github_cncf_udpa//udpa/annotations:pkg_py_proto",
         "@com_github_cncf_udpa//xds/annotations/v3:pkg_py_proto",
-        "@com_google_googleapis//google/api:annotations_py_proto",
+        "@com_google_googleapis//google/api:annotations_py_pb2",
         "@com_google_protobuf//:protobuf_python",
         "@envoy_api//envoy/annotations:pkg_py_proto",
     ],

tools/protoxform/BUILD

@@ -18,7 +18,7 @@ py_binary(
         "@com_envoyproxy_protoc_gen_validate//validate:validate_py",
         "@com_github_cncf_udpa//udpa/annotations:pkg_py_proto",
         "@com_github_cncf_udpa//xds/annotations/v3:pkg_py_proto",
-        "@com_google_googleapis//google/api:annotations_py_proto",
+        "@com_google_googleapis//google/api:annotations_py_pb2",
         "@envoy_api//envoy/annotations:pkg_py_proto",
     ],
 )