diff --git a/api/bazel/api_build_system.bzl b/api/bazel/api_build_system.bzl index 8832aec434d21..d4c52078acd89 100644 --- a/api/bazel/api_build_system.bzl +++ b/api/bazel/api_build_system.bzl @@ -1,6 +1,6 @@ load("@com_envoyproxy_protoc_gen_validate//bazel:pgv_proto_library.bzl", "pgv_cc_proto_library") load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") -load("@com_google_protobuf//:protobuf.bzl", _py_proto_library = "py_proto_library") +load("@rules_python//python:proto.bzl", _py_proto_library = "py_proto_library") load("@io_bazel_rules_go//proto:def.bzl", "go_proto_library") load("@io_bazel_rules_go//go:def.bzl", "go_test") load("@rules_proto//proto:defs.bzl", "proto_library") @@ -59,53 +59,68 @@ def _py_proto_mapping(dep): # https://github.com/bazelbuild/bazel/issues/3935 and/or # https://github.com/bazelbuild/bazel/issues/2626 are resolved. def _api_py_proto_library(name, srcs = [], deps = []): - mapped_deps = [_py_proto_mapping(dep) for dep in deps] - mapped_unique_deps = {k: True for k in mapped_deps}.keys() _py_proto_library( name = name + _PY_PROTO_SUFFIX, - srcs = srcs, - default_runtime = "@com_google_protobuf//:protobuf_python", - protoc = "@com_google_protobuf//:protoc", - deps = mapped_unique_deps + [ - "@com_envoyproxy_protoc_gen_validate//validate:validate_py", - "@com_google_googleapis//google/rpc:status_py_proto", - "@com_google_googleapis//google/api:annotations_py_proto", - "@com_google_googleapis//google/api:http_py_proto", - "@com_google_googleapis//google/api:httpbody_py_proto", + deps = deps + [ + "@com_google_googleapis//google/api:http_proto", ], visibility = ["//visibility:public"], ) - -# This defines googleapis py_proto_library. The repository does not provide its definition and requires -# overriding it in the consuming project (see https://github.com/grpc/grpc/issues/19255 for more details). -def py_proto_library(name, deps = [], plugin = None): - srcs = [dep[:-6] + ".proto" if dep.endswith("_proto") else dep for dep in deps] - proto_deps = [] - - # py_proto_library in googleapis specifies *_proto rules in dependencies. - # By rewriting *_proto to *.proto above, the dependencies in *_proto rules are not preserved. - # As a workaround, manually specify the proto dependencies for the imported python rules. - if name == "annotations_py_proto": - proto_deps = proto_deps + [":http_py_proto"] - - # checked.proto depends on syntax.proto, we have to add this dependency manually as well. - if name == "checked_py_proto": - proto_deps = proto_deps + [":syntax_py_proto"] - - # Special handling for expr_proto target - if srcs[0] == ":expr_moved.proto": - srcs = ["checked.proto", "eval.proto", "explain.proto", "syntax.proto", "value.proto"] - proto_deps = proto_deps + ["@com_google_googleapis//google/rpc:status_py_proto"] - - # py_proto_library does not support plugin as an argument yet at gRPC v1.25.0: - # https://github.com/grpc/grpc/blob/v1.25.0/bazel/python_rules.bzl#L72. - # plugin should also be passed in here when gRPC version is greater than v1.25.x. +# mapped_deps = [_py_proto_mapping(dep) for dep in deps] +# mapped_unique_deps = {k: True for k in mapped_deps}.keys() +# _py_proto_library( +# name = name + _PY_PROTO_SUFFIX, +# srcs = srcs, +# default_runtime = "@com_google_protobuf//:protobuf_python", +# protoc = "@com_google_protobuf//:protoc", +# deps = mapped_unique_deps + [ +# "@com_envoyproxy_protoc_gen_validate//validate:validate_py", +# "@com_google_googleapis//google/rpc:status_py_proto", +# "@com_google_googleapis//google/api:annotations_py_proto", +# "@com_google_googleapis//google/api:http_py_proto", +# "@com_google_googleapis//google/api:httpbody_py_proto", +# ], +# visibility = ["//visibility:public"], +# ) + +## This defines googleapis py_proto_library. The repository does not provide its definition and requires +## overriding it in the consuming project (see https://github.com/grpc/grpc/issues/19255 for more details). +#def py_proto_library(name, deps = [], plugin = None): +# srcs = [dep[:-6] + ".proto" if dep.endswith("_proto") else dep for dep in deps] +# proto_deps = [] +# +# # py_proto_library in googleapis specifies *_proto rules in dependencies. +# # By rewriting *_proto to *.proto above, the dependencies in *_proto rules are not preserved. +# # As a workaround, manually specify the proto dependencies for the imported python rules. +# if name == "annotations_py_proto": +# proto_deps = proto_deps + [":http_py_proto"] +# +# # checked.proto depends on syntax.proto, we have to add this dependency manually as well. +# if name == "checked_py_proto": +# proto_deps = proto_deps + [":syntax_py_proto"] +# +# # Special handling for expr_proto target +# if srcs[0] == ":expr_moved.proto": +# srcs = ["checked.proto", "eval.proto", "explain.proto", "syntax.proto", "value.proto"] +# proto_deps = proto_deps + ["@com_google_googleapis//google/rpc:status_py_proto"] +# +# # py_proto_library does not support plugin as an argument yet at gRPC v1.25.0: +# # https://github.com/grpc/grpc/blob/v1.25.0/bazel/python_rules.bzl#L72. +# # plugin should also be passed in here when gRPC version is greater than v1.25.x. +# _py_proto_library( +# name = name, +# srcs = srcs, +# default_runtime = "@com_google_protobuf//:protobuf_python", +# protoc = "@com_google_protobuf//:protoc", +# deps = proto_deps + ["@com_google_protobuf//:protobuf_python"], +# visibility = ["//visibility:public"], +# ) + +def cc_proto_library(name, deps = [], **kwargs): + native.cc_proto_library(name=name, deps=deps, **kwargs) _py_proto_library( - name = name, - srcs = srcs, - default_runtime = "@com_google_protobuf//:protobuf_python", - protoc = "@com_google_protobuf//:protoc", - deps = proto_deps + ["@com_google_protobuf//:protobuf_python"], + name = name.replace('_cc_proto', '') + "_py_pb2", + deps = deps, visibility = ["//visibility:public"], ) @@ -157,7 +172,6 @@ def api_cc_py_proto_library( deps = [relative_name], visibility = ["//visibility:public"], ) - _api_py_proto_library(name, srcs, deps) # Optionally define gRPC services if has_services: @@ -196,6 +210,8 @@ def api_proto_package( has_services = has_services, ) + _api_py_proto_library(name = name, srcs = srcs, deps = deps) + compilers = ["@io_bazel_rules_go//proto:go_proto", "@envoy_api//bazel:pgv_plugin_go"] if has_services: compilers = ["@io_bazel_rules_go//proto:go_grpc", "@envoy_api//bazel:pgv_plugin_go"] diff --git a/api/bazel/repositories.bzl b/api/bazel/repositories.bzl index c6ab16fc1fbfa..fed5619809c60 100644 --- a/api/bazel/repositories.bzl +++ b/api/bazel/repositories.bzl @@ -36,6 +36,9 @@ def api_dependencies(): external_http_archive( name = "opencensus_proto", ) + external_http_archive( + name = "rules_python", + ) external_http_archive( name = "rules_proto", ) @@ -56,6 +59,10 @@ def api_dependencies(): name = "com_github_chrusty_protoc_gen_jsonschema", ) + external_http_archive( + name = "dev_cel", + ) + external_http_archive( name = "envoy_toolshed", ) diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index 6ed9c130450e4..99702ed2215d5 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -39,11 +39,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_desc = "xDS API Working Group (xDS-WG)", project_url = "https://github.com/cncf/xds", # During the UDPA -> xDS migration, we aren't working with releases. - version = "e9ce68804cb4e64cab5a52e3c8baf840d4ff87b7", - sha256 = "0d33b83f8c6368954e72e7785539f0d272a8aba2f6e2e336ed15fd1514bc9899", + version = "expression-new-fields", + sha256 = "5144c11e16044df6d9152a3bac56974c7d8c462c88578e33d8f9e03cd03ffce6", release_date = "2023-06-07", - strip_prefix = "xds-{version}", - urls = ["https://github.com/cncf/xds/archive/{version}.tar.gz"], + strip_prefix = "xds-cel-expression-new-fields", + urls = ["https://github.com/sergiitk/xds/archive/refs/heads/cel-{version}.zip"], use_category = ["api"], license = "Apache-2.0", license_url = "https://github.com/cncf/xds/blob/{version}/LICENSE", @@ -101,13 +101,26 @@ REPOSITORY_LOCATIONS_SPEC = dict( license = "Apache-2.0", license_url = "https://github.com/prometheus/client_model/blob/v{version}/LICENSE", ), + rules_python = dict( + project_name = "Bazel Python Rules", + project_desc = "Core Python rules -- py_library, py_binary, py_test, py_proto_library", + project_url = "https://github.com/bazelbuild/rules_python", + version = "0.26.0", + sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b", + release_date = "2023-10-05", + strip_prefix = "rules_python-{version}", + urls = ["https://github.com/bazelbuild/rules_python/releases/download/{version}.tar.gz"], + use_category = ["api"], + license = "Apache-2.0", + license_url = "https://github.com/bazelbuild/rules_proto/blob/{version}/LICENSE", + ), rules_proto = dict( project_name = "Protobuf Rules for Bazel", project_desc = "Protocol buffer rules for Bazel", project_url = "https://github.com/bazelbuild/rules_proto", - version = "4.0.0", - sha256 = "66bfdf8782796239d3875d37e7de19b1d94301e8972b3cbd2446b332429b4df1", - release_date = "2021-09-15", + version = "5.3.0-21.7", + sha256 = "dc3fb206a2cb3441b485eb1e423165b231235a1ea9b031b4433cf7bc1fa460dd", + release_date = "2022-12-27", strip_prefix = "rules_proto-{version}", urls = ["https://github.com/bazelbuild/rules_proto/archive/refs/tags/{version}.tar.gz"], use_category = ["api"], @@ -151,6 +164,17 @@ REPOSITORY_LOCATIONS_SPEC = dict( use_category = ["build"], release_date = "2023-05-30", ), + dev_cel = dict( + project_name = "CEL", + project_desc = "Common Expression Language -- specification and binary representation", + project_url = "https://github.com/google/cel-spec", + strip_prefix = "cel-spec-{version}", + sha256 = "3de60ea3a29b6246faf04d206b7ee4633c155042e040086205c83edf713aee29", + version = "2657e88023e5e7dfbb62e74de7a7cfd6e8284d7b", + urls = ["https://github.com/google/cel-spec/archive/{version}.zip"], + use_category = ["api"], + release_date = "2023-11-09", + ), envoy_toolshed = dict( project_name = "envoy_toolshed", project_desc = "Tooling, libraries, runners and checkers for Envoy proxy's CI", diff --git a/bazel/protobuf2.patch b/bazel/protobuf2.patch new file mode 100644 index 0000000000000..08f71b48cc6c5 --- /dev/null +++ b/bazel/protobuf2.patch @@ -0,0 +1,90 @@ +diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py +index e7555ee10..a93beb1c5 100644 +--- a/python/google/protobuf/__init__.py ++++ b/python/google/protobuf/__init__.py +@@ -31,3 +31,10 @@ + # Copyright 2007 Google Inc. All Rights Reserved. + + __version__ = '4.23.4' ++ ++ ++if __name__ != '__main__': ++ try: ++ __import__('pkg_resources').declare_namespace(__name__) ++ except ImportError: ++ __path__ = __import__('pkgutil').extend_path(__path__, __name__) +diff --git a/src/google/protobuf/io/BUILD.bazel b/src/google/protobuf/io/BUILD.bazel +--- a/src/google/protobuf/io/BUILD.bazel ++++ b/src/google/protobuf/io/BUILD.bazel +@@ -138,7 +138,7 @@ cc_library( + "@com_google_absl//absl/log:absl_log", + ] + select({ + "//build_defs:config_msvc": [], +- "//conditions:default": ["@zlib//:zlib"], ++ "//conditions:default": ["//external:zlib"], + }), + ) + +diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc +--- a/src/google/protobuf/port_def.inc 2023-06-27 01:17:34.917105764 +0000 ++++ b/src/google/protobuf/port_def.inc 2023-06-27 01:18:12.069060142 +0000 +@@ -1004,7 +1004,7 @@ + #pragma clang diagnostic ignored "-Wshorten-64-to-32" + // Turn on -Wdeprecated-enum-enum-conversion. This deprecation comes in C++20 + // via http://wg21.link/p1120r0. +-#pragma clang diagnostic error "-Wdeprecated-enum-enum-conversion" ++// #pragma clang diagnostic error "-Wdeprecated-enum-enum-conversion" + // This error has been generally flaky, but we need to disable it specifically + // to fix https://github.com/protocolbuffers/protobuf/issues/12313 + #pragma clang diagnostic ignored "-Wunused-parameter" +diff --git a/src/google/protobuf/map.h b/src/google/protobuf/map.h +--- a/src/google/protobuf/map.h 2023-06-30 14:32:34.892148233 +0000 ++++ b/src/google/protobuf/map.h 2023-06-30 14:35:37.447992493 +0000 +@@ -883,7 +883,7 @@ + TreeConvert(b); + } + ABSL_DCHECK(TableEntryIsTree(b)) +- << (void*)table_[b] << " " << (uintptr_t)table_[b]; ++ << reinterpret_cast(table_[b]) << " " << static_cast(table_[b]); + InsertUniqueInTree(b, node); + index_of_first_non_null_ = (std::min)(index_of_first_non_null_, b); + } +diff --git a/src/google/protobuf/map_field.h b/src/google/protobuf/map_field.h +--- a/src/google/protobuf/map_field.h 2023-06-30 17:14:18.934528580 +0000 ++++ b/src/google/protobuf/map_field.h 2023-06-30 17:14:52.098500807 +0000 +@@ -345,7 +345,7 @@ + + protected: + // "protected" stops users from deleting a `MapFieldBase *` +- ~MapFieldBase(); ++ virtual ~MapFieldBase(); + + public: + // Returns reference to internal repeated field. Data written using +diff --git a/src/google/protobuf/compiler/BUILD.bazel b/src/google/protobuf/compiler/BUILD.bazel +--- a/src/google/protobuf/compiler/BUILD.bazel ++++ b/src/google/protobuf/compiler/BUILD.bazel +@@ -306,7 +306,7 @@ + srcs = ["retention.cc"], + hdrs = ["retention.h"], + include_prefix = "google/protobuf/compiler", +- visibility = ["//src/google/protobuf:__subpackages__"], ++ visibility = ["//visibility:public"], + deps = [ + "//src/google/protobuf:protobuf_nowkt", + "@com_google_absl//absl/types:span", + +diff --git a/src/google/protobuf/port_def.inc b/src/google/protobuf/port_def.inc +index 1c6a24945..c27d0bf2a 100644 +--- a/src/google/protobuf/port_def.inc ++++ b/src/google/protobuf/port_def.inc +@@ -1062,6 +1062,9 @@ static_assert(PROTOBUF_ABSL_MIN(20230125, 3), + #pragma warning(disable: 4125) + #endif + ++#if defined(__GNUC__) ++#pragma GCC diagnostic ignored "-Wundef" ++#endif + #if PROTOBUF_ENABLE_DEBUG_LOGGING_MAY_LEAK_PII + #define PROTOBUF_DEBUG true + #else diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index a28ce2e347adf..f70706b1d4367 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -3,6 +3,8 @@ load("@envoy_api//bazel:envoy_http_archive.bzl", "envoy_http_archive") load("@envoy_api//bazel:external_deps.bzl", "load_repository_locations") load(":repository_locations.bzl", "PROTOC_VERSIONS", "REPOSITORY_LOCATIONS_SPEC") load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") +load("@rules_python//python:repositories.bzl", "py_repositories") +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") PPC_SKIP_TARGETS = ["envoy.filters.http.lua"] @@ -364,13 +366,18 @@ def envoy_dependencies(skip_targets = []): _com_github_wasmtime() _com_github_wasm_c_api() + py_repositories() + rules_proto_dependencies() + rules_proto_toolchains() + switched_rules_by_language( name = "com_google_googleapis_imports", cc = True, go = True, grpc = True, rules_override = { - "py_proto_library": ["@envoy_api//bazel:api_build_system.bzl", ""], + # "py_proto_library": ["@envoy_api//bazel:api_build_system.bzl", ""], + "cc_proto_library": ["@envoy_api//bazel:api_build_system.bzl", "",], }, ) native.bind( @@ -936,6 +943,12 @@ def _com_google_protobuf(): patch_args = ["-p1"], ) + external_http_archive( + "com_github_protocolbuffers_protobuf", + patches = ["@envoy//bazel:protobuf2.patch"], + patch_args = ["-p1"], + ) + native.bind( name = "protobuf", actual = "@com_google_protobuf//:protobuf", diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 93635b523a57f..f59056a0eabb5 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -842,6 +842,24 @@ REPOSITORY_LOCATIONS_SPEC = dict( license = "Protocol Buffers", license_url = "https://github.com/protocolbuffers/protobuf/blob/v{version}/LICENSE", ), + com_github_protocolbuffers_protobuf = dict( + project_name = "Protocol Buffers", + project_desc = "Language-neutral, platform-neutral extensible mechanism for serializing structured data", + project_url = "https://developers.google.com/protocol-buffers", + version = PROTOBUF_VERSION, + # When upgrading the protobuf library, please re-run + # test/common/json:gen_excluded_unicodes to recompute the ranges + # excluded from differential fuzzing that are populated in + # test/common/json/json_sanitizer_test_util.cc. + sha256 = "a700a49470d301f1190a487a923b5095bf60f08f4ae4cac9f5f7c36883d17971", + strip_prefix = "protobuf-{version}", + urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v{version}/protobuf-{version}.tar.gz"], + use_category = ["dataplane_core", "controlplane"], + release_date = "2023-07-06", + cpe = "cpe:2.3:a:google:protobuf:*", + license = "Protocol Buffers", + license_url = "https://github.com/protocolbuffers/protobuf/blob/v{version}/LICENSE", + ), grpc_httpjson_transcoding = dict( project_name = "grpc-httpjson-transcoding", project_desc = "Library that supports transcoding so that HTTP/JSON can be converted to gRPC", diff --git a/tools/protoprint/BUILD b/tools/protoprint/BUILD index 37659b927bcf5..213e1afcccbef 100644 --- a/tools/protoprint/BUILD +++ b/tools/protoprint/BUILD @@ -32,7 +32,7 @@ envoy_pytool_binary( "@com_envoyproxy_protoc_gen_validate//validate:validate_py", "@com_github_cncf_udpa//udpa/annotations:pkg_py_proto", "@com_github_cncf_udpa//xds/annotations/v3:pkg_py_proto", - "@com_google_googleapis//google/api:annotations_py_proto", + "@com_google_googleapis//google/api:annotations_py_pb2", "@com_google_protobuf//:protobuf_python", "@envoy_api//envoy/annotations:pkg_py_proto", ], diff --git a/tools/protoxform/BUILD b/tools/protoxform/BUILD index 73c95708fe52d..cc6e9e192c9c6 100644 --- a/tools/protoxform/BUILD +++ b/tools/protoxform/BUILD @@ -18,7 +18,7 @@ py_binary( "@com_envoyproxy_protoc_gen_validate//validate:validate_py", "@com_github_cncf_udpa//udpa/annotations:pkg_py_proto", "@com_github_cncf_udpa//xds/annotations/v3:pkg_py_proto", - "@com_google_googleapis//google/api:annotations_py_proto", + "@com_google_googleapis//google/api:annotations_py_pb2", "@envoy_api//envoy/annotations:pkg_py_proto", ], )