[xDS] allow C2P resolver to explicitly set the bootstrap config (#40623)
This avoids the need for the TD bootstrap generator (or a user-provided bootstrap config) to include the C2P authority. The bootstrap for C2P channels now always comes from the C2P resolver.
b/442819521
Closes #40623
COPYBARA_INTEGRATE_REVIEW=https://github.com/grpc/grpc/pull/40623 from markdroth:xds_bootstrap_override 5e5d3e22bc2792f9df47acca234243ef4619b9f8
PiperOrigin-RevId: 805559420
diff --git a/src/core/resolver/google_c2p/google_c2p_resolver.cc b/src/core/resolver/google_c2p/google_c2p_resolver.cc
index f017c66..8b24124 100644
--- a/src/core/resolver/google_c2p/google_c2p_resolver.cc
+++ b/src/core/resolver/google_c2p/google_c2p_resolver.cc
@@ -71,6 +71,7 @@
void ZoneQueryDone(std::string zone);
void IPv6QueryDone(bool ipv6_supported);
void StartXdsResolver();
+ std::shared_ptr<GrpcXdsBootstrap> ConstructBootstrap() const;
ResourceQuotaRefPtr resource_quota_;
std::shared_ptr<WorkSerializer> work_serializer_;
@@ -80,6 +81,10 @@
std::string metadata_server_name_ = "metadata.google.internal.";
bool shutdown_ = false;
+ // Used to hold state between ctor and StartLocked().
+ std::string xds_uri_;
+ ChannelArgs args_;
+
OrphanablePtr<GcpMetadataQuery> zone_query_;
std::string zone_;
};
@@ -96,13 +101,13 @@
GoogleCloud2ProdResolver::GoogleCloud2ProdResolver(ResolverArgs args)
: resource_quota_(args.args.GetObjectRef<ResourceQuota>()),
work_serializer_(std::move(args.work_serializer)),
- pollent_(grpc_polling_entity_create_from_pollset_set(args.pollset_set)) {
+ pollent_(grpc_polling_entity_create_from_pollset_set(args.pollset_set)),
+ args_(std::move(args.args)) {
absl::string_view name_to_resolve = absl::StripPrefix(args.uri.path(), "/");
// If we're not running on GCP, we can't use DirectPath, so delegate
// to the DNS resolver.
const bool test_only_pretend_running_on_gcp =
- args.args
- .GetBool("grpc.testing.google_c2p_resolver_pretend_running_on_gcp")
+ args_.GetBool("grpc.testing.google_c2p_resolver_pretend_running_on_gcp")
.value_or(false);
const bool running_on_gcp =
test_only_pretend_running_on_gcp || grpc_alts_is_running_on_gcp();
@@ -117,26 +122,25 @@
using_dns_ = true;
child_resolver_ =
CoreConfiguration::Get().resolver_registry().CreateResolver(
- absl::StrCat("dns:", name_to_resolve), args.args, args.pollset_set,
+ absl::StrCat("dns:", name_to_resolve), args_, args.pollset_set,
work_serializer_, std::move(args.result_handler));
CHECK(child_resolver_ != nullptr);
return;
}
// Maybe override metadata server name for testing
std::optional<std::string> test_only_metadata_server_override =
- args.args.GetOwnedString(
+ args_.GetOwnedString(
"grpc.testing.google_c2p_resolver_metadata_server_override");
if (test_only_metadata_server_override.has_value() &&
!test_only_metadata_server_override->empty()) {
metadata_server_name_ = std::move(*test_only_metadata_server_override);
}
// Create xds resolver.
- std::string xds_uri =
- federation_enabled
- ? absl::StrCat("xds://", kC2PAuthority, "/", name_to_resolve)
- : absl::StrCat("xds:", name_to_resolve);
+ xds_uri_ = federation_enabled
+ ? absl::StrCat("xds://", kC2PAuthority, "/", name_to_resolve)
+ : absl::StrCat("xds:", name_to_resolve);
child_resolver_ = CoreConfiguration::Get().resolver_registry().CreateResolver(
- xds_uri, args.args, args.pollset_set, work_serializer_,
+ xds_uri_, args_, args.pollset_set, work_serializer_,
std::move(args.result_handler));
CHECK(child_resolver_ != nullptr);
}
@@ -185,10 +189,8 @@
StartXdsResolver();
}
-void GoogleCloud2ProdResolver::StartXdsResolver() {
- if (shutdown_) {
- return;
- }
+std::shared_ptr<GrpcXdsBootstrap> GoogleCloud2ProdResolver::ConstructBootstrap()
+ const {
// Construct bootstrap JSON.
std::random_device rd;
std::mt19937 mt(rd());
@@ -235,8 +237,20 @@
})},
{"node", Json::FromObject(std::move(node))},
});
- // Inject bootstrap JSON as fallback config.
- internal::SetXdsFallbackBootstrapConfig(JsonDump(bootstrap).c_str());
+ return GrpcXdsBootstrap::Create(JsonDump(bootstrap)).value();
+}
+
+void GoogleCloud2ProdResolver::StartXdsResolver() {
+ if (shutdown_) return;
+ // Create XdsClient here with a custom bootstrap for C2P. When the
+ // xds resolver calls GrpcXdsClient::GetOrCreate(), it will get back
+ // the same instance rather than creating one with the normal
+ // bootstrap config from env vars.
+ static NoDestruct<std::shared_ptr<GrpcXdsBootstrap>> bootstrap(
+ ConstructBootstrap());
+ auto xds_client = GrpcXdsClient::GetOrCreate(
+ xds_uri_, args_, "google-c2p resolver", *bootstrap);
+ args_ = ChannelArgs(); // Don't need the args anymore.
// Now start xDS resolver.
child_resolver_->StartLocked();
}
diff --git a/src/core/xds/grpc/xds_client_grpc.cc b/src/core/xds/grpc/xds_client_grpc.cc
index 2a814a5..1b7f601 100644
--- a/src/core/xds/grpc/xds_client_grpc.cc
+++ b/src/core/xds/grpc/xds_client_grpc.cc
@@ -228,6 +228,22 @@
"not defined");
}
+absl::StatusOr<std::shared_ptr<GrpcXdsBootstrap>> GetOrCreateGlobalBootstrap()
+ ABSL_EXCLUSIVE_LOCKS_REQUIRED(*g_mu) {
+ if (*g_parsed_bootstrap == nullptr) {
+ // First, find bootstrap contents.
+ auto bootstrap_contents = FindBootstrapContents();
+ if (!bootstrap_contents.ok()) return bootstrap_contents.status();
+ GRPC_TRACE_LOG(xds_client, INFO)
+ << "xDS bootstrap contents: " << *bootstrap_contents;
+ // Parse bootstrap.
+ auto bootstrap = GrpcXdsBootstrap::Create(*bootstrap_contents);
+ if (!bootstrap.ok()) return bootstrap.status();
+ *g_parsed_bootstrap = std::move(*bootstrap);
+ }
+ return *g_parsed_bootstrap;
+}
+
std::shared_ptr<GlobalStatsPluginRegistry::StatsPluginGroup>
GetStatsPluginGroupForKeyAndChannelArgs(absl::string_view key,
const ChannelArgs& channel_args) {
@@ -248,7 +264,8 @@
} // namespace
absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GrpcXdsClient::GetOrCreate(
- absl::string_view key, const ChannelArgs& args, const char* reason) {
+ absl::string_view key, const ChannelArgs& args, const char* reason,
+ std::shared_ptr<GrpcXdsBootstrap> bootstrap_override) {
// If getting bootstrap from channel args, create a local XdsClient
// instance for the channel or server instead of using the global instance.
std::optional<absl::string_view> bootstrap_config = args.GetString(
@@ -277,22 +294,15 @@
}
}
// The XdsClient doesn't exist, so we'll create it.
- // If the bootstrap hasn't already been parsed, do that.
- if (*g_parsed_bootstrap == nullptr) {
- // First, find bootstrap contents.
- auto bootstrap_contents = FindBootstrapContents();
- if (!bootstrap_contents.ok()) return bootstrap_contents.status();
- GRPC_TRACE_LOG(xds_client, INFO)
- << "xDS bootstrap contents: " << *bootstrap_contents;
- // Parse bootstrap.
- auto bootstrap = GrpcXdsBootstrap::Create(*bootstrap_contents);
- if (!bootstrap.ok()) return bootstrap.status();
- *g_parsed_bootstrap = std::move(*bootstrap);
+ std::shared_ptr<GrpcXdsBootstrap> bootstrap = std::move(bootstrap_override);
+ if (bootstrap == nullptr) {
+ auto global_bootstrap = GetOrCreateGlobalBootstrap();
+ if (!global_bootstrap.ok()) return global_bootstrap.status();
+ bootstrap = std::move(*global_bootstrap);
}
- // Instantiate XdsClient.
auto channel_args = ChannelArgs::FromC(g_channel_args);
auto xds_client = MakeRefCounted<GrpcXdsClient>(
- key, *g_parsed_bootstrap, channel_args,
+ key, std::move(bootstrap), channel_args,
MakeRefCounted<GrpcXdsTransportFactory>(channel_args),
GetStatsPluginGroupForKeyAndChannelArgs(key, args));
g_xds_client_map->emplace(xds_client->key(), xds_client.get());
diff --git a/src/core/xds/grpc/xds_client_grpc.h b/src/core/xds/grpc/xds_client_grpc.h
index f412495..2bb00a1 100644
--- a/src/core/xds/grpc/xds_client_grpc.h
+++ b/src/core/xds/grpc/xds_client_grpc.h
@@ -45,8 +45,11 @@
static constexpr absl::string_view kServerKey = "#server";
// Factory function to get or create the global XdsClient instance.
+ // If bootstrap_override is null, the default bootstrap is used based
+ // on environment variables, channel args, etc.
static absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GetOrCreate(
- absl::string_view key, const ChannelArgs& args, const char* reason);
+ absl::string_view key, const ChannelArgs& args, const char* reason,
+ std::shared_ptr<GrpcXdsBootstrap> bootstrap_override = nullptr);
// Do not instantiate directly -- use GetOrCreate() instead.
// TODO(roth): The transport factory is injectable here to support
