1 //
2 // Copyright 2020 gRPC authors.
3 //
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
7 //
8 //     http://www.apache.org/licenses/LICENSE-2.0
9 //
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
15 //
16 
17 #include "src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h"
18 
19 #include <gmock/gmock.h>
20 #include <grpc/support/alloc.h>
21 #include <grpc/support/log.h>
22 #include <grpc/support/string_util.h>
23 #include <gtest/gtest.h>
24 
25 #include <deque>
26 #include <list>
27 #include <string>
28 #include <thread>
29 
30 #include "src/core/lib/slice/slice_internal.h"
31 #include "test/core/util/test_config.h"
32 #include "test/core/util/tls_utils.h"
33 
34 namespace grpc_core {
35 
36 namespace testing {
37 
38 constexpr const char* kCertName1 = "cert_1_name";
39 constexpr const char* kCertName2 = "cert_2_name";
40 constexpr const char* kRootCert1Name = "root_cert_1_name";
41 constexpr const char* kRootCert1Contents = "root_cert_1_contents";
42 constexpr const char* kRootCert2Name = "root_cert_2_name";
43 constexpr const char* kRootCert2Contents = "root_cert_2_contents";
44 constexpr const char* kIdentityCert1Name = "identity_cert_1_name";
45 constexpr const char* kIdentityCert1PrivateKey = "identity_private_key_1";
46 constexpr const char* kIdentityCert1Contents = "identity_cert_1_contents";
47 constexpr const char* kIdentityCert2Name = "identity_cert_2_name";
48 constexpr const char* kIdentityCert2PrivateKey = "identity_private_key_2";
49 constexpr const char* kIdentityCert2Contents = "identity_cert_2_contents";
50 constexpr const char* kErrorMessage = "error_message";
51 constexpr const char* kRootErrorMessage = "root_error_message";
52 constexpr const char* kIdentityErrorMessage = "identity_error_message";
53 
54 class GrpcTlsCertificateDistributorTest : public ::testing::Test {
55  protected:
56   // Forward declaration.
57   class TlsCertificatesTestWatcher;
58 
59   // CredentialInfo contains the parameters when calling OnCertificatesChanged
60   // of a watcher. When OnCertificatesChanged is invoked, we will push a
61   // CredentialInfo to the cert_update_queue of state_, and check in each test
62   // if the status updates are correct.
63   struct CredentialInfo {
64     std::string root_certs;
65     PemKeyCertPairList key_cert_pairs;
CredentialInfogrpc_core::testing::GrpcTlsCertificateDistributorTest::CredentialInfo66     CredentialInfo(std::string root, PemKeyCertPairList key_cert)
67         : root_certs(std::move(root)), key_cert_pairs(std::move(key_cert)) {}
operator ==grpc_core::testing::GrpcTlsCertificateDistributorTest::CredentialInfo68     bool operator==(const CredentialInfo& other) const {
69       return root_certs == other.root_certs &&
70              key_cert_pairs == other.key_cert_pairs;
71     }
72   };
73 
74   // ErrorInfo contains the parameters when calling OnError of a watcher. When
75   // OnError is invoked, we will push a ErrorInfo to the error_queue of state_,
76   // and check in each test if the status updates are correct.
77   struct ErrorInfo {
78     std::string root_cert_str;
79     std::string identity_cert_str;
ErrorInfogrpc_core::testing::GrpcTlsCertificateDistributorTest::ErrorInfo80     ErrorInfo(std::string root, std::string identity)
81         : root_cert_str(std::move(root)),
82           identity_cert_str(std::move(identity)) {}
operator ==grpc_core::testing::GrpcTlsCertificateDistributorTest::ErrorInfo83     bool operator==(const ErrorInfo& other) const {
84       return root_cert_str == other.root_cert_str &&
85              identity_cert_str == other.identity_cert_str;
86     }
87   };
88 
89   struct WatcherState {
90     TlsCertificatesTestWatcher* watcher = nullptr;
91     std::deque<CredentialInfo> cert_update_queue;
92     std::deque<ErrorInfo> error_queue;
93 
GetCredentialQueuegrpc_core::testing::GrpcTlsCertificateDistributorTest::WatcherState94     std::deque<CredentialInfo> GetCredentialQueue() {
95       // We move the data member value so the data member will be re-initiated
96       // with size 0, and ready for the next check.
97       return std::move(cert_update_queue);
98     }
GetErrorQueuegrpc_core::testing::GrpcTlsCertificateDistributorTest::WatcherState99     std::deque<ErrorInfo> GetErrorQueue() {
100       // We move the data member value so the data member will be re-initiated
101       // with size 0, and ready for the next check.
102       return std::move(error_queue);
103     }
104   };
105 
106   class TlsCertificatesTestWatcher : public grpc_tls_certificate_distributor::
107                                          TlsCertificatesWatcherInterface {
108    public:
109     // ctor sets state->watcher to this.
TlsCertificatesTestWatcher(WatcherState * state)110     explicit TlsCertificatesTestWatcher(WatcherState* state) : state_(state) {
111       state_->watcher = this;
112     }
113 
114     // dtor sets state->watcher to nullptr.
~TlsCertificatesTestWatcher()115     ~TlsCertificatesTestWatcher() override { state_->watcher = nullptr; }
116 
OnCertificatesChanged(absl::optional<absl::string_view> root_certs,absl::optional<PemKeyCertPairList> key_cert_pairs)117     void OnCertificatesChanged(
118         absl::optional<absl::string_view> root_certs,
119         absl::optional<PemKeyCertPairList> key_cert_pairs) override {
120       std::string updated_root;
121       if (root_certs.has_value()) {
122         updated_root = std::string(*root_certs);
123       }
124       PemKeyCertPairList updated_identity;
125       if (key_cert_pairs.has_value()) {
126         updated_identity = std::move(*key_cert_pairs);
127       }
128       state_->cert_update_queue.emplace_back(std::move(updated_root),
129                                              std::move(updated_identity));
130     }
131 
OnError(grpc_error * root_cert_error,grpc_error * identity_cert_error)132     void OnError(grpc_error* root_cert_error,
133                  grpc_error* identity_cert_error) override {
134       GPR_ASSERT(root_cert_error != GRPC_ERROR_NONE ||
135                  identity_cert_error != GRPC_ERROR_NONE);
136       std::string root_error_str;
137       std::string identity_error_str;
138       if (root_cert_error != GRPC_ERROR_NONE) {
139         grpc_slice root_error_slice;
140         GPR_ASSERT(grpc_error_get_str(
141             root_cert_error, GRPC_ERROR_STR_DESCRIPTION, &root_error_slice));
142         root_error_str = std::string(StringViewFromSlice(root_error_slice));
143       }
144       if (identity_cert_error != GRPC_ERROR_NONE) {
145         grpc_slice identity_error_slice;
146         GPR_ASSERT(grpc_error_get_str(identity_cert_error,
147                                       GRPC_ERROR_STR_DESCRIPTION,
148                                       &identity_error_slice));
149         identity_error_str =
150             std::string(StringViewFromSlice(identity_error_slice));
151       }
152       state_->error_queue.emplace_back(std::move(root_error_str),
153                                        std::move(identity_error_str));
154       GRPC_ERROR_UNREF(root_cert_error);
155       GRPC_ERROR_UNREF(identity_cert_error);
156     }
157 
158    private:
159     WatcherState* state_;
160   };
161 
162   // CallbackStatus contains the parameters when calling watch_status_callback_
163   // of the distributor. When a particular callback is invoked, we will push a
164   // CallbackStatus to a callback_queue_, and check in each test if the status
165   // updates are correct.
166   struct CallbackStatus {
167     std::string cert_name;
168     bool root_being_watched;
169     bool identity_being_watched;
CallbackStatusgrpc_core::testing::GrpcTlsCertificateDistributorTest::CallbackStatus170     CallbackStatus(std::string name, bool root_watched, bool identity_watched)
171         : cert_name(std::move(name)),
172           root_being_watched(root_watched),
173           identity_being_watched(identity_watched) {}
operator ==grpc_core::testing::GrpcTlsCertificateDistributorTest::CallbackStatus174     bool operator==(const CallbackStatus& other) const {
175       return cert_name == other.cert_name &&
176              root_being_watched == other.root_being_watched &&
177              identity_being_watched == other.identity_being_watched;
178     }
179   };
180 
SetUp()181   void SetUp() override {
182     distributor_.SetWatchStatusCallback([this](std::string cert_name,
183                                                bool root_being_watched,
184                                                bool identity_being_watched) {
185       callback_queue_.emplace_back(std::move(cert_name), root_being_watched,
186                                    identity_being_watched);
187     });
188   }
189 
MakeWatcher(absl::optional<std::string> root_cert_name,absl::optional<std::string> identity_cert_name)190   WatcherState* MakeWatcher(absl::optional<std::string> root_cert_name,
191                             absl::optional<std::string> identity_cert_name) {
192     MutexLock lock(&mu_);
193     watchers_.emplace_back();
194     // TlsCertificatesTestWatcher ctor takes a pointer to the WatcherState.
195     // It sets WatcherState::watcher to point to itself.
196     // The TlsCertificatesTestWatcher dtor will set WatcherState::watcher back
197     // to nullptr to indicate that it's been destroyed.
198     auto watcher =
199         absl::make_unique<TlsCertificatesTestWatcher>(&watchers_.back());
200     distributor_.WatchTlsCertificates(std::move(watcher),
201                                       std::move(root_cert_name),
202                                       std::move(identity_cert_name));
203     return &watchers_.back();
204   }
205 
CancelWatch(WatcherState * state)206   void CancelWatch(WatcherState* state) {
207     MutexLock lock(&mu_);
208     distributor_.CancelTlsCertificatesWatch(state->watcher);
209     EXPECT_EQ(state->watcher, nullptr);
210   }
211 
GetCallbackQueue()212   std::deque<CallbackStatus> GetCallbackQueue() {
213     // We move the data member value so the data member will be re-initiated
214     // with size 0, and ready for the next check.
215     return std::move(callback_queue_);
216   }
217 
218   grpc_tls_certificate_distributor distributor_;
219   // Use a std::list<> here to avoid the address invalidation caused by internal
220   // reallocation of std::vector<>.
221   std::list<WatcherState> watchers_;
222   std::deque<CallbackStatus> callback_queue_;
223   // This is to make watchers_ and callback_queue_ thread-safe.
224   Mutex mu_;
225 };
226 
TEST_F(GrpcTlsCertificateDistributorTest,BasicCredentialBehaviors)227 TEST_F(GrpcTlsCertificateDistributorTest, BasicCredentialBehaviors) {
228   EXPECT_FALSE(distributor_.HasRootCerts(kRootCert1Name));
229   EXPECT_FALSE(distributor_.HasKeyCertPairs(kIdentityCert1Name));
230   // After setting the certificates to the corresponding cert names, the
231   // distributor should possess the corresponding certs.
232   distributor_.SetKeyMaterials(kRootCert1Name, kRootCert1Contents,
233                                absl::nullopt);
234   EXPECT_TRUE(distributor_.HasRootCerts(kRootCert1Name));
235   distributor_.SetKeyMaterials(
236       kIdentityCert1Name, absl::nullopt,
237       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
238   EXPECT_TRUE(distributor_.HasKeyCertPairs(kIdentityCert1Name));
239   // Querying a non-existing cert name should return false.
240   EXPECT_FALSE(distributor_.HasRootCerts(kRootCert2Name));
241   EXPECT_FALSE(distributor_.HasKeyCertPairs(kIdentityCert2Name));
242 }
243 
TEST_F(GrpcTlsCertificateDistributorTest,UpdateCredentialsOnAnySide)244 TEST_F(GrpcTlsCertificateDistributorTest, UpdateCredentialsOnAnySide) {
245   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
246   EXPECT_THAT(GetCallbackQueue(),
247               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
248   // SetKeyMaterials should trigger watcher's OnCertificatesChanged method.
249   distributor_.SetKeyMaterials(
250       kCertName1, kRootCert1Contents,
251       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
252   EXPECT_THAT(
253       watcher_state_1->GetCredentialQueue(),
254       ::testing::ElementsAre(CredentialInfo(
255           kRootCert1Contents,
256           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
257   // Set root certs should trigger watcher's OnCertificatesChanged again.
258   distributor_.SetKeyMaterials(kCertName1, kRootCert2Contents, absl::nullopt);
259   EXPECT_THAT(
260       watcher_state_1->GetCredentialQueue(),
261       ::testing::ElementsAre(CredentialInfo(
262           kRootCert2Contents,
263           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
264   // Set identity certs should trigger watcher's OnCertificatesChanged again.
265   distributor_.SetKeyMaterials(
266       kCertName1, absl::nullopt,
267       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
268   EXPECT_THAT(
269       watcher_state_1->GetCredentialQueue(),
270       ::testing::ElementsAre(CredentialInfo(
271           kRootCert2Contents,
272           MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents))));
273   CancelWatch(watcher_state_1);
274 }
275 
TEST_F(GrpcTlsCertificateDistributorTest,SameIdentityNameDiffRootName)276 TEST_F(GrpcTlsCertificateDistributorTest, SameIdentityNameDiffRootName) {
277   // Register watcher 1.
278   WatcherState* watcher_state_1 =
279       MakeWatcher(kRootCert1Name, kIdentityCert1Name);
280   EXPECT_THAT(
281       GetCallbackQueue(),
282       ::testing::ElementsAre(CallbackStatus(kRootCert1Name, true, false),
283                              CallbackStatus(kIdentityCert1Name, false, true)));
284   // Register watcher 2.
285   WatcherState* watcher_state_2 =
286       MakeWatcher(kRootCert2Name, kIdentityCert1Name);
287   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre(CallbackStatus(
288                                       kRootCert2Name, true, false)));
289   // Push credential updates to kRootCert1Name and check if the status works as
290   // expected.
291   distributor_.SetKeyMaterials(kRootCert1Name, kRootCert1Contents,
292                                absl::nullopt);
293   // Check the updates are delivered to watcher 1.
294   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
295               ::testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
296   // Push credential updates to kRootCert2Name.
297   distributor_.SetKeyMaterials(kRootCert2Name, kRootCert2Contents,
298                                absl::nullopt);
299   // Check the updates are delivered to watcher 2.
300   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
301               ::testing::ElementsAre(CredentialInfo(kRootCert2Contents, {})));
302   // Push credential updates to kIdentityCert1Name and check if the status works
303   // as expected.
304   distributor_.SetKeyMaterials(
305       kIdentityCert1Name, absl::nullopt,
306       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
307   // Check the updates are delivered to watcher 1 and watcher 2.
308   EXPECT_THAT(
309       watcher_state_1->GetCredentialQueue(),
310       ::testing::ElementsAre(CredentialInfo(
311           kRootCert1Contents,
312           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
313   EXPECT_THAT(
314       watcher_state_2->GetCredentialQueue(),
315       ::testing::ElementsAre(CredentialInfo(
316           kRootCert2Contents,
317           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
318   // Cancel watcher 1.
319   CancelWatch(watcher_state_1);
320   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre(CallbackStatus(
321                                       kRootCert1Name, false, false)));
322   // Cancel watcher 2.
323   CancelWatch(watcher_state_2);
324   EXPECT_THAT(
325       GetCallbackQueue(),
326       ::testing::ElementsAre(CallbackStatus(kRootCert2Name, false, false),
327                              CallbackStatus(kIdentityCert1Name, false, false)));
328 }
329 
TEST_F(GrpcTlsCertificateDistributorTest,SameRootNameDiffIdentityName)330 TEST_F(GrpcTlsCertificateDistributorTest, SameRootNameDiffIdentityName) {
331   // Register watcher 1.
332   WatcherState* watcher_state_1 =
333       MakeWatcher(kRootCert1Name, kIdentityCert1Name);
334   EXPECT_THAT(
335       GetCallbackQueue(),
336       ::testing::ElementsAre(CallbackStatus(kRootCert1Name, true, false),
337                              CallbackStatus(kIdentityCert1Name, false, true)));
338   // Register watcher 2.
339   WatcherState* watcher_state_2 =
340       MakeWatcher(kRootCert1Name, kIdentityCert2Name);
341   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre(CallbackStatus(
342                                       kIdentityCert2Name, false, true)));
343   // Push credential updates to kRootCert1Name and check if the status works as
344   // expected.
345   distributor_.SetKeyMaterials(kRootCert1Name, kRootCert1Contents,
346                                absl::nullopt);
347   // Check the updates are delivered to watcher 1.
348   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
349               ::testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
350   // Check the updates are delivered to watcher 2.
351   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
352               ::testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
353   // Push credential updates to SetKeyMaterials.
354   distributor_.SetKeyMaterials(
355       kIdentityCert1Name, absl::nullopt,
356       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
357   // Check the updates are delivered to watcher 1.
358   EXPECT_THAT(
359       watcher_state_1->GetCredentialQueue(),
360       ::testing::ElementsAre(CredentialInfo(
361           kRootCert1Contents,
362           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
363   // Push credential updates to kIdentityCert2Name.
364   distributor_.SetKeyMaterials(
365       kIdentityCert2Name, absl::nullopt,
366       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
367   // Check the updates are delivered to watcher 2.
368   EXPECT_THAT(
369       watcher_state_2->GetCredentialQueue(),
370       ::testing::ElementsAre(CredentialInfo(
371           kRootCert1Contents,
372           MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents))));
373   // Cancel watcher 1.
374   CancelWatch(watcher_state_1);
375   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre(CallbackStatus(
376                                       kIdentityCert1Name, false, false)));
377   // Cancel watcher 2.
378   CancelWatch(watcher_state_2);
379   EXPECT_THAT(
380       GetCallbackQueue(),
381       ::testing::ElementsAre(CallbackStatus(kRootCert1Name, false, false),
382                              CallbackStatus(kIdentityCert2Name, false, false)));
383 }
384 
TEST_F(GrpcTlsCertificateDistributorTest,AddAndCancelFirstWatcherForSameRootAndIdentityCertName)385 TEST_F(GrpcTlsCertificateDistributorTest,
386        AddAndCancelFirstWatcherForSameRootAndIdentityCertName) {
387   // Register watcher 1 watching kCertName1 for both root and identity certs.
388   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
389   EXPECT_THAT(GetCallbackQueue(),
390               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
391   // Push credential updates to kCertName1 and check if the status works as
392   // expected.
393   distributor_.SetKeyMaterials(
394       kCertName1, kRootCert1Contents,
395       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
396   // Check the updates are delivered to watcher 1.
397   EXPECT_THAT(
398       watcher_state_1->GetCredentialQueue(),
399       ::testing::ElementsAre(CredentialInfo(
400           kRootCert1Contents,
401           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
402   // Cancel watcher 1.
403   CancelWatch(watcher_state_1);
404   EXPECT_THAT(GetCallbackQueue(),
405               ::testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
406 }
407 
TEST_F(GrpcTlsCertificateDistributorTest,AddAndCancelFirstWatcherForIdentityCertNameWithRootBeingWatched)408 TEST_F(GrpcTlsCertificateDistributorTest,
409        AddAndCancelFirstWatcherForIdentityCertNameWithRootBeingWatched) {
410   // Register watcher 1 watching kCertName1 for root certs.
411   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, absl::nullopt);
412   EXPECT_THAT(GetCallbackQueue(),
413               ::testing::ElementsAre(CallbackStatus(kCertName1, true, false)));
414   // Register watcher 2 watching kCertName1 for identity certs.
415   WatcherState* watcher_state_2 = MakeWatcher(absl::nullopt, kCertName1);
416   EXPECT_THAT(GetCallbackQueue(),
417               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
418   // Push credential updates to kCertName1 and check if the status works as
419   // expected.
420   distributor_.SetKeyMaterials(
421       kCertName1, kRootCert1Contents,
422       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
423   // Check the updates are delivered to watcher 1.
424   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
425               ::testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
426   // Check the updates are delivered to watcher 2.
427   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
428               ::testing::ElementsAre(CredentialInfo(
429                   "", MakeCertKeyPairs(kIdentityCert1PrivateKey,
430                                        kIdentityCert1Contents))));
431   // Push root cert updates to kCertName1.
432   distributor_.SetKeyMaterials(kCertName1, kRootCert2Contents, absl::nullopt);
433   // Check the updates are delivered to watcher 1.
434   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
435               ::testing::ElementsAre(CredentialInfo(kRootCert2Contents, {})));
436   // Check the updates are not delivered to watcher 2.
437   EXPECT_THAT(watcher_state_2->GetCredentialQueue(), ::testing::ElementsAre());
438   // Push identity cert updates to kCertName1.
439   distributor_.SetKeyMaterials(
440       kCertName1, absl::nullopt,
441       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
442   // Check the updates are not delivered to watcher 1.
443   EXPECT_THAT(watcher_state_1->GetCredentialQueue(), ::testing::ElementsAre());
444   // Check the updates are delivered to watcher 2.
445   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
446               ::testing::ElementsAre(CredentialInfo(
447                   "", MakeCertKeyPairs(kIdentityCert2PrivateKey,
448                                        kIdentityCert2Contents))));
449   watcher_state_2->cert_update_queue.clear();
450   // Cancel watcher 2.
451   CancelWatch(watcher_state_2);
452   EXPECT_THAT(GetCallbackQueue(),
453               ::testing::ElementsAre(CallbackStatus(kCertName1, true, false)));
454   // Cancel watcher 1.
455   CancelWatch(watcher_state_1);
456   EXPECT_THAT(GetCallbackQueue(),
457               ::testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
458 }
459 
TEST_F(GrpcTlsCertificateDistributorTest,AddAndCancelFirstWatcherForRootCertNameWithIdentityBeingWatched)460 TEST_F(GrpcTlsCertificateDistributorTest,
461        AddAndCancelFirstWatcherForRootCertNameWithIdentityBeingWatched) {
462   // Register watcher 1 watching kCertName1 for identity certs.
463   WatcherState* watcher_state_1 = MakeWatcher(absl::nullopt, kCertName1);
464   EXPECT_THAT(GetCallbackQueue(),
465               ::testing::ElementsAre(CallbackStatus(kCertName1, false, true)));
466   // Register watcher 2 watching kCertName1 for root certs.
467   WatcherState* watcher_state_2 = MakeWatcher(kCertName1, absl::nullopt);
468   EXPECT_THAT(GetCallbackQueue(),
469               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
470   // Push credential updates to kCertName1 and check if the status works as
471   // expected.
472   distributor_.SetKeyMaterials(
473       kCertName1, kRootCert1Contents,
474       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
475   // Check the updates are delivered to watcher 1.
476   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
477               ::testing::ElementsAre(CredentialInfo(
478                   "", MakeCertKeyPairs(kIdentityCert1PrivateKey,
479                                        kIdentityCert1Contents))));
480   // Check the updates are delivered to watcher 2.
481   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
482               ::testing::ElementsAre(CredentialInfo(kRootCert1Contents, {})));
483   // Push root cert updates to kCertName1.
484   distributor_.SetKeyMaterials(kCertName1, kRootCert2Contents, absl::nullopt);
485   // Check the updates are delivered to watcher 2.
486   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
487               ::testing::ElementsAre(CredentialInfo(kRootCert2Contents, {})));
488   // Check the updates are not delivered to watcher 1.
489   EXPECT_THAT(watcher_state_1->GetCredentialQueue(), ::testing::ElementsAre());
490   // Push identity cert updates to kCertName1.
491   distributor_.SetKeyMaterials(
492       kCertName1, absl::nullopt,
493       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
494   // Check the updates are not delivered to watcher 2.
495   EXPECT_THAT(watcher_state_2->GetCredentialQueue(), ::testing::ElementsAre());
496   // Check the updates are delivered to watcher 1.
497   EXPECT_THAT(watcher_state_1->GetCredentialQueue(),
498               ::testing::ElementsAre(CredentialInfo(
499                   "", MakeCertKeyPairs(kIdentityCert2PrivateKey,
500                                        kIdentityCert2Contents))));
501   // Cancel watcher 2.
502   CancelWatch(watcher_state_2);
503   EXPECT_THAT(GetCallbackQueue(),
504               ::testing::ElementsAre(CallbackStatus(kCertName1, false, true)));
505   // Cancel watcher 1.
506   CancelWatch(watcher_state_1);
507   EXPECT_THAT(GetCallbackQueue(),
508               ::testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
509 }
510 
TEST_F(GrpcTlsCertificateDistributorTest,RemoveAllWatchersForCertNameAndAddAgain)511 TEST_F(GrpcTlsCertificateDistributorTest,
512        RemoveAllWatchersForCertNameAndAddAgain) {
513   // Register watcher 1 and watcher 2 watching kCertName1 for root and identity
514   // certs.
515   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
516   EXPECT_THAT(GetCallbackQueue(),
517               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
518   WatcherState* watcher_state_2 = MakeWatcher(kCertName1, kCertName1);
519   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre());
520   // Push credential updates to kCertName1.
521   distributor_.SetKeyMaterials(
522       kCertName1, kRootCert1Contents,
523       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
524   // Cancel watcher 2.
525   CancelWatch(watcher_state_2);
526   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre());
527   // Cancel watcher 1.
528   CancelWatch(watcher_state_1);
529   EXPECT_THAT(GetCallbackQueue(),
530               ::testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
531   // Register watcher 3 watching kCertName for root and identity certs.
532   WatcherState* watcher_state_3 = MakeWatcher(kCertName1, kCertName1);
533   EXPECT_THAT(GetCallbackQueue(),
534               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
535   // Push credential updates to kCertName1.
536   distributor_.SetKeyMaterials(
537       kCertName1, kRootCert2Contents,
538       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
539   // Check the updates are delivered to watcher 3.
540   EXPECT_THAT(
541       watcher_state_3->GetCredentialQueue(),
542       ::testing::ElementsAre(CredentialInfo(
543           kRootCert2Contents,
544           MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents))));
545   // Cancel watcher 3.
546   CancelWatch(watcher_state_3);
547   EXPECT_THAT(GetCallbackQueue(),
548               ::testing::ElementsAre(CallbackStatus(kCertName1, false, false)));
549 }
550 
TEST_F(GrpcTlsCertificateDistributorTest,ResetCallbackToNull)551 TEST_F(GrpcTlsCertificateDistributorTest, ResetCallbackToNull) {
552   // Register watcher 1 watching kCertName1 for root and identity certs.
553   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
554   EXPECT_THAT(GetCallbackQueue(),
555               ::testing::ElementsAre(CallbackStatus(kCertName1, true, true)));
556   // Reset callback to nullptr.
557   distributor_.SetWatchStatusCallback(nullptr);
558   // Cancel watcher 1 shouldn't trigger any callback.
559   CancelWatch(watcher_state_1);
560   EXPECT_THAT(GetCallbackQueue(), ::testing::ElementsAre());
561 }
562 
TEST_F(GrpcTlsCertificateDistributorTest,SetKeyMaterialsInCallback)563 TEST_F(GrpcTlsCertificateDistributorTest, SetKeyMaterialsInCallback) {
564   distributor_.SetWatchStatusCallback([this](std::string cert_name,
565                                              bool root_being_watched,
566                                              bool identity_being_watched) {
567     distributor_.SetKeyMaterials(
568         cert_name, kRootCert1Contents,
569         MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
570   });
571   auto verify_function = [this](std::string cert_name) {
572     WatcherState* watcher_state_1 = MakeWatcher(cert_name, cert_name);
573     // Check the updates are delivered to watcher 1.
574     EXPECT_THAT(
575         watcher_state_1->GetCredentialQueue(),
576         ::testing::ElementsAre(CredentialInfo(
577             kRootCert1Contents, MakeCertKeyPairs(kIdentityCert1PrivateKey,
578                                                  kIdentityCert1Contents))));
579     CancelWatch(watcher_state_1);
580   };
581   // Start 1000 threads that will register a watcher to a new cert name, verify
582   // the key materials being set, and then cancel the watcher, to make sure the
583   // lock mechanism in the distributor is safe.
584   std::vector<std::thread> threads;
585   threads.reserve(1000);
586   for (int i = 0; i < 1000; ++i) {
587     threads.emplace_back(verify_function, std::to_string(i));
588   }
589   for (auto& th : threads) {
590     th.join();
591   }
592 }
593 
TEST_F(GrpcTlsCertificateDistributorTest,WatchACertInfoWithValidCredentials)594 TEST_F(GrpcTlsCertificateDistributorTest, WatchACertInfoWithValidCredentials) {
595   // Push credential updates to kCertName1.
596   distributor_.SetKeyMaterials(
597       kCertName1, kRootCert1Contents,
598       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
599   // Push root credential updates to kCertName2.
600   distributor_.SetKeyMaterials(kRootCert2Name, kRootCert2Contents,
601                                absl::nullopt);
602   // Push identity credential updates to kCertName2.
603   distributor_.SetKeyMaterials(
604       kIdentityCert2Name, absl::nullopt,
605       MakeCertKeyPairs(kIdentityCert2PrivateKey, kIdentityCert2Contents));
606   // Register watcher 1.
607   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
608   // watcher 1 should receive the credentials right away.
609   EXPECT_THAT(
610       watcher_state_1->GetCredentialQueue(),
611       ::testing::ElementsAre(CredentialInfo(
612           kRootCert1Contents,
613           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
614   CancelWatch(watcher_state_1);
615   // Register watcher 2.
616   WatcherState* watcher_state_2 = MakeWatcher(kRootCert2Name, absl::nullopt);
617   // watcher 2 should receive the root credentials right away.
618   EXPECT_THAT(watcher_state_2->GetCredentialQueue(),
619               ::testing::ElementsAre(CredentialInfo(kRootCert2Contents, {})));
620   // Register watcher 3.
621   WatcherState* watcher_state_3 =
622       MakeWatcher(absl::nullopt, kIdentityCert2Name);
623   // watcher 3 should received the identity credentials right away.
624   EXPECT_THAT(watcher_state_3->GetCredentialQueue(),
625               ::testing::ElementsAre(CredentialInfo(
626                   "", MakeCertKeyPairs(kIdentityCert2PrivateKey,
627                                        kIdentityCert2Contents))));
628   CancelWatch(watcher_state_2);
629   CancelWatch(watcher_state_3);
630 }
631 
TEST_F(GrpcTlsCertificateDistributorTest,SetErrorForCertForBothRootAndIdentity)632 TEST_F(GrpcTlsCertificateDistributorTest,
633        SetErrorForCertForBothRootAndIdentity) {
634   // Register watcher 1.
635   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
636   // Calling SetErrorForCert on both cert names should only call one OnError
637   // on watcher 1.
638   distributor_.SetErrorForCert(
639       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
640       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
641   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
642               ::testing::ElementsAre(
643                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
644   // Calling SetErrorForCert on root cert name should call OnError
645   // on watcher 1 again.
646   distributor_.SetErrorForCert(
647       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kErrorMessage),
648       absl::nullopt);
649   EXPECT_THAT(
650       watcher_state_1->GetErrorQueue(),
651       ::testing::ElementsAre(ErrorInfo(kErrorMessage, kIdentityErrorMessage)));
652   // Calling SetErrorForCert on identity cert name should call OnError
653   // on watcher 1 again.
654   distributor_.SetErrorForCert(
655       kCertName1, absl::nullopt,
656       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kErrorMessage));
657   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
658               ::testing::ElementsAre(ErrorInfo(kErrorMessage, kErrorMessage)));
659   distributor_.CancelTlsCertificatesWatch(watcher_state_1->watcher);
660   EXPECT_EQ(watcher_state_1->watcher, nullptr);
661 }
662 
TEST_F(GrpcTlsCertificateDistributorTest,SetErrorForCertForRootOrIdentity)663 TEST_F(GrpcTlsCertificateDistributorTest, SetErrorForCertForRootOrIdentity) {
664   // Register watcher 1.
665   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, absl::nullopt);
666   // Calling SetErrorForCert on root name should only call one OnError
667   // on watcher 1.
668   distributor_.SetErrorForCert(
669       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
670       absl::nullopt);
671   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
672               ::testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
673   // Calling SetErrorForCert on identity name should do nothing.
674   distributor_.SetErrorForCert(
675       kCertName1, absl::nullopt,
676       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
677   EXPECT_THAT(watcher_state_1->GetErrorQueue(), ::testing::ElementsAre());
678   // Calling SetErrorForCert on both names should still get one OnError call.
679   distributor_.SetErrorForCert(
680       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
681       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
682   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
683               ::testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
684   CancelWatch(watcher_state_1);
685   // Register watcher 2.
686   WatcherState* watcher_state_2 = MakeWatcher(absl::nullopt, kCertName1);
687   // Calling SetErrorForCert on identity name should only call one OnError
688   // on watcher 2.
689   distributor_.SetErrorForCert(
690       kCertName1, absl::nullopt,
691       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
692   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
693               ::testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
694   // Calling SetErrorForCert on root name should do nothing.
695   distributor_.SetErrorForCert(
696       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
697       absl::nullopt);
698   EXPECT_THAT(watcher_state_2->GetErrorQueue(), ::testing::ElementsAre());
699   // Calling SetErrorForCert on both names should still get one OnError call.
700   distributor_.SetErrorForCert(
701       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
702       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
703   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
704               ::testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
705   CancelWatch(watcher_state_2);
706 }
707 
TEST_F(GrpcTlsCertificateDistributorTest,SetErrorForIdentityNameWithPreexistingErrorForRootName)708 TEST_F(GrpcTlsCertificateDistributorTest,
709        SetErrorForIdentityNameWithPreexistingErrorForRootName) {
710   // SetErrorForCert for kCertName1.
711   distributor_.SetErrorForCert(
712       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
713       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
714   // Register watcher 1 for kCertName1 as root and kCertName2 as identity.
715   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName2);
716   // Should trigger OnError call right away since kCertName1 has error.
717   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
718               ::testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
719   // Calling SetErrorForCert on kCertName2 should trigger OnError with both
720   // errors, because kCertName1 also has error.
721   distributor_.SetErrorForCert(
722       kCertName2, absl::nullopt,
723       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
724   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
725               ::testing::ElementsAre(
726                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
727   CancelWatch(watcher_state_1);
728 }
729 
TEST_F(GrpcTlsCertificateDistributorTest,SetErrorForCertForRootNameWithSameNameForIdentityErrored)730 TEST_F(GrpcTlsCertificateDistributorTest,
731        SetErrorForCertForRootNameWithSameNameForIdentityErrored) {
732   // SetErrorForCert for kCertName1.
733   distributor_.SetErrorForCert(
734       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
735       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
736   // Register watcher 1 for kCertName2 as root and kCertName1 as identity.
737   WatcherState* watcher_state_1 = MakeWatcher(kCertName2, kCertName1);
738   // Should trigger OnError call right away since kCertName2 has error.
739   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
740               ::testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
741   // Calling SetErrorForCert on kCertName2 should trigger OnError with both
742   // errors, because kCertName1 also has error.
743   distributor_.SetErrorForCert(
744       kCertName2, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
745       absl::nullopt);
746   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
747               ::testing::ElementsAre(
748                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
749   CancelWatch(watcher_state_1);
750 }
751 
TEST_F(GrpcTlsCertificateDistributorTest,SetErrorForIdentityNameWithoutErrorForRootName)752 TEST_F(GrpcTlsCertificateDistributorTest,
753        SetErrorForIdentityNameWithoutErrorForRootName) {
754   // Register watcher 1 for kCertName1 as root and kCertName2 as identity.
755   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName2);
756   // Should not trigger OnError.
757   EXPECT_THAT(watcher_state_1->GetErrorQueue(), ::testing::ElementsAre());
758   // Calling SetErrorForCert on kCertName2 should trigger OnError.
759   distributor_.SetErrorForCert(
760       kCertName2, absl::nullopt,
761       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
762   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
763               ::testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
764   CancelWatch(watcher_state_1);
765   // Register watcher 2 for kCertName2 as identity and a non-existing name
766   // kRootCert1Name as root.
767   WatcherState* watcher_state_2 = MakeWatcher(kRootCert1Name, kCertName2);
768   // Should not trigger OnError.
769   EXPECT_THAT(watcher_state_2->GetErrorQueue(), ::testing::ElementsAre());
770   // Calling SetErrorForCert on kCertName2 should trigger OnError.
771   distributor_.SetErrorForCert(
772       kCertName2, absl::nullopt,
773       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
774   EXPECT_THAT(watcher_state_2->error_queue,
775               ::testing::ElementsAre(ErrorInfo("", kIdentityErrorMessage)));
776   CancelWatch(watcher_state_2);
777 }
778 
TEST_F(GrpcTlsCertificateDistributorTest,SetErrorForRootNameWithPreexistingErrorForIdentityName)779 TEST_F(GrpcTlsCertificateDistributorTest,
780        SetErrorForRootNameWithPreexistingErrorForIdentityName) {
781   WatcherState* watcher_state_1 = MakeWatcher(kCertName2, kCertName1);
782   // Should not trigger OnError.
783   EXPECT_THAT(watcher_state_1->GetErrorQueue(), ::testing::ElementsAre());
784   // Calling SetErrorForCert on kCertName2 should trigger OnError.
785   distributor_.SetErrorForCert(
786       kCertName2, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
787       absl::nullopt);
788   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
789               ::testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
790   CancelWatch(watcher_state_1);
791   // Register watcher 2 for kCertName2 as root and a non-existing name
792   // kIdentityCert1Name as identity.
793   WatcherState* watcher_state_2 = MakeWatcher(kCertName2, kIdentityCert1Name);
794   // Should not trigger OnError.
795   EXPECT_THAT(watcher_state_2->GetErrorQueue(), ::testing::ElementsAre());
796   // Calling SetErrorForCert on kCertName2 should trigger OnError.
797   distributor_.SetErrorForCert(
798       kCertName2, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
799       absl::nullopt);
800   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
801               ::testing::ElementsAre(ErrorInfo(kRootErrorMessage, "")));
802   CancelWatch(watcher_state_2);
803 }
804 
TEST_F(GrpcTlsCertificateDistributorTest,CancelTheLastWatcherOnAnErroredCertInfo)805 TEST_F(GrpcTlsCertificateDistributorTest,
806        CancelTheLastWatcherOnAnErroredCertInfo) {
807   // Register watcher 1.
808   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
809   // Calling SetErrorForCert on both cert names should only call one OnError
810   // on watcher 1.
811   distributor_.SetErrorForCert(
812       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
813       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
814   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
815               ::testing::ElementsAre(
816                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
817   // When watcher 1 is removed, the cert info entry should be removed.
818   CancelWatch(watcher_state_1);
819   // Register watcher 2 on the same cert name.
820   WatcherState* watcher_state_2 = MakeWatcher(kCertName1, kCertName1);
821   // Should not trigger OnError call on watcher 2 right away.
822   EXPECT_THAT(watcher_state_2->GetErrorQueue(), ::testing::ElementsAre());
823   CancelWatch(watcher_state_2);
824 }
825 
TEST_F(GrpcTlsCertificateDistributorTest,WatchErroredCertInfoWithValidCredentialData)826 TEST_F(GrpcTlsCertificateDistributorTest,
827        WatchErroredCertInfoWithValidCredentialData) {
828   // Push credential updates to kCertName1.
829   distributor_.SetKeyMaterials(
830       kCertName1, kRootCert1Contents,
831       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
832   // Calling SetErrorForCert on both cert names.
833   distributor_.SetErrorForCert(
834       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
835       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
836   // Register watcher 1.
837   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
838   // watcher 1 should receive both the old credentials and the error right away.
839   EXPECT_THAT(
840       watcher_state_1->GetCredentialQueue(),
841       ::testing::ElementsAre(CredentialInfo(
842           kRootCert1Contents,
843           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
844   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
845               ::testing::ElementsAre(
846                   ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
847   CancelWatch(watcher_state_1);
848 }
849 
TEST_F(GrpcTlsCertificateDistributorTest,SetErrorForCertThenSuccessfulCredentialUpdates)850 TEST_F(GrpcTlsCertificateDistributorTest,
851        SetErrorForCertThenSuccessfulCredentialUpdates) {
852   // Calling SetErrorForCert on both cert names.
853   distributor_.SetErrorForCert(
854       kCertName1, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
855       GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
856   // Push credential updates to kCertName1.
857   distributor_.SetKeyMaterials(
858       kCertName1, kRootCert1Contents,
859       MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents));
860   // Register watcher 1.
861   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
862   // watcher 1 should only receive credential updates without any error, because
863   // the previous error is wiped out by a successful update.
864   EXPECT_THAT(
865       watcher_state_1->GetCredentialQueue(),
866       ::testing::ElementsAre(CredentialInfo(
867           kRootCert1Contents,
868           MakeCertKeyPairs(kIdentityCert1PrivateKey, kIdentityCert1Contents))));
869   EXPECT_THAT(watcher_state_1->GetErrorQueue(), ::testing::ElementsAre());
870   CancelWatch(watcher_state_1);
871 }
872 
TEST_F(GrpcTlsCertificateDistributorTest,WatchCertInfoThenInvokeSetError)873 TEST_F(GrpcTlsCertificateDistributorTest, WatchCertInfoThenInvokeSetError) {
874   // Register watcher 1.
875   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, kCertName1);
876   // Register watcher 2.
877   WatcherState* watcher_state_2 = MakeWatcher(kRootCert1Name, absl::nullopt);
878   // Register watcher 3.
879   WatcherState* watcher_state_3 =
880       MakeWatcher(absl::nullopt, kIdentityCert1Name);
881   distributor_.SetError(GRPC_ERROR_CREATE_FROM_STATIC_STRING(kErrorMessage));
882   EXPECT_THAT(watcher_state_1->GetErrorQueue(),
883               ::testing::ElementsAre(ErrorInfo(kErrorMessage, kErrorMessage)));
884   EXPECT_THAT(watcher_state_2->GetErrorQueue(),
885               ::testing::ElementsAre(ErrorInfo(kErrorMessage, "")));
886   EXPECT_THAT(watcher_state_3->GetErrorQueue(),
887               ::testing::ElementsAre(ErrorInfo("", kErrorMessage)));
888   CancelWatch(watcher_state_1);
889   CancelWatch(watcher_state_2);
890   CancelWatch(watcher_state_3);
891 }
892 
TEST_F(GrpcTlsCertificateDistributorTest,WatchErroredCertInfoBySetError)893 TEST_F(GrpcTlsCertificateDistributorTest, WatchErroredCertInfoBySetError) {
894   // Register watcher 1 watching kCertName1 as root.
895   WatcherState* watcher_state_1 = MakeWatcher(kCertName1, absl::nullopt);
896   // Register watcher 2 watching kCertName2 as identity.
897   WatcherState* watcher_state_2 = MakeWatcher(absl::nullopt, kCertName2);
898   // Call SetError and then cancel all watchers.
899   distributor_.SetError(GRPC_ERROR_CREATE_FROM_STATIC_STRING(kErrorMessage));
900   CancelWatch(watcher_state_1);
901   CancelWatch(watcher_state_2);
902   // Register watcher 3 watching kCertName1 as root and kCertName2 as identity
903   // should not get the error updates.
904   WatcherState* watcher_state_3 = MakeWatcher(kCertName1, kCertName2);
905   EXPECT_THAT(watcher_state_3->GetErrorQueue(), ::testing::ElementsAre());
906   CancelWatch(watcher_state_3);
907   // Register watcher 4 watching kCertName2 as root and kCertName1 as identity
908   // should not get the error updates.
909   WatcherState* watcher_state_4 = MakeWatcher(kCertName2, kCertName1);
910   EXPECT_THAT(watcher_state_4->GetErrorQueue(), ::testing::ElementsAre());
911   CancelWatch(watcher_state_4);
912 }
913 
TEST_F(GrpcTlsCertificateDistributorTest,SetErrorForCertInCallback)914 TEST_F(GrpcTlsCertificateDistributorTest, SetErrorForCertInCallback) {
915   distributor_.SetWatchStatusCallback([this](std::string cert_name,
916                                              bool root_being_watched,
917                                              bool identity_being_watched) {
918     this->distributor_.SetErrorForCert(
919         cert_name, GRPC_ERROR_CREATE_FROM_STATIC_STRING(kRootErrorMessage),
920         GRPC_ERROR_CREATE_FROM_STATIC_STRING(kIdentityErrorMessage));
921   });
922   auto verify_function = [this](std::string cert_name) {
923     WatcherState* watcher_state_1 = MakeWatcher(cert_name, cert_name);
924     // Check the errors are delivered to watcher 1.
925     EXPECT_THAT(watcher_state_1->GetErrorQueue(),
926                 ::testing::ElementsAre(
927                     ErrorInfo(kRootErrorMessage, kIdentityErrorMessage)));
928     CancelWatch(watcher_state_1);
929   };
930   // Start 1000 threads that will register a watcher to a new cert name, verify
931   // the key materials being set, and then cancel the watcher, to make sure the
932   // lock mechanism in the distributor is safe.
933   std::vector<std::thread> threads;
934   threads.reserve(1000);
935   for (int i = 0; i < 1000; ++i) {
936     threads.emplace_back(verify_function, std::to_string(i));
937   }
938   for (auto& th : threads) {
939     th.join();
940   }
941 }
942 
943 }  // namespace testing
944 
945 }  // namespace grpc_core
946 
main(int argc,char ** argv)947 int main(int argc, char** argv) {
948   grpc::testing::TestEnvironment env(argc, argv);
949   ::testing::InitGoogleTest(&argc, argv);
950   grpc_init();
951   int ret = RUN_ALL_TESTS();
952   grpc_shutdown();
953   return ret;
954 }
955