[lib] Add reboot_operations and fix WaitForNot.

lib/utils/BUILD.bazel

@@ -122,6 +122,20 @@ cc_test(
     ],
 )
 
+cc_library(
+    name = "reboot_operations",
+    testonly = True,
+    srcs = ["reboot_operations.cc"],
+    hdrs = ["reboot_operations.h"],
+    deps = [
+        "//lib/validator:validator_lib",
+        "//thinkit:ssh_client",
+        "//thinkit:switch",
+        "@com_google_absl//absl/status",
+        "@com_google_absl//absl/time",
+    ],
+)
+
 cc_library(
     name = "merging_status_bundle",
     srcs = ["merging_status_bundle.cc"],

lib/utils/reboot_operations.cc

@@ -0,0 +1,30 @@
+#include "lib/utils/reboot_operations.h"
+
+#include "absl/status/status.h"
+#include "absl/time/time.h"
+#include "lib/validator/validator_lib.h"
+#include "thinkit/ssh_client.h"
+#include "thinkit/switch.h"
+
+namespace pins_test {
+namespace {
+
+constexpr absl::Duration kRebootSshTimeout = absl::Seconds(10);
+
+}  // namespace
+
+absl::Status RebootSwitchWithSsh(thinkit::Switch& thinkit_switch,
+                                 thinkit::SSHClient& ssh_client,
+                                 absl::Duration down_timeout) {
+  // Ignoring the error check of SSH reboot as it is non deterministic.
+  ssh_client
+      .RunCommand(thinkit_switch.ChassisName(), "reboot", kRebootSshTimeout)
+      .IgnoreError();
+  return WaitForNot(
+      [&](absl::Duration timeout) {
+        return SSHable(thinkit_switch, ssh_client, timeout);
+      },
+      down_timeout);
+}
+
+}  // namespace pins_test

lib/utils/reboot_operations.h

@@ -0,0 +1,20 @@
+#ifndef PINS_LIB_UTILS_REBOOT_OPERATIONS_H_
+#define PINS_LIB_UTILS_REBOOT_OPERATIONS_H_
+
+#include "absl/status/status.h"
+#include "absl/time/time.h"
+#include "thinkit/ssh_client.h"
+#include "thinkit/switch.h"
+
+namespace pins_test {
+
+constexpr absl::Duration kDefaultDownTimeout = absl::Seconds(75);
+
+// Reboots the switch using SSH and waits for the switch to go down.
+absl::Status RebootSwitchWithSsh(
+    thinkit::Switch& thinkit_switch, thinkit::SSHClient& ssh_client,
+    absl::Duration down_timeout = kDefaultDownTimeout);
+
+}  // namespace pins_test
+
+#endif  // PINS_LIB_UTILS_REBOOT_OPERATIONS_H_

lib/utils/reboot_operations_test.cc

@@ -0,0 +1,53 @@
+#include "lib/utils/reboot_operations.h"
+
+#include <string>
+
+#include "absl/status/status.h"
+#include "absl/time/time.h"
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "gutil/gutil/status_matchers.h"
+#include "thinkit/mock_ssh_client.h"
+#include "thinkit/mock_switch.h"
+
+namespace pins_test {
+namespace {
+
+using ::gutil::StatusIs;
+using ::testing::_;
+using ::testing::Return;
+using ::testing::ReturnRefOfCopy;
+
+TEST(RebootOperationsTest, RebootSwitchWithSshIgnoresRebootFail) {
+  thinkit::MockSwitch mock_switch;
+  EXPECT_CALL(mock_switch, ChassisName)
+      .WillRepeatedly(ReturnRefOfCopy(std::string("chassis")));
+
+  thinkit::MockSSHClient mock_ssh_client;
+  EXPECT_CALL(mock_ssh_client, RunCommand("chassis", "echo foo", _))
+      .WillOnce(Return("foo\n"))
+      .WillOnce(Return(absl::UnavailableError("can't connect")));
+  EXPECT_CALL(mock_ssh_client, RunCommand("chassis", "reboot", _))
+      .WillOnce(Return(absl::UnavailableError("can't connect")));
+
+  EXPECT_OK(RebootSwitchWithSsh(mock_switch, mock_ssh_client));
+}
+
+TEST(RebootOperationsTest, RebootSwitchWithSshDoesntGoDown) {
+  thinkit::MockSwitch mock_switch;
+  EXPECT_CALL(mock_switch, ChassisName)
+      .WillRepeatedly(ReturnRefOfCopy(std::string("chassis")));
+
+  thinkit::MockSSHClient mock_ssh_client;
+  EXPECT_CALL(mock_ssh_client, RunCommand("chassis", "echo foo", _))
+      .WillRepeatedly(Return("foo\n"));
+  EXPECT_CALL(mock_ssh_client, RunCommand("chassis", "reboot", _))
+      .WillOnce(Return(absl::UnavailableError("can't connect")));
+
+  EXPECT_THAT(RebootSwitchWithSsh(mock_switch, mock_ssh_client,
+                                  /*down_timeout=*/absl::Seconds(1)),
+              StatusIs(absl::StatusCode::kDeadlineExceeded));
+}
+
+}  // namespace
+}  // namespace pins_test

lib/validator/BUILD.bazel

@@ -106,6 +106,7 @@ cc_library(
         "//gutil/gutil:status",
         "//lib/gnmi:gnmi_helper",
         "//lib/gnoi:gnoi_helper",
+        "//lib/ssh:ssh_wrapper_client",
         "//thinkit:ssh_client",
         "//thinkit:switch",
         "@com_github_gnmi//proto/gnmi:gnmi_cc_grpc_proto",

lib/validator/validator_lib.cc

@@ -37,6 +37,7 @@
 #include "gutil/gutil/status.h"
 #include "lib/gnmi/gnmi_helper.h"
 #include "lib/gnoi/gnoi_helper.h"
+#include "lib/ssh/ssh_wrapper_client.h"
 #include "p4/v1/p4runtime.grpc.pb.h"
 #include "p4/v1/p4runtime.pb.h"
 #include "system/system.grpc.pb.h"
@@ -270,6 +271,18 @@ absl::Status NoAlarms(thinkit::Switch& thinkit_switch, absl::Duration timeout) {
       absl::StrCat("The system has alarms set: ", absl::StrJoin(alarms, "; ")));
 }
 
+absl::Status SwitchUp(thinkit::Switch& thinkit_switch,
+                      thinkit::SSHClient& ssh_client,
+                      thinkit::SshWrapperClient& ssh_wrapper_client,
+                      absl::Duration timeout) {
+  RETURN_IF_ERROR(SwitchReadyWithSsh(thinkit_switch, ssh_client,
+                                     /*interfaces=*/{},
+                                     /*check_interfaces=*/false,
+                                     /*with_healthz=*/false, timeout));
+  return ssh_wrapper_client.CheckContainersUp(thinkit_switch.ChassisName(),
+                                              ssh_client);
+}
+
 absl::Status SwitchReady(thinkit::Switch& thinkit_switch,
                          absl::Span<const std::string> interfaces,
                          absl::Duration timeout) {

lib/validator/validator_lib.h

@@ -21,6 +21,7 @@
 #include <string>
 #include <utility>
 
+#include "absl/log/log.h"
 #include "absl/status/status.h"
 #include "absl/status/statusor.h"
 #include "absl/strings/str_cat.h"
@@ -29,6 +30,7 @@
 #include "absl/time/clock.h"
 #include "absl/time/time.h"
 #include "absl/types/span.h"
+#include "lib/ssh/ssh_wrapper_client.h"
 #include "thinkit/ssh_client.h"
 #include "thinkit/switch.h"
 
@@ -113,6 +115,12 @@ inline absl::Status AllPortsUp(thinkit::Switch &thinkit_switch,
 absl::Status NoAlarms(thinkit::Switch& thinkit_switch,
                       absl::Duration timeout = kDefaultTimeout);
 
+// Checks if switch is SSHable, gNXI/P4RT are responding, and containers are up.
+absl::Status SwitchUp(thinkit::Switch& thinkit_switch,
+                      thinkit::SSHClient& ssh_client,
+                      thinkit::SshWrapperClient& ssh_wrapper_client,
+                      absl::Duration timeout);
+
 // Checks if the switch is ready by running the following validations:
 // Pingable, P4rtAble, GnmiAble, GnoiAble.
 absl::Status SwitchReady(thinkit::Switch &thinkit_switch,
@@ -179,16 +187,33 @@ absl::Status WaitForCondition(Func &&condition, absl::Duration timeout,
 template <typename Func, typename... Args>
 absl::Status WaitForNot(Func &&condition, absl::Duration timeout,
                         Args &&...args) {
-  return WaitForCondition(
-      [condition = std::forward<Func>(condition)](Args &&...args) {
-	absl::Status status =
-            std::invoke(condition, std::forward<Args>(args)...);
-        if (status.ok()) {
-          return absl::InternalError("Validator still returns okay.");
-        }
-        return absl::OkStatus();
-      },
-      timeout, std::forward<Args>(args)...);
+  absl::Time deadline = absl::Now() + timeout;
+  absl::Status final_status;
+  uint64_t number_of_function_invocations = 0;
+  do {
+    number_of_function_invocations++;
+    if constexpr (std::is_invocable_r_v<absl::Status, Func, Args...,
+                                        absl::Duration>) {
+      final_status = std::invoke(condition, std::forward<Args>(args)...,
+                                 /*timeout=*/deadline - absl::Now());
+    } else {
+      final_status = std::invoke(condition, std::forward<Args>(args)...);
+    }
+  } while (final_status.ok() && absl::Now() < deadline);
+  if (!final_status.ok()) {
+    // deadline - absl::Now() is the remaining time left in the timeout, so
+    // timeout - (deadline - absl::Now()) is the time it took to reach the not
+    // ok condition.
+    LOG(INFO) << "Condition became not ok after "
+              << timeout - (deadline - absl::Now()) << " with "
+              << number_of_function_invocations << " function invocations:\n"
+              << final_status;
+    return absl::OkStatus();
+  }
+
+  return absl::DeadlineExceededError(absl::StrCat(
+      "Condition still ok after ", absl::FormatDuration(timeout), " with ",
+      number_of_function_invocations, " function invocations."));
 }
 
 } // namespace pins_test