More quality-of-life improvements: Exp/Eval/Gen limits, and soft limits

src/search/search_algorithm.cc

@@ -40,7 +40,15 @@ static successor_generator::SuccessorGenerator &get_successor_generator(
 }
 
 SearchAlgorithm::SearchAlgorithm(
-    OperatorCost cost_type, int bound, double max_time,
+    OperatorCost cost_type, int bound,
+    double min_gen,
+    double min_eval,
+    double min_exp,
+    double min_time,
+    double max_gen,
+    double max_eval,
+    double max_exp,
+    double max_time,
     const string &description, utils::Verbosity verbosity)
     : description(description),
       status(IN_PROGRESS),
@@ -55,6 +63,13 @@ SearchAlgorithm::SearchAlgorithm(
       bound(bound),
       cost_type(cost_type),
       is_unit_cost(task_properties::is_unit_cost(task_proxy)),
+      min_gen(min_gen),
+      min_eval(min_eval),
+      min_exp(min_exp),
+      min_time(min_time),
+      max_gen(max_gen),
+      max_eval(max_eval),
+      max_exp(max_exp),
       max_time(max_time) {
     if (bound < 0) {
         cerr << "error: negative cost bound " << bound << endl;
@@ -77,6 +92,13 @@ SearchAlgorithm::SearchAlgorithm(const plugins::Options &opts) // TODO options o
       statistics(log),
       cost_type(opts.get<OperatorCost>("cost_type")),
       is_unit_cost(task_properties::is_unit_cost(task_proxy)),
+      min_gen(opts.get<double>("min_gen")),
+      min_eval(opts.get<double>("min_eval")),
+      min_exp(opts.get<double>("min_exp")),
+      min_time(opts.get<double>("min_time")),
+      max_gen(opts.get<double>("max_gen")),
+      max_eval(opts.get<double>("max_eval")),
+      max_exp(opts.get<double>("max_exp")),
       max_time(opts.get<double>("max_time")) {
     if (opts.get<int>("bound") < 0) {
         cerr << "error: negative cost bound " << opts.get<int>("bound") << endl;
@@ -109,17 +131,57 @@ void SearchAlgorithm::set_plan(const Plan &p) {
 
 void SearchAlgorithm::search() {
     initialize();
-    utils::CountdownTimer timer(max_time);
+    utils::g_log << "Hard limits:" << endl;
+    utils::g_log << "Max runtime: " << max_time << " sec" << endl;
+    utils::g_log << "Max evaluations: " << max_eval << " states" << endl;
+    utils::g_log << "Max expansions: " << max_exp << " states" << endl;
+    utils::g_log << "Max generations: " << max_gen << " states" << endl;
+
+    // Implementing the default values for soft limits is a bit complicated because
+    // the default "unspecified" value requires a special treatment.
+    //
+    // Setting the default value to a small (e.g. negative) value which is always reached is wrong because
+    // the search stops immediately when no soft limits are given for exp/gen/eval/elapsed.
+    //
+    // Setting the default value to a large (e.g. infinity) value is also wrong,
+    // because such a limit is never reached even if all other *specified* soft limits are reached,
+    // making the search continue although it is supposed to stop.
+
+    bool use_soft_limit = (min_time > 0) || (min_gen > 0) || (min_eval > 0) || (min_exp > 0);
+    if (use_soft_limit){
+	utils::g_log << "Soft limits:" << endl;
+	utils::g_log << "Min runtime: " << min_time << " sec" << endl;
+	utils::g_log << "Min evaluations: " << min_eval << " states" << endl;
+	utils::g_log << "Min expansions: " << min_exp << " states" << endl;
+	utils::g_log << "Min generations: " << min_gen << " states" << endl;
+    }
+
+    utils::CountdownTimer timer_max(max_time);
+    utils::CountdownTimer timer_min(min_time);
+
     while (status == IN_PROGRESS) {
         status = step();
-        if (timer.is_expired()) {
-            log << "Time limit reached. Abort search." << endl;
+        auto ex = statistics.get_expanded();
+        auto ev = statistics.get_evaluated_states();
+        auto gen = statistics.get_generated();
+        if (timer_max.is_expired() || (gen >= max_gen) || (ev >= max_eval) || (ex >= max_exp)) {
+            utils::g_log << "One of the hard limits is reached. Aborting search." << endl;
             status = TIMEOUT;
             break;
         }
+	if (use_soft_limit) {	// note: without this, it stops immediately when all soft limits are 0
+	    if (   (min_time > 0 && timer_min.is_expired())
+		&& (min_gen  > 0 && gen >= min_gen )
+		&& (min_eval > 0 && ev  >= min_eval)
+		&& (min_exp  > 0 && ex  >= min_exp )) {
+		utils::g_log << "Soft limit: Spent all minimum required amount of computation. Aborting search." << endl;
+		status = TIMEOUT;
+		break;
+	    }
+	}
     }
     // TODO: Revise when and which search times are logged.
-    log << "Actual search time: " << timer.get_elapsed_time() << endl;
+    log << "Actual search time: " << timer_max.get_elapsed_time() << endl;
 }
 
 bool SearchAlgorithm::check_goal_and_set_plan(const State &state) {
@@ -192,20 +254,76 @@ void add_search_algorithm_options_to_feature(
         "experiments. Timed-out searches are treated as failed searches, "
         "just like incomplete search algorithms that exhaust their search space.",
         "infinity");
+    feature.add_option<double>(
+        "max_eval",
+        "maximum number of evaluated states (measured by statistics.get_evaluated_states()).",
+        "infinity");
+    feature.add_option<double>(
+        "max_gen",
+        "maximum number of generated states (measured by statistics.get_generated()).",
+        "infinity");
+    feature.add_option<double>(
+        "max_exp",
+        "maximum number of expanded states (measured by statistics.get_expanded()).",
+        "infinity");
+    feature.add_option<double>(
+        "min_time",
+        "specifies a soft limit i.e. minimum time in seconds the search must run for. "
+	"The same accuracy statement as the one for max_time applies to this option. "
+	"\n"
+	"0 and negative values indicate that the soft limit is disabled. "
+	"\n"
+	"Soft limit addresses the limitation of hard limits (e.g. max_time) that one limit interferes another. "
+	"For example, when max_eval = 100k and max_time = 5 sec, "
+	"majority of runs are cut off before reaching 100k evaluations, "
+	"and the result is not an appropriate representation of running 100k evaluations. "
+	"To measure the evaluations and time separately, therefore one would run two separate experiments, "
+	"which is a waste of compute for instances solved by both. "
+	"Instead of running them separately, this option allows one to run the union of the two configurations "
+	"which can be later filtered to produce distinct tables/figures. "
+	"\n"
+	"Soft limit works as follows: "
+	"It continues the search until the solution is found, or if all soft limits are met. ",
+        "0");
+    feature.add_option<double>(
+        "min_eval",
+        "specifies a soft limit i.e. the minimum number of evaluated states (measured by statistics.get_evaluated_states())."
+	"\n"
+	"0 and negative values indicate that the soft limit is disabled. ",
+        "0");
+    feature.add_option<double>(
+        "min_gen",
+        "specifies a soft limit i.e. minimum number of generated states (measured by statistics.get_generated())."
+	"\n"
+	"0 and negative values indicate that the soft limit is disabled. ",
+        "0");
+    feature.add_option<double>(
+        "min_exp",
+        "specifies a soft limit i.e. minimum number of expanded states (measured by statistics.get_expanded())."
+	"\n"
+	"0 and negative values indicate that the soft limit is disabled. ",
+        "0");
     feature.add_option<string>(
         "description",
         "description used to identify search algorithm in logs",
         "\"" + description + "\"");
     utils::add_log_options_to_feature(feature);
 }
 
-tuple<OperatorCost, int, double, string, utils::Verbosity>
+tuple<OperatorCost, int, double, double, double, double, double, double, double, double, string, utils::Verbosity>
 get_search_algorithm_arguments_from_options(
     const plugins::Options &opts) {
     return tuple_cat(
         ::get_cost_type_arguments_from_options(opts),
         make_tuple(
             opts.get<int>("bound"),
+            opts.get<double>("min_gen"),
+            opts.get<double>("min_eval"),
+            opts.get<double>("min_exp"),
+            opts.get<double>("min_time"),
+            opts.get<double>("max_gen"),
+            opts.get<double>("max_eval"),
+            opts.get<double>("max_exp"),
             opts.get<double>("max_time"),
             opts.get<string>("description")
             ),

src/search/search_algorithm.h

@@ -51,6 +51,13 @@ class SearchAlgorithm {
     int bound;
     OperatorCost cost_type;
     bool is_unit_cost;
+    double min_gen;
+    double min_eval;
+    double min_exp;
+    double min_time;
+    double max_gen;
+    double max_eval;
+    double max_exp;
     double max_time;
 
     virtual void initialize() {}
@@ -61,7 +68,8 @@ class SearchAlgorithm {
     int get_adjusted_cost(const OperatorProxy &op) const;
 public:
     SearchAlgorithm(
-        OperatorCost cost_type, int bound, double max_time,
+        OperatorCost cost_type, int bound,
+	double, double, double, double, double, double, double, double,
         const std::string &description, utils::Verbosity verbosity);
     explicit SearchAlgorithm(const plugins::Options &opts); // TODO options object is needed for iterated search, the prototype for issue559 resolves this
     virtual ~SearchAlgorithm();
@@ -97,7 +105,7 @@ get_search_pruning_arguments_from_options(const plugins::Options &opts);
 extern void add_search_algorithm_options_to_feature(
     plugins::Feature &feature, const std::string &description);
 extern std::tuple<
-    OperatorCost, int, double, std::string, utils::Verbosity>
+    OperatorCost, int, double, double, double, double, double, double, double, double, std::string, utils::Verbosity>
 get_search_algorithm_arguments_from_options(
     const plugins::Options &opts);
 extern void add_successors_order_options_to_feature(

src/search/search_algorithms/eager_search.cc

@@ -25,10 +25,28 @@ EagerSearch::EagerSearch(
     const vector<shared_ptr<Evaluator>> &preferred,
     const shared_ptr<PruningMethod> &pruning,
     const shared_ptr<Evaluator> &lazy_evaluator, OperatorCost cost_type,
-    int bound, double max_time, const string &description,
+    int bound,
+    double min_gen,
+    double min_eval,
+    double min_exp,
+    double min_time,
+    double max_gen,
+    double max_eval,
+    double max_exp,
+    double max_time,
+    const string &description,
     utils::Verbosity verbosity)
     : SearchAlgorithm(
-          cost_type, bound, max_time, description, verbosity),
+          cost_type, bound,
+	  min_gen,
+	  min_eval,
+	  min_exp,
+	  min_time,
+	  max_gen,
+	  max_eval,
+	  max_exp,
+	  max_time,
+	  description, verbosity),
       reopen_closed_nodes(reopen_closed),
       open_list(open->create_state_open_list()),
       f_evaluator(f_eval),     // default nullptr
@@ -319,7 +337,9 @@ void add_eager_search_options_to_feature(
 }
 
 tuple<shared_ptr<PruningMethod>, shared_ptr<Evaluator>, OperatorCost,
-      int, double, string, utils::Verbosity>
+      int,
+      double, double, double, double, double, double, double, double,
+      string, utils::Verbosity>
 get_eager_search_arguments_from_options(const plugins::Options &opts) {
     return tuple_cat(
         get_search_pruning_arguments_from_options(opts),

src/search/search_algorithms/eager_search.h

@@ -43,7 +43,8 @@ class EagerSearch : public SearchAlgorithm {
         const std::vector<std::shared_ptr<Evaluator>> &preferred,
         const std::shared_ptr<PruningMethod> &pruning,
         const std::shared_ptr<Evaluator> &lazy_evaluator,
-        OperatorCost cost_type, int bound, double max_time,
+        OperatorCost cost_type, int bound,
+	double, double, double, double, double, double, double, double,
         const std::string &description, utils::Verbosity verbosity);
 
     virtual void print_statistics() const override;
@@ -54,7 +55,8 @@ class EagerSearch : public SearchAlgorithm {
 extern void add_eager_search_options_to_feature(
     plugins::Feature &feature, const std::string &description);
 extern std::tuple<std::shared_ptr<PruningMethod>,
-                  std::shared_ptr<Evaluator>, OperatorCost, int, double,
+                  std::shared_ptr<Evaluator>, OperatorCost, int,
+		  double, double, double, double, double, double, double, double,
                   std::string, utils::Verbosity>
 get_eager_search_arguments_from_options(const plugins::Options &opts);
 }

src/search/search_algorithms/enforced_hill_climbing_search.cc

@@ -53,10 +53,27 @@ static shared_ptr<OpenListFactory> create_ehc_open_list_factory(
 EnforcedHillClimbingSearch::EnforcedHillClimbingSearch(
     const shared_ptr<Evaluator> &h, PreferredUsage preferred_usage,
     const vector<shared_ptr<Evaluator>> &preferred,
-    OperatorCost cost_type, int bound, double max_time,
+    OperatorCost cost_type, int bound,
+    double min_gen,
+    double min_eval,
+    double min_exp,
+    double min_time,
+    double max_gen,
+    double max_eval,
+    double max_exp,
+    double max_time,
     const string &description, utils::Verbosity verbosity)
     : SearchAlgorithm(
-          cost_type, bound, max_time, description, verbosity),
+          cost_type, bound,
+	  min_gen,
+	  min_eval,
+	  min_exp,
+	  min_time,
+	  max_gen,
+	  max_eval,
+	  max_exp,
+	  max_time,
+	  description, verbosity),
       evaluator(h),
       preferred_operator_evaluators(preferred),
       preferred_usage(preferred_usage),

src/search/search_algorithms/enforced_hill_climbing_search.h

@@ -64,7 +64,15 @@ class EnforcedHillClimbingSearch : public SearchAlgorithm {
         const std::shared_ptr<Evaluator> &h,
         PreferredUsage preferred_usage,
         const std::vector<std::shared_ptr<Evaluator>> &preferred,
-        OperatorCost cost_type, int bound, double max_time,
+        OperatorCost cost_type, int bound,
+	double min_gen,
+	double min_eval,
+	double min_exp,
+	double min_time,
+	double max_gen,
+	double max_eval,
+	double max_exp,
+	double max_time,
         const std::string &description, utils::Verbosity verbosity);
 
     virtual void print_statistics() const override;

src/search/search_algorithms/lazy_search.cc

@@ -21,10 +21,26 @@ LazySearch::LazySearch(
     const shared_ptr<OpenListFactory> &open, bool reopen_closed,
     const vector<shared_ptr<Evaluator>> &preferred,
     bool randomize_successors, bool preferred_successors_first,
-    int random_seed, OperatorCost cost_type, int bound, double max_time,
+    int random_seed, OperatorCost cost_type, int bound,
+    double min_gen,
+    double min_eval,
+    double min_exp,
+    double min_time,
+    double max_gen,
+    double max_eval,
+    double max_exp,
+    double max_time,
     const string &description, utils::Verbosity verbosity)
     : SearchAlgorithm(
-          cost_type, bound, max_time, description, verbosity),
+          cost_type, bound,
+	  min_gen,
+	  min_eval,
+	  min_exp,
+	  min_time,
+	  max_gen,
+	  max_eval,
+	  max_exp,
+	  max_time, description, verbosity),
       open_list(open->create_edge_open_list()),
       reopen_closed_nodes(reopen_closed),
       randomize_successors(randomize_successors),

src/search/search_algorithms/lazy_search.h

@@ -55,7 +55,15 @@ class LazySearch : public SearchAlgorithm {
         const std::vector<std::shared_ptr<Evaluator>> &evaluators,
         bool randomize_successors, bool preferred_successors_first,
         int random_seed, OperatorCost cost_type, int bound,
-        double max_time, const std::string &description,
+	double min_gen,
+	double min_eval,
+	double min_exp,
+	double min_time,
+	double max_gen,
+	double max_eval,
+	double max_exp,
+	double max_time,
+	const std::string &description,
         utils::Verbosity verbosity);
 
     virtual void print_statistics() const override;