From f415762853e669b2f4702dc6df3f71f23abcc9bb Mon Sep 17 00:00:00 2001 From: Masataro Asai Date: Tue, 14 Jan 2025 13:05:44 -0500 Subject: [PATCH 1/3] implemented exp/eval/gen limits --- src/search/search_algorithm.cc | 31 +++++++++++++++++++++++++++---- src/search/search_algorithm.h | 3 +++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/src/search/search_algorithm.cc b/src/search/search_algorithm.cc index c018c45c65..4b2a8340fa 100644 --- a/src/search/search_algorithm.cc +++ b/src/search/search_algorithm.cc @@ -77,6 +77,9 @@ SearchAlgorithm::SearchAlgorithm(const plugins::Options &opts) // TODO options o statistics(log), cost_type(opts.get("cost_type")), is_unit_cost(task_properties::is_unit_cost(task_proxy)), + max_gen(opts.get("max_gen")), + max_eval(opts.get("max_eval")), + max_exp(opts.get("max_exp")), max_time(opts.get("max_time")) { if (opts.get("bound") < 0) { cerr << "error: negative cost bound " << opts.get("bound") << endl; @@ -109,17 +112,25 @@ void SearchAlgorithm::set_plan(const Plan &p) { void SearchAlgorithm::search() { initialize(); - utils::CountdownTimer timer(max_time); + utils::g_log << "Hard limits:" << endl; + utils::g_log << "Max runtime: " << max_time << " sec" << endl; + utils::g_log << "Max evaluations: " << max_eval << " states" << endl; + utils::g_log << "Max expansions: " << max_exp << " states" << endl; + utils::g_log << "Max generations: " << max_gen << " states" << endl; + utils::CountdownTimer timer_max(max_time); while (status == IN_PROGRESS) { status = step(); - if (timer.is_expired()) { - log << "Time limit reached. Abort search." << endl; + auto ex = statistics.get_expanded(); + auto ev = statistics.get_evaluated_states(); + auto gen = statistics.get_generated(); + if (timer_max.is_expired() || (gen >= max_gen) || (ev >= max_eval) || (ex >= max_exp)) { + utils::g_log << "One of the hard limits is reached. Aborting search." << endl; status = TIMEOUT; break; } } // TODO: Revise when and which search times are logged. - log << "Actual search time: " << timer.get_elapsed_time() << endl; + log << "Actual search time: " << timer_max.get_elapsed_time() << endl; } bool SearchAlgorithm::check_goal_and_set_plan(const State &state) { @@ -192,6 +203,18 @@ void add_search_algorithm_options_to_feature( "experiments. Timed-out searches are treated as failed searches, " "just like incomplete search algorithms that exhaust their search space.", "infinity"); + feature.add_option( + "max_eval", + "maximum number of evaluated states (measured by statistics.get_evaluated_states()).", + "infinity"); + feature.add_option( + "max_gen", + "maximum number of generated states (measured by statistics.get_generated()).", + "infinity"); + feature.add_option( + "max_exp", + "maximum number of expanded states (measured by statistics.get_expanded()).", + "infinity"); feature.add_option( "description", "description used to identify search algorithm in logs", diff --git a/src/search/search_algorithm.h b/src/search/search_algorithm.h index 29103c7b0f..d87bf56d4f 100644 --- a/src/search/search_algorithm.h +++ b/src/search/search_algorithm.h @@ -51,6 +51,9 @@ class SearchAlgorithm { int bound; OperatorCost cost_type; bool is_unit_cost; + double max_gen; + double max_eval; + double max_exp; double max_time; virtual void initialize() {} From 7c2e2fa7c793115ee966d79e726f5fba2128a56a Mon Sep 17 00:00:00 2001 From: Masataro Asai Date: Tue, 14 Jan 2025 13:07:04 -0500 Subject: [PATCH 2/3] implemented soft limits --- src/search/search_algorithm.cc | 73 ++++++++++++++++++++++++++++++++++ src/search/search_algorithm.h | 4 ++ 2 files changed, 77 insertions(+) diff --git a/src/search/search_algorithm.cc b/src/search/search_algorithm.cc index 4b2a8340fa..66649fb1a9 100644 --- a/src/search/search_algorithm.cc +++ b/src/search/search_algorithm.cc @@ -77,6 +77,10 @@ SearchAlgorithm::SearchAlgorithm(const plugins::Options &opts) // TODO options o statistics(log), cost_type(opts.get("cost_type")), is_unit_cost(task_properties::is_unit_cost(task_proxy)), + min_gen(opts.get("min_gen")), + min_eval(opts.get("min_eval")), + min_exp(opts.get("min_exp")), + min_time(opts.get("min_time")), max_gen(opts.get("max_gen")), max_eval(opts.get("max_eval")), max_exp(opts.get("max_exp")), @@ -117,7 +121,29 @@ void SearchAlgorithm::search() { utils::g_log << "Max evaluations: " << max_eval << " states" << endl; utils::g_log << "Max expansions: " << max_exp << " states" << endl; utils::g_log << "Max generations: " << max_gen << " states" << endl; + + // Implementing the default values for soft limits is a bit complicated because + // the default "unspecified" value requires a special treatment. + // + // Setting the default value to a small (e.g. negative) value which is always reached is wrong because + // the search stops immediately when no soft limits are given for exp/gen/eval/elapsed. + // + // Setting the default value to a large (e.g. infinity) value is also wrong, + // because such a limit is never reached even if all other *specified* soft limits are reached, + // making the search continue although it is supposed to stop. + + bool use_soft_limit = (min_time > 0) || (min_gen > 0) || (min_eval > 0) || (min_exp > 0); + if (use_soft_limit){ + utils::g_log << "Soft limits:" << endl; + utils::g_log << "Min runtime: " << min_time << " sec" << endl; + utils::g_log << "Min evaluations: " << min_eval << " states" << endl; + utils::g_log << "Min expansions: " << min_exp << " states" << endl; + utils::g_log << "Min generations: " << min_gen << " states" << endl; + } + utils::CountdownTimer timer_max(max_time); + utils::CountdownTimer timer_min(min_time); + while (status == IN_PROGRESS) { status = step(); auto ex = statistics.get_expanded(); @@ -128,6 +154,16 @@ void SearchAlgorithm::search() { status = TIMEOUT; break; } + if (use_soft_limit) { // note: without this, it stops immediately when all soft limits are 0 + if ( (min_time > 0 && timer_min.is_expired()) + && (min_gen > 0 && gen >= min_gen ) + && (min_eval > 0 && ev >= min_eval) + && (min_exp > 0 && ex >= min_exp )) { + utils::g_log << "Soft limit: Spent all minimum required amount of computation. Aborting search." << endl; + status = TIMEOUT; + break; + } + } } // TODO: Revise when and which search times are logged. log << "Actual search time: " << timer_max.get_elapsed_time() << endl; @@ -215,6 +251,43 @@ void add_search_algorithm_options_to_feature( "max_exp", "maximum number of expanded states (measured by statistics.get_expanded()).", "infinity"); + feature.add_option( + "min_time", + "specifies a soft limit i.e. minimum time in seconds the search must run for. " + "The same accuracy statement as the one for max_time applies to this option. " + "\n" + "0 and negative values indicate that the soft limit is disabled. " + "\n" + "Soft limit addresses the limitation of hard limits (e.g. max_time) that one limit interferes another. " + "For example, when max_eval = 100k and max_time = 5 sec, " + "majority of runs are cut off before reaching 100k evaluations, " + "and the result is not an appropriate representation of running 100k evaluations. " + "To measure the evaluations and time separately, therefore one would run two separate experiments, " + "which is a waste of compute for instances solved by both. " + "Instead of running them separately, this option allows one to run the union of the two configurations " + "which can be later filtered to produce distinct tables/figures. " + "\n" + "Soft limit works as follows: " + "It continues the search until the solution is found, or if all soft limits are met. ", + "0"); + feature.add_option( + "min_eval", + "specifies a soft limit i.e. the minimum number of evaluated states (measured by statistics.get_evaluated_states())." + "\n" + "0 and negative values indicate that the soft limit is disabled. ", + "0"); + feature.add_option( + "min_gen", + "specifies a soft limit i.e. minimum number of generated states (measured by statistics.get_generated())." + "\n" + "0 and negative values indicate that the soft limit is disabled. ", + "0"); + feature.add_option( + "min_exp", + "specifies a soft limit i.e. minimum number of expanded states (measured by statistics.get_expanded())." + "\n" + "0 and negative values indicate that the soft limit is disabled. ", + "0"); feature.add_option( "description", "description used to identify search algorithm in logs", diff --git a/src/search/search_algorithm.h b/src/search/search_algorithm.h index d87bf56d4f..6f4757c832 100644 --- a/src/search/search_algorithm.h +++ b/src/search/search_algorithm.h @@ -51,6 +51,10 @@ class SearchAlgorithm { int bound; OperatorCost cost_type; bool is_unit_cost; + double min_gen; + double min_eval; + double min_exp; + double min_time; double max_gen; double max_eval; double max_exp; From 8f720b8a97f37f588515799f5c14fd190ab45b59 Mon Sep 17 00:00:00 2001 From: Masataro Asai Date: Tue, 14 Jan 2025 13:21:03 -0500 Subject: [PATCH 3/3] fixing constructors --- src/search/search_algorithm.cc | 26 +++++++++++++++++-- src/search/search_algorithm.h | 5 ++-- src/search/search_algorithms/eager_search.cc | 26 ++++++++++++++++--- src/search/search_algorithms/eager_search.h | 6 +++-- .../enforced_hill_climbing_search.cc | 21 +++++++++++++-- .../enforced_hill_climbing_search.h | 10 ++++++- src/search/search_algorithms/lazy_search.cc | 20 ++++++++++++-- src/search/search_algorithms/lazy_search.h | 10 ++++++- 8 files changed, 109 insertions(+), 15 deletions(-) diff --git a/src/search/search_algorithm.cc b/src/search/search_algorithm.cc index 66649fb1a9..c21bea9813 100644 --- a/src/search/search_algorithm.cc +++ b/src/search/search_algorithm.cc @@ -40,7 +40,15 @@ static successor_generator::SuccessorGenerator &get_successor_generator( } SearchAlgorithm::SearchAlgorithm( - OperatorCost cost_type, int bound, double max_time, + OperatorCost cost_type, int bound, + double min_gen, + double min_eval, + double min_exp, + double min_time, + double max_gen, + double max_eval, + double max_exp, + double max_time, const string &description, utils::Verbosity verbosity) : description(description), status(IN_PROGRESS), @@ -55,6 +63,13 @@ SearchAlgorithm::SearchAlgorithm( bound(bound), cost_type(cost_type), is_unit_cost(task_properties::is_unit_cost(task_proxy)), + min_gen(min_gen), + min_eval(min_eval), + min_exp(min_exp), + min_time(min_time), + max_gen(max_gen), + max_eval(max_eval), + max_exp(max_exp), max_time(max_time) { if (bound < 0) { cerr << "error: negative cost bound " << bound << endl; @@ -295,13 +310,20 @@ void add_search_algorithm_options_to_feature( utils::add_log_options_to_feature(feature); } -tuple +tuple get_search_algorithm_arguments_from_options( const plugins::Options &opts) { return tuple_cat( ::get_cost_type_arguments_from_options(opts), make_tuple( opts.get("bound"), + opts.get("min_gen"), + opts.get("min_eval"), + opts.get("min_exp"), + opts.get("min_time"), + opts.get("max_gen"), + opts.get("max_eval"), + opts.get("max_exp"), opts.get("max_time"), opts.get("description") ), diff --git a/src/search/search_algorithm.h b/src/search/search_algorithm.h index 6f4757c832..6226a211a5 100644 --- a/src/search/search_algorithm.h +++ b/src/search/search_algorithm.h @@ -68,7 +68,8 @@ class SearchAlgorithm { int get_adjusted_cost(const OperatorProxy &op) const; public: SearchAlgorithm( - OperatorCost cost_type, int bound, double max_time, + OperatorCost cost_type, int bound, + double, double, double, double, double, double, double, double, const std::string &description, utils::Verbosity verbosity); explicit SearchAlgorithm(const plugins::Options &opts); // TODO options object is needed for iterated search, the prototype for issue559 resolves this virtual ~SearchAlgorithm(); @@ -104,7 +105,7 @@ get_search_pruning_arguments_from_options(const plugins::Options &opts); extern void add_search_algorithm_options_to_feature( plugins::Feature &feature, const std::string &description); extern std::tuple< - OperatorCost, int, double, std::string, utils::Verbosity> + OperatorCost, int, double, double, double, double, double, double, double, double, std::string, utils::Verbosity> get_search_algorithm_arguments_from_options( const plugins::Options &opts); extern void add_successors_order_options_to_feature( diff --git a/src/search/search_algorithms/eager_search.cc b/src/search/search_algorithms/eager_search.cc index 951906fbcc..fb78abdbc2 100644 --- a/src/search/search_algorithms/eager_search.cc +++ b/src/search/search_algorithms/eager_search.cc @@ -25,10 +25,28 @@ EagerSearch::EagerSearch( const vector> &preferred, const shared_ptr &pruning, const shared_ptr &lazy_evaluator, OperatorCost cost_type, - int bound, double max_time, const string &description, + int bound, + double min_gen, + double min_eval, + double min_exp, + double min_time, + double max_gen, + double max_eval, + double max_exp, + double max_time, + const string &description, utils::Verbosity verbosity) : SearchAlgorithm( - cost_type, bound, max_time, description, verbosity), + cost_type, bound, + min_gen, + min_eval, + min_exp, + min_time, + max_gen, + max_eval, + max_exp, + max_time, + description, verbosity), reopen_closed_nodes(reopen_closed), open_list(open->create_state_open_list()), f_evaluator(f_eval), // default nullptr @@ -319,7 +337,9 @@ void add_eager_search_options_to_feature( } tuple, shared_ptr, OperatorCost, - int, double, string, utils::Verbosity> + int, + double, double, double, double, double, double, double, double, + string, utils::Verbosity> get_eager_search_arguments_from_options(const plugins::Options &opts) { return tuple_cat( get_search_pruning_arguments_from_options(opts), diff --git a/src/search/search_algorithms/eager_search.h b/src/search/search_algorithms/eager_search.h index bb328e1fc7..3236e9075b 100644 --- a/src/search/search_algorithms/eager_search.h +++ b/src/search/search_algorithms/eager_search.h @@ -43,7 +43,8 @@ class EagerSearch : public SearchAlgorithm { const std::vector> &preferred, const std::shared_ptr &pruning, const std::shared_ptr &lazy_evaluator, - OperatorCost cost_type, int bound, double max_time, + OperatorCost cost_type, int bound, + double, double, double, double, double, double, double, double, const std::string &description, utils::Verbosity verbosity); virtual void print_statistics() const override; @@ -54,7 +55,8 @@ class EagerSearch : public SearchAlgorithm { extern void add_eager_search_options_to_feature( plugins::Feature &feature, const std::string &description); extern std::tuple, - std::shared_ptr, OperatorCost, int, double, + std::shared_ptr, OperatorCost, int, + double, double, double, double, double, double, double, double, std::string, utils::Verbosity> get_eager_search_arguments_from_options(const plugins::Options &opts); } diff --git a/src/search/search_algorithms/enforced_hill_climbing_search.cc b/src/search/search_algorithms/enforced_hill_climbing_search.cc index a8c1b3b945..de52092df1 100644 --- a/src/search/search_algorithms/enforced_hill_climbing_search.cc +++ b/src/search/search_algorithms/enforced_hill_climbing_search.cc @@ -53,10 +53,27 @@ static shared_ptr create_ehc_open_list_factory( EnforcedHillClimbingSearch::EnforcedHillClimbingSearch( const shared_ptr &h, PreferredUsage preferred_usage, const vector> &preferred, - OperatorCost cost_type, int bound, double max_time, + OperatorCost cost_type, int bound, + double min_gen, + double min_eval, + double min_exp, + double min_time, + double max_gen, + double max_eval, + double max_exp, + double max_time, const string &description, utils::Verbosity verbosity) : SearchAlgorithm( - cost_type, bound, max_time, description, verbosity), + cost_type, bound, + min_gen, + min_eval, + min_exp, + min_time, + max_gen, + max_eval, + max_exp, + max_time, + description, verbosity), evaluator(h), preferred_operator_evaluators(preferred), preferred_usage(preferred_usage), diff --git a/src/search/search_algorithms/enforced_hill_climbing_search.h b/src/search/search_algorithms/enforced_hill_climbing_search.h index f7b3247dfd..6d5ed3c6e2 100644 --- a/src/search/search_algorithms/enforced_hill_climbing_search.h +++ b/src/search/search_algorithms/enforced_hill_climbing_search.h @@ -64,7 +64,15 @@ class EnforcedHillClimbingSearch : public SearchAlgorithm { const std::shared_ptr &h, PreferredUsage preferred_usage, const std::vector> &preferred, - OperatorCost cost_type, int bound, double max_time, + OperatorCost cost_type, int bound, + double min_gen, + double min_eval, + double min_exp, + double min_time, + double max_gen, + double max_eval, + double max_exp, + double max_time, const std::string &description, utils::Verbosity verbosity); virtual void print_statistics() const override; diff --git a/src/search/search_algorithms/lazy_search.cc b/src/search/search_algorithms/lazy_search.cc index b4f933ceac..d72d713e15 100644 --- a/src/search/search_algorithms/lazy_search.cc +++ b/src/search/search_algorithms/lazy_search.cc @@ -21,10 +21,26 @@ LazySearch::LazySearch( const shared_ptr &open, bool reopen_closed, const vector> &preferred, bool randomize_successors, bool preferred_successors_first, - int random_seed, OperatorCost cost_type, int bound, double max_time, + int random_seed, OperatorCost cost_type, int bound, + double min_gen, + double min_eval, + double min_exp, + double min_time, + double max_gen, + double max_eval, + double max_exp, + double max_time, const string &description, utils::Verbosity verbosity) : SearchAlgorithm( - cost_type, bound, max_time, description, verbosity), + cost_type, bound, + min_gen, + min_eval, + min_exp, + min_time, + max_gen, + max_eval, + max_exp, + max_time, description, verbosity), open_list(open->create_edge_open_list()), reopen_closed_nodes(reopen_closed), randomize_successors(randomize_successors), diff --git a/src/search/search_algorithms/lazy_search.h b/src/search/search_algorithms/lazy_search.h index 3c5ac25ebe..0f3c099565 100644 --- a/src/search/search_algorithms/lazy_search.h +++ b/src/search/search_algorithms/lazy_search.h @@ -55,7 +55,15 @@ class LazySearch : public SearchAlgorithm { const std::vector> &evaluators, bool randomize_successors, bool preferred_successors_first, int random_seed, OperatorCost cost_type, int bound, - double max_time, const std::string &description, + double min_gen, + double min_eval, + double min_exp, + double min_time, + double max_gen, + double max_eval, + double max_exp, + double max_time, + const std::string &description, utils::Verbosity verbosity); virtual void print_statistics() const override;