From 935019896ae2f91bce02731e2999d0929059b9f3 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 13 Feb 2026 13:03:31 -0600 Subject: [PATCH] RB-462 adding a integrated RtBot Diagnosis procedure Co-Authored-By: Claude Opus 4.6 --- .bazelrc | 2 +- libs/api/BUILD.bazel | 1 + libs/api/include/rtbot/Diagnostics.h | 640 ++++++++++++++++++++++++++ libs/api/include/rtbot/bindings.h | 1 + libs/api/src/bindings.cpp | 6 + libs/api/test/test_diagnostics.cpp | 465 +++++++++++++++++++ libs/api/wasm/emscripten-bindings.cpp | 1 + tools/BUILD.bazel | 9 + tools/diagnose_file.cpp | 31 ++ 9 files changed, 1155 insertions(+), 1 deletion(-) create mode 100644 libs/api/include/rtbot/Diagnostics.h create mode 100644 libs/api/test/test_diagnostics.cpp create mode 100644 tools/BUILD.bazel create mode 100644 tools/diagnose_file.cpp diff --git a/.bazelrc b/.bazelrc index be5857ff..7ea62806 100644 --- a/.bazelrc +++ b/.bazelrc @@ -16,7 +16,7 @@ test --notest_verbose_timeout_warnings # jsonschema validation library to compile # the issue appears while including the formatter and specifically # for the stmp format validation, which we don't use -build --action_env=EMCC_CFLAGS="-fexceptions -Wno-c++11-narrowing -Wno-unused-const-variable" +build --action_env=EMCC_CFLAGS="-fexceptions -Wno-c++11-narrowing -Wno-unused-const-variable -Wno-unused-command-line-argument" build --incompatible_enable_cc_toolchain_resolution build --enable_platform_specific_config diff --git a/libs/api/BUILD.bazel b/libs/api/BUILD.bazel index 60c618b7..909a83b8 100644 --- a/libs/api/BUILD.bazel +++ b/libs/api/BUILD.bazel @@ -86,6 +86,7 @@ genrule( echo " getProgramEntryOperatorId(programId: string): string;" >> $@ echo " getProgramEntryPorts(programId: string): string;" >> $@ echo " getProgramOutputFilter(programId: string): string;" >> $@ + echo " diagnoseProgram(programStr: string): string;" >> $@ echo " processBatch(programId: string, times: number[], values: number[], ports: string[]): string;" >> $@ echo "}" >> $@ echo "declare function factory(): Promise;" >> $@ diff --git a/libs/api/include/rtbot/Diagnostics.h b/libs/api/include/rtbot/Diagnostics.h new file mode 100644 index 00000000..3c94bebe --- /dev/null +++ b/libs/api/include/rtbot/Diagnostics.h @@ -0,0 +1,640 @@ +#ifndef RTBOT_DIAGNOSTICS_H +#define RTBOT_DIAGNOSTICS_H + +#include +#include +#include +#include +#include +#include + +#include "rtbot/OperatorJson.h" +#include "rtbot/Prototype.h" +#include "rtbot/jsonschema.hpp" + +namespace rtbot { + +using json = nlohmann::json; + +// --------------------------------------------------------------------------- +// Error codes +// --------------------------------------------------------------------------- +enum class DiagnosticCode { + // Phase 1: JSON parse + INVALID_JSON, + + // Phase 2: Prototype resolution + UNKNOWN_PROTOTYPE, + UNKNOWN_PARAMETER, + MISSING_PARAMETER, + INVALID_PARAMETER_TYPE, + UNKNOWN_PARAM_REFERENCE, + + // Phase 3: Schema validation + SCHEMA_VALIDATION, + + // Phase 4: Semantic + UNKNOWN_OPERATOR_TYPE, + INVALID_PORT_TYPE, + INVALID_PARAMETER_VALUE, + MISSING_REQUIRED_FIELD, + + INVALID_OPERATOR_REF, + PORT_INDEX_OUT_OF_BOUNDS, + PORT_TYPE_MISMATCH, + + INVALID_ENTRY_OPERATOR, + ENTRY_PORT_MISMATCH, + + MISSING_OUTPUT_FIELD, + INVALID_OUTPUT_MAPPING, +}; + +inline std::string diagnostic_code_to_string(DiagnosticCode code) { + switch (code) { + case DiagnosticCode::INVALID_JSON: + return "INVALID_JSON"; + case DiagnosticCode::UNKNOWN_PROTOTYPE: + return "UNKNOWN_PROTOTYPE"; + case DiagnosticCode::UNKNOWN_PARAMETER: + return "UNKNOWN_PARAMETER"; + case DiagnosticCode::MISSING_PARAMETER: + return "MISSING_PARAMETER"; + case DiagnosticCode::INVALID_PARAMETER_TYPE: + return "INVALID_PARAMETER_TYPE"; + case DiagnosticCode::UNKNOWN_PARAM_REFERENCE: + return "UNKNOWN_PARAM_REFERENCE"; + case DiagnosticCode::SCHEMA_VALIDATION: + return "SCHEMA_VALIDATION"; + case DiagnosticCode::UNKNOWN_OPERATOR_TYPE: + return "UNKNOWN_OPERATOR_TYPE"; + case DiagnosticCode::INVALID_PORT_TYPE: + return "INVALID_PORT_TYPE"; + case DiagnosticCode::INVALID_PARAMETER_VALUE: + return "INVALID_PARAMETER_VALUE"; + case DiagnosticCode::MISSING_REQUIRED_FIELD: + return "MISSING_REQUIRED_FIELD"; + case DiagnosticCode::INVALID_OPERATOR_REF: + return "INVALID_OPERATOR_REF"; + case DiagnosticCode::PORT_INDEX_OUT_OF_BOUNDS: + return "PORT_INDEX_OUT_OF_BOUNDS"; + case DiagnosticCode::PORT_TYPE_MISMATCH: + return "PORT_TYPE_MISMATCH"; + case DiagnosticCode::INVALID_ENTRY_OPERATOR: + return "INVALID_ENTRY_OPERATOR"; + case DiagnosticCode::ENTRY_PORT_MISMATCH: + return "ENTRY_PORT_MISMATCH"; + case DiagnosticCode::MISSING_OUTPUT_FIELD: + return "MISSING_OUTPUT_FIELD"; + case DiagnosticCode::INVALID_OUTPUT_MAPPING: + return "INVALID_OUTPUT_MAPPING"; + } + return "UNKNOWN"; +} + +// --------------------------------------------------------------------------- +// Diagnostic structs +// --------------------------------------------------------------------------- +struct DiagnosticError { + std::string severity; // "error" or "warning" + std::string path; // JSON pointer, e.g. "/operators/2/window_size" + std::string message; // Human-readable explanation + DiagnosticCode code; // Programmatic error code + std::string suggestion; // Optional remediation hint (may be empty) +}; + +inline void to_json(json& j, const DiagnosticError& e) { + j = json{ + {"severity", e.severity}, + {"path", e.path}, + {"message", e.message}, + {"code", diagnostic_code_to_string(e.code)}, + }; + if (!e.suggestion.empty()) { + j["suggestion"] = e.suggestion; + } +} + +struct DiagnosticResult { + bool valid; + std::vector errors; +}; + +inline void to_json(json& j, const DiagnosticResult& r) { + j = json{{"valid", r.valid}, {"errors", r.errors}}; +} + +// --------------------------------------------------------------------------- +// ProgramDiagnostics +// --------------------------------------------------------------------------- +class ProgramDiagnostics { + public: + static DiagnosticResult diagnose(const std::string& json_string) { + DiagnosticResult result; + result.valid = true; + + // Phase 1: JSON Parse + json j; + if (!phase_json_parse(json_string, j, result)) { + result.valid = false; + return result; + } + + // Phase 2: Prototype Resolution + if (!phase_prototype_resolution(j, result)) { + result.valid = false; + return result; + } + + // Phase 3: Schema Validation + if (!phase_schema_validation(j, result)) { + result.valid = false; + return result; + } + + // Phase 4: Semantic Validation + std::map> operators; + + // 4a: Operator creation + phase_operator_creation(j, operators, result); + bool had_operator_errors = !result.errors.empty(); + + // 4b: Connection wiring (only if all operators created successfully) + if (!had_operator_errors) { + phase_connection_wiring(j, operators, result); + } + + // 4c: Entry operator validation + phase_entry_operator(j, operators, result); + + // 4d: Output mappings validation + phase_output_mappings(j, operators, result); + + result.valid = result.errors.empty(); + return result; + } + + private: + // ------------------------------------------------------------------------- + // Utility helpers + // ------------------------------------------------------------------------- + static std::string join_vec(const std::vector& v) { + std::string result; + for (size_t i = 0; i < v.size(); i++) { + if (i > 0) result += ", "; + result += v[i]; + } + return result; + } + + static std::vector get_operator_ids(const std::map>& operators) { + std::vector ids; + for (const auto& kv : operators) { + ids.push_back(kv.first); + } + return ids; + } + + static bool check_param_type(const std::string& type, const json& value) { + if (type == "number") return value.is_number(); + if (type == "string") return value.is_string(); + if (type == "boolean") return value.is_boolean(); + if (type == "array") return value.is_array(); + if (type == "object") return value.is_object(); + return false; + } + + // C++17-compatible suffix check (no std::string::ends_with) + static bool ends_with(const std::string& str, const std::string& suffix) { + if (suffix.size() > str.size()) return false; + return str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0; + } + + static std::shared_ptr resolve_operator( + const std::string& id, const std::map>& operators) { + auto it = operators.find(id); + if (it != operators.end()) return it->second; + + std::string suffix = "::" + id; + for (const auto& kv : operators) { + if (ends_with(kv.first, suffix) || kv.first == id) { + return kv.second; + } + } + return nullptr; + } + + // ------------------------------------------------------------------------- + // Error classification helpers + // ------------------------------------------------------------------------- + static DiagnosticCode classify_operator_error(const std::string& msg) { + if (msg.find("Unknown operator type") != std::string::npos) return DiagnosticCode::UNKNOWN_OPERATOR_TYPE; + if (msg.find("port type") != std::string::npos || msg.find("Invalid input port type") != std::string::npos || + msg.find("Invalid output port type") != std::string::npos) + return DiagnosticCode::INVALID_PORT_TYPE; + if (msg.find("must have id and type") != std::string::npos || + msg.find("must specify from and to") != std::string::npos || + msg.find("must have at least") != std::string::npos || + msg.find("must contain at least") != std::string::npos) + return DiagnosticCode::MISSING_REQUIRED_FIELD; + if (msg.find("cannot be empty") != std::string::npos || msg.find("must be at least") != std::string::npos || + msg.find("must be odd") != std::string::npos || msg.find("requires at least") != std::string::npos || + msg.find("must be positive") != std::string::npos || msg.find("At least") != std::string::npos) + return DiagnosticCode::INVALID_PARAMETER_VALUE; + if (msg.find("Invalid port name format") != std::string::npos) return DiagnosticCode::PORT_INDEX_OUT_OF_BOUNDS; + if (msg.find("Entry operator not found") != std::string::npos) return DiagnosticCode::INVALID_ENTRY_OPERATOR; + if (msg.find("Entry operator has less data ports") != std::string::npos) return DiagnosticCode::ENTRY_PORT_MISMATCH; + if (msg.find("Output operator not found") != std::string::npos) return DiagnosticCode::INVALID_OUTPUT_MAPPING; + if (msg.find("Invalid pipeline output port") != std::string::npos) return DiagnosticCode::INVALID_OUTPUT_MAPPING; + if (msg.find("invalid operator reference") != std::string::npos) return DiagnosticCode::INVALID_OPERATOR_REF; + if (msg.find("type mismatch") != std::string::npos) return DiagnosticCode::PORT_TYPE_MISMATCH; + if (msg.find("Invalid output port index") != std::string::npos || + msg.find("Invalid child data port") != std::string::npos || + msg.find("Invalid child control port") != std::string::npos) + return DiagnosticCode::PORT_INDEX_OUT_OF_BOUNDS; + return DiagnosticCode::INVALID_PARAMETER_VALUE; + } + + static std::string refine_operator_error_path(const std::string& base_path, const std::string& msg, + const json& op_json) { + if ((msg.find("coefficients") != std::string::npos || msg.find("FIR") != std::string::npos) && + op_json.contains("coeff")) + return base_path + "/coeff"; + if (msg.find("coefficient") != std::string::npos && op_json.contains("coefficients")) + return base_path + "/coefficients"; + if (msg.find("b_coeffs") != std::string::npos || msg.find("a_coeffs") != std::string::npos) { + if (op_json.contains("b_coeffs")) return base_path + "/b_coeffs"; + if (op_json.contains("a_coeffs")) return base_path + "/a_coeffs"; + } + if (msg.find("window size") != std::string::npos || msg.find("window_size") != std::string::npos) + return base_path + "/window_size"; + if (msg.find("port type") != std::string::npos && op_json.contains("portTypes")) + return base_path + "/portTypes"; + if (msg.find("port type") != std::string::npos && op_json.contains("input_port_types")) + return base_path + "/input_port_types"; + if (msg.find("interval") != std::string::npos || msg.find("Time interval") != std::string::npos || + msg.find("Resampling interval") != std::string::npos) + return base_path + "/interval"; + if (msg.find("points") != std::string::npos || msg.find("interpolation") != std::string::npos) + return base_path + "/points"; + if (msg.find("Entry operator") != std::string::npos) return base_path + "/entryOperator"; + return base_path; + } + + static std::string generate_operator_suggestion(DiagnosticCode code, const std::string& msg) { + switch (code) { + case DiagnosticCode::UNKNOWN_OPERATOR_TYPE: + return "Available types include: Input, Output, MovingAverage, StandardDeviation, " + "FiniteImpulseResponse, InfiniteImpulseResponse, PeakDetector, Linear, " + "Scale, Add, Power, Pipeline, Join, Function, Identity, Difference, etc."; + case DiagnosticCode::INVALID_PORT_TYPE: + return "Valid port types: number, boolean, vector_number, vector_boolean"; + case DiagnosticCode::INVALID_PARAMETER_VALUE: + if (msg.find("PeakDetector") != std::string::npos && msg.find("odd") != std::string::npos) + return "Use an odd number >= 3 (e.g., 3, 5, 7)"; + if (msg.find("PeakDetector") != std::string::npos && msg.find("at least 3") != std::string::npos) + return "Window size must be an odd number >= 3"; + if (msg.find("FIR") != std::string::npos || msg.find("cannot be empty") != std::string::npos) + return "Provide at least one coefficient"; + if (msg.find("interval") != std::string::npos) return "Interval must be a positive integer"; + break; + default: + break; + } + return ""; + } + + // ------------------------------------------------------------------------- + // Phase 1: JSON Parse + // ------------------------------------------------------------------------- + static bool phase_json_parse(const std::string& input, json& parsed, DiagnosticResult& result) { + try { + parsed = json::parse(input); + return true; + } catch (const json::parse_error& e) { + result.errors.push_back({"error", "", std::string("Invalid JSON: ") + e.what(), DiagnosticCode::INVALID_JSON, + "Ensure the input is valid JSON"}); + return false; + } + } + + // ------------------------------------------------------------------------- + // Phase 2: Prototype Resolution + // ------------------------------------------------------------------------- + static bool phase_prototype_resolution(json& j, DiagnosticResult& result) { + if (!j.contains("prototypes")) { + return true; + } + + std::vector errors; + + if (j.contains("operators") && j["operators"].is_array()) { + for (size_t i = 0; i < j["operators"].size(); i++) { + const auto& op = j["operators"][i]; + if (!op.contains("prototype")) continue; + + std::string proto_name = op["prototype"].get(); + std::string instance_id = op.value("id", ""); + std::string base_path = "/operators/" + std::to_string(i); + + // Check prototype exists + if (!j["prototypes"].contains(proto_name)) { + std::vector available; + for (auto it = j["prototypes"].begin(); it != j["prototypes"].end(); ++it) { + available.push_back(it.key()); + } + errors.push_back({"error", base_path + "/prototype", + "Unknown prototype: " + proto_name, DiagnosticCode::UNKNOWN_PROTOTYPE, + "Available prototypes: " + join_vec(available)}); + continue; + } + + const auto& proto_def = j["prototypes"][proto_name]; + if (!proto_def.contains("parameters")) continue; + + // Build parameter lookup + std::map param_types; + std::vector required_params; + std::vector all_param_names; + for (const auto& p : proto_def["parameters"]) { + std::string pname = p["name"].get(); + param_types[pname] = p["type"].get(); + all_param_names.push_back(pname); + if (!p.contains("default") || p["default"].is_null()) { + required_params.push_back(pname); + } + } + + json provided = op.value("parameters", json::object()); + + // Check for unknown parameters + for (auto it = provided.begin(); it != provided.end(); ++it) { + if (param_types.find(it.key()) == param_types.end()) { + errors.push_back({"error", base_path + "/parameters/" + it.key(), + "Unknown parameter '" + it.key() + "' in prototype instance '" + instance_id + "'", + DiagnosticCode::UNKNOWN_PARAMETER, + "Available parameters: " + join_vec(all_param_names)}); + } + } + + // Check for missing required parameters + for (const auto& req : required_params) { + if (!provided.contains(req)) { + errors.push_back({"error", base_path + "/parameters", + "Missing required parameter '" + req + "' in prototype instance '" + instance_id + "'", + DiagnosticCode::MISSING_PARAMETER, ""}); + } + } + + // Type-check provided parameters + for (auto it = provided.begin(); it != provided.end(); ++it) { + auto pt = param_types.find(it.key()); + if (pt == param_types.end()) continue; + if (!check_param_type(pt->second, it.value())) { + errors.push_back({"error", base_path + "/parameters/" + it.key(), + "Parameter '" + it.key() + "' must be a " + pt->second, + DiagnosticCode::INVALID_PARAMETER_TYPE, ""}); + } + } + } + } + + if (!errors.empty()) { + result.errors.insert(result.errors.end(), errors.begin(), errors.end()); + return false; + } + + // Pre-validation passed; actually resolve prototypes + try { + PrototypeHandler::resolve_prototypes(j); + return true; + } catch (const std::exception& e) { + std::string msg = e.what(); + DiagnosticCode code = DiagnosticCode::UNKNOWN_PARAM_REFERENCE; + if (msg.find("Unknown parameter '") != std::string::npos) + code = DiagnosticCode::UNKNOWN_PARAMETER; + else if (msg.find("Missing required parameter") != std::string::npos) + code = DiagnosticCode::MISSING_PARAMETER; + else if (msg.find("must be a") != std::string::npos) + code = DiagnosticCode::INVALID_PARAMETER_TYPE; + + result.errors.push_back({"error", "", msg, code, ""}); + return false; + } + } + + // ------------------------------------------------------------------------- + // Phase 3: Schema Validation + // ------------------------------------------------------------------------- + class DiagnosticSchemaHandler : public nlohmann::json_schema::basic_error_handler { + public: + std::vector errors; + + void error(const nlohmann::json::json_pointer& ptr, const nlohmann::json& /*instance*/, + const std::string& message) override { + nlohmann::json_schema::basic_error_handler::error(ptr, {}, message); + errors.push_back({"error", ptr.to_string(), message, DiagnosticCode::SCHEMA_VALIDATION, ""}); + } + }; + + static bool phase_schema_validation(const json& j, DiagnosticResult& result) { + nlohmann::json_schema::json_validator validator(nullptr, nlohmann::json_schema::default_string_format_check); + + try { + validator.set_root_schema(rtbot_schema); + } catch (const std::exception& e) { + result.errors.push_back( + {"error", "", std::string("Schema setup error: ") + e.what(), DiagnosticCode::SCHEMA_VALIDATION, ""}); + return false; + } + + DiagnosticSchemaHandler handler; + validator.validate(j, handler); + + if (!handler.errors.empty()) { + result.errors.insert(result.errors.end(), handler.errors.begin(), handler.errors.end()); + return false; + } + return true; + } + + // ------------------------------------------------------------------------- + // Phase 4a: Operator Creation + // ------------------------------------------------------------------------- + static void phase_operator_creation(const json& j, std::map>& operators, + DiagnosticResult& result) { + if (!j.contains("operators") || !j["operators"].is_array()) return; + + const auto& ops = j["operators"]; + for (size_t i = 0; i < ops.size(); i++) { + const auto& op_json = ops[i]; + std::string base_path = "/operators/" + std::to_string(i); + std::string op_id = op_json.value("id", ""); + + try { + auto op = OperatorJson::read_op(op_json.dump()); + std::string qualified_id = op_id; + operators[qualified_id] = op; + + // For Pipelines, also register internal operators with qualified IDs + if (auto pipeline = std::dynamic_pointer_cast(op)) { + register_pipeline_operators(qualified_id, pipeline, operators); + } + } catch (const std::exception& e) { + std::string msg = e.what(); + DiagnosticCode code = classify_operator_error(msg); + std::string path = refine_operator_error_path(base_path, msg, op_json); + std::string suggestion = generate_operator_suggestion(code, msg); + + result.errors.push_back({"error", path, msg, code, suggestion}); + } + } + } + + static void register_pipeline_operators(const std::string& parent_prefix, + const std::shared_ptr& pipeline, + std::map>& operators) { + for (const auto& kv : pipeline->get_operators()) { + std::string qualified_id = parent_prefix + "::" + kv.first; + operators[qualified_id] = kv.second; + + if (auto nested_pipeline = std::dynamic_pointer_cast(kv.second)) { + register_pipeline_operators(qualified_id, nested_pipeline, operators); + } + } + } + + // ------------------------------------------------------------------------- + // Phase 4b: Connection Wiring + // ------------------------------------------------------------------------- + static void phase_connection_wiring(const json& j, std::map>& operators, + DiagnosticResult& result) { + if (!j.contains("connections") || !j["connections"].is_array()) return; + + std::vector known_ids = get_operator_ids(operators); + const auto& conns = j["connections"]; + + for (size_t i = 0; i < conns.size(); i++) { + const auto& conn = conns[i]; + std::string base_path = "/connections/" + std::to_string(i); + std::string from_id = conn.value("from", ""); + std::string to_id = conn.value("to", ""); + + auto from_op = resolve_operator(from_id, operators); + auto to_op = resolve_operator(to_id, operators); + + if (!from_op) { + result.errors.push_back({"error", base_path + "/from", + "Referenced operator \"" + from_id + "\" does not exist", + DiagnosticCode::INVALID_OPERATOR_REF, + "Available operators: " + join_vec(known_ids)}); + } + if (!to_op) { + result.errors.push_back({"error", base_path + "/to", + "Referenced operator \"" + to_id + "\" does not exist", + DiagnosticCode::INVALID_OPERATOR_REF, + "Available operators: " + join_vec(known_ids)}); + } + + if (!from_op || !to_op) continue; + + // Try connecting to detect port errors + try { + auto from_port = OperatorJson::parse_port_name(conn.value("fromPort", "o1")); + auto to_port = OperatorJson::parse_port_name(conn.value("toPort", "i1")); + PortKind to_kind = to_port.kind; + if (conn.contains("toPortType")) { + to_kind = conn["toPortType"] == "control" ? PortKind::CONTROL : PortKind::DATA; + } + from_op->connect(to_op, from_port.index, to_port.index, to_kind); + } catch (const std::exception& e) { + std::string msg = e.what(); + DiagnosticCode code = DiagnosticCode::PORT_TYPE_MISMATCH; + std::string path = base_path; + + if (msg.find("Invalid output port index") != std::string::npos) { + code = DiagnosticCode::PORT_INDEX_OUT_OF_BOUNDS; + path = base_path + "/fromPort"; + } else if (msg.find("Invalid child data port") != std::string::npos || + msg.find("Invalid child control port") != std::string::npos) { + code = DiagnosticCode::PORT_INDEX_OUT_OF_BOUNDS; + path = base_path + "/toPort"; + } else if (msg.find("type mismatch") != std::string::npos) { + code = DiagnosticCode::PORT_TYPE_MISMATCH; + } else if (msg.find("Invalid port name format") != std::string::npos) { + code = DiagnosticCode::PORT_INDEX_OUT_OF_BOUNDS; + } + + result.errors.push_back({"error", path, msg, code, ""}); + } + } + } + + // ------------------------------------------------------------------------- + // Phase 4c: Entry Operator Validation + // ------------------------------------------------------------------------- + static void phase_entry_operator(const json& j, + const std::map>& operators, + DiagnosticResult& result) { + if (!j.contains("entryOperator")) { + // Schema should catch this, but be defensive + return; + } + + std::string entry_id = j["entryOperator"].get(); + auto op = resolve_operator(entry_id, operators); + + if (!op) { + std::vector ids = get_operator_ids(operators); + result.errors.push_back({"error", "/entryOperator", "Entry operator not found: " + entry_id, + DiagnosticCode::INVALID_ENTRY_OPERATOR, + "Available operators: " + join_vec(ids)}); + } + } + + // ------------------------------------------------------------------------- + // Phase 4d: Output Mappings Validation + // ------------------------------------------------------------------------- + static void phase_output_mappings(const json& j, + const std::map>& operators, + DiagnosticResult& result) { + if (!j.contains("output")) { + result.errors.push_back( + {"error", "", "Program JSON must contain 'output' field", DiagnosticCode::MISSING_OUTPUT_FIELD, ""}); + return; + } + + const auto& output = j["output"]; + for (auto it = output.begin(); it != output.end(); ++it) { + std::string op_id = it.key(); + std::string path = "/output/" + op_id; + + auto op = resolve_operator(op_id, operators); + if (!op) { + std::vector ids = get_operator_ids(operators); + result.errors.push_back({"error", path, "Output operator not found: " + op_id, + DiagnosticCode::INVALID_OUTPUT_MAPPING, + "Available operators: " + join_vec(ids)}); + continue; + } + + // Validate port references + if (it.value().is_array()) { + for (size_t pi = 0; pi < it.value().size(); pi++) { + std::string port_str = it.value()[pi].get(); + try { + OperatorJson::parse_port_name(port_str); + } catch (const std::exception& e) { + result.errors.push_back({"error", path + "/" + std::to_string(pi), + "Invalid port reference '" + port_str + "': " + e.what(), + DiagnosticCode::INVALID_OUTPUT_MAPPING, "Use format like 'o1', 'o2', etc."}); + } + } + } + } + } +}; + +} // namespace rtbot + +#endif // RTBOT_DIAGNOSTICS_H diff --git a/libs/api/include/rtbot/bindings.h b/libs/api/include/rtbot/bindings.h index 3bd1d38e..20869cd3 100644 --- a/libs/api/include/rtbot/bindings.h +++ b/libs/api/include/rtbot/bindings.h @@ -32,6 +32,7 @@ std::string get_program_entry_operator_id(const std::string& program_id); // Validation functions std::string validate_program(const std::string& json_program); std::string validate_operator(const std::string& type, const std::string& json_op); +std::string diagnose_program(const std::string& json_program); // Message handling std::string add_to_message_buffer(const std::string& program_id, const std::string& port_id, uint64_t time, diff --git a/libs/api/src/bindings.cpp b/libs/api/src/bindings.cpp index 7451fb65..14434d64 100644 --- a/libs/api/src/bindings.cpp +++ b/libs/api/src/bindings.cpp @@ -10,6 +10,7 @@ #include #include +#include "rtbot/Diagnostics.h" #include "rtbot/Message.h" #include "rtbot/Program.h" #include "rtbot/jsonschema.hpp" @@ -353,6 +354,11 @@ std::string pretty_print_validation_error(const std::string& validation_result) } } +std::string diagnose_program(const std::string& json_program) { + auto result = ProgramDiagnostics::diagnose(json_program); + return json(result).dump(); +} + } // namespace rtbot #endif // RTBOT_BINDINGS_H \ No newline at end of file diff --git a/libs/api/test/test_diagnostics.cpp b/libs/api/test/test_diagnostics.cpp new file mode 100644 index 00000000..9f668ab9 --- /dev/null +++ b/libs/api/test/test_diagnostics.cpp @@ -0,0 +1,465 @@ +#include + +#include "rtbot/Diagnostics.h" + +using namespace rtbot; + +// --------------------------------------------------------------------------- +// Phase 1: JSON parse errors +// --------------------------------------------------------------------------- +SCENARIO("diagnoseProgram detects JSON parse errors", "[diagnostics]") { + GIVEN("Invalid JSON input") { + auto result = ProgramDiagnostics::diagnose("{invalid json}"); + THEN("Returns INVALID_JSON error") { + REQUIRE_FALSE(result.valid); + REQUIRE(result.errors.size() == 1); + REQUIRE(result.errors[0].code == DiagnosticCode::INVALID_JSON); + REQUIRE(result.errors[0].severity == "error"); + } + } + + GIVEN("Empty string input") { + auto result = ProgramDiagnostics::diagnose(""); + THEN("Returns INVALID_JSON error") { + REQUIRE_FALSE(result.valid); + REQUIRE(result.errors.size() == 1); + REQUIRE(result.errors[0].code == DiagnosticCode::INVALID_JSON); + } + } +} + +// --------------------------------------------------------------------------- +// Phase 2: Prototype resolution errors +// --------------------------------------------------------------------------- +SCENARIO("diagnoseProgram detects prototype errors", "[diagnostics]") { + GIVEN("A program referencing an unknown prototype") { + std::string json_str = R"({ + "prototypes": { + "myProto": { + "parameters": [{"name": "window", "type": "number"}], + "operators": [{"type": "MovingAverage", "id": "ma", "window_size": 5}], + "connections": [], + "entry": {"operator": "ma"}, + "output": {"operator": "ma"} + } + }, + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"id": "inst", "prototype": "nonExistent", "parameters": {}}, + {"type": "Output", "id": "output1", "portTypes": ["number"]} + ], + "connections": [ + {"from": "input1", "to": "inst"}, + {"from": "inst", "to": "output1"} + ], + "entryOperator": "input1", + "output": {"output1": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + THEN("Returns UNKNOWN_PROTOTYPE error") { + REQUIRE_FALSE(result.valid); + bool found = false; + for (const auto& e : result.errors) { + if (e.code == DiagnosticCode::UNKNOWN_PROTOTYPE) { + REQUIRE(e.path == "/operators/1/prototype"); + REQUIRE_FALSE(e.suggestion.empty()); + found = true; + } + } + REQUIRE(found); + } + } + + GIVEN("A program with unknown and missing prototype parameters") { + std::string json_str = R"({ + "prototypes": { + "myProto": { + "parameters": [ + {"name": "window", "type": "number"}, + {"name": "label", "type": "string"} + ], + "operators": [{"type": "MovingAverage", "id": "ma", "window_size": 5}], + "connections": [], + "entry": {"operator": "ma"}, + "output": {"operator": "ma"} + } + }, + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"id": "inst", "prototype": "myProto", "parameters": {"wrong_param": 5}}, + {"type": "Output", "id": "output1", "portTypes": ["number"]} + ], + "connections": [ + {"from": "input1", "to": "inst"}, + {"from": "inst", "to": "output1"} + ], + "entryOperator": "input1", + "output": {"output1": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + THEN("Returns UNKNOWN_PARAMETER and MISSING_PARAMETER errors") { + REQUIRE_FALSE(result.valid); + bool found_unknown = false; + bool found_missing_window = false; + bool found_missing_label = false; + for (const auto& e : result.errors) { + if (e.code == DiagnosticCode::UNKNOWN_PARAMETER && e.message.find("wrong_param") != std::string::npos) + found_unknown = true; + if (e.code == DiagnosticCode::MISSING_PARAMETER && e.message.find("window") != std::string::npos) + found_missing_window = true; + if (e.code == DiagnosticCode::MISSING_PARAMETER && e.message.find("label") != std::string::npos) + found_missing_label = true; + } + REQUIRE(found_unknown); + REQUIRE(found_missing_window); + REQUIRE(found_missing_label); + } + } + + GIVEN("A program with wrong prototype parameter type") { + std::string json_str = R"({ + "prototypes": { + "myProto": { + "parameters": [{"name": "window", "type": "number"}], + "operators": [{"type": "MovingAverage", "id": "ma", "window_size": 5}], + "connections": [], + "entry": {"operator": "ma"}, + "output": {"operator": "ma"} + } + }, + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"id": "inst", "prototype": "myProto", "parameters": {"window": "not_a_number"}}, + {"type": "Output", "id": "output1", "portTypes": ["number"]} + ], + "connections": [ + {"from": "input1", "to": "inst"}, + {"from": "inst", "to": "output1"} + ], + "entryOperator": "input1", + "output": {"output1": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + THEN("Returns INVALID_PARAMETER_TYPE error") { + REQUIRE_FALSE(result.valid); + bool found = false; + for (const auto& e : result.errors) { + if (e.code == DiagnosticCode::INVALID_PARAMETER_TYPE) { + REQUIRE(e.path == "/operators/1/parameters/window"); + found = true; + } + } + REQUIRE(found); + } + } +} + +// --------------------------------------------------------------------------- +// Phase 3: Schema validation errors +// --------------------------------------------------------------------------- +SCENARIO("diagnoseProgram detects schema validation errors", "[diagnostics]") { + GIVEN("A program missing entryOperator") { + std::string json_str = R"({ + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"type": "Output", "id": "output1", "portTypes": ["number"]} + ], + "connections": [{"from": "input1", "to": "output1"}], + "output": {"output1": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + THEN("Returns SCHEMA_VALIDATION error") { + REQUIRE_FALSE(result.valid); + bool found = false; + for (const auto& e : result.errors) { + if (e.code == DiagnosticCode::SCHEMA_VALIDATION) { + found = true; + } + } + REQUIRE(found); + } + } +} + +// --------------------------------------------------------------------------- +// Phase 3/4: Operator-level errors (schema or semantic depending on what the schema covers) +// --------------------------------------------------------------------------- +SCENARIO("diagnoseProgram detects operator creation errors", "[diagnostics]") { + GIVEN("A program with an unknown operator type") { + std::string json_str = R"({ + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"type": "NonExistentOperator", "id": "bad1"}, + {"type": "Output", "id": "output1", "portTypes": ["number"]} + ], + "connections": [ + {"from": "input1", "to": "bad1"}, + {"from": "bad1", "to": "output1"} + ], + "entryOperator": "input1", + "output": {"output1": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + THEN("Returns errors (schema catches unknown types via oneOf)") { + REQUIRE_FALSE(result.valid); + REQUIRE(result.errors.size() > 0); + // The schema uses oneOf for known operator types, so unknown types + // are caught at Phase 3 as SCHEMA_VALIDATION errors + bool found = false; + for (const auto& e : result.errors) { + if (e.code == DiagnosticCode::SCHEMA_VALIDATION || e.code == DiagnosticCode::UNKNOWN_OPERATOR_TYPE) { + found = true; + } + } + REQUIRE(found); + } + } + + GIVEN("A program with invalid operator parameter (PeakDetector even window)") { + std::string json_str = R"({ + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"type": "PeakDetector", "id": "pd1", "window_size": 4}, + {"type": "Output", "id": "output1", "portTypes": ["number"]} + ], + "connections": [ + {"from": "input1", "to": "pd1"}, + {"from": "pd1", "to": "output1"} + ], + "entryOperator": "input1", + "output": {"output1": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + THEN("Returns an error for the invalid parameter") { + REQUIRE_FALSE(result.valid); + REQUIRE(result.errors.size() > 0); + // May be caught at schema level (if schema has minimum constraint) + // or at semantic level as INVALID_PARAMETER_VALUE + bool found = false; + for (const auto& e : result.errors) { + if (e.code == DiagnosticCode::INVALID_PARAMETER_VALUE || e.code == DiagnosticCode::SCHEMA_VALIDATION) { + found = true; + } + } + REQUIRE(found); + } + } + + GIVEN("A program with multiple operator errors") { + std::string json_str = R"({ + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"type": "PeakDetector", "id": "pd1", "window_size": 4}, + {"type": "FiniteImpulseResponse", "id": "fir1", "coeff": []}, + {"type": "Output", "id": "output1", "portTypes": ["number"]} + ], + "connections": [ + {"from": "input1", "to": "pd1"}, + {"from": "pd1", "to": "fir1"}, + {"from": "fir1", "to": "output1"} + ], + "entryOperator": "input1", + "output": {"output1": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + THEN("Returns errors for the bad operators") { + REQUIRE_FALSE(result.valid); + // Errors may be at schema or semantic level; either way there should be multiple + REQUIRE(result.errors.size() >= 1); + } + } +} + +// --------------------------------------------------------------------------- +// Phase 4b: Connection wiring errors +// --------------------------------------------------------------------------- +SCENARIO("diagnoseProgram detects connection errors", "[diagnostics]") { + GIVEN("A program with invalid connection reference") { + std::string json_str = R"({ + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"type": "Output", "id": "output1", "portTypes": ["number"]} + ], + "connections": [ + {"from": "input1", "to": "nonexistent"} + ], + "entryOperator": "input1", + "output": {"output1": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + THEN("Returns INVALID_OPERATOR_REF with path and suggestion") { + REQUIRE_FALSE(result.valid); + bool found = false; + for (const auto& e : result.errors) { + if (e.code == DiagnosticCode::INVALID_OPERATOR_REF) { + REQUIRE(e.path == "/connections/0/to"); + REQUIRE_FALSE(e.suggestion.empty()); + found = true; + } + } + REQUIRE(found); + } + } + + GIVEN("A program with port type mismatch") { + std::string json_str = R"({ + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"type": "Output", "id": "output1", "portTypes": ["boolean"]} + ], + "connections": [ + {"from": "input1", "to": "output1", "fromPort": "o1", "toPort": "i1"} + ], + "entryOperator": "input1", + "output": {"output1": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + THEN("Returns PORT_TYPE_MISMATCH error") { + REQUIRE_FALSE(result.valid); + bool found = false; + for (const auto& e : result.errors) { + if (e.code == DiagnosticCode::PORT_TYPE_MISMATCH) { + found = true; + } + } + REQUIRE(found); + } + } +} + +// --------------------------------------------------------------------------- +// Phase 4c: Entry operator errors +// --------------------------------------------------------------------------- +SCENARIO("diagnoseProgram detects entry operator errors", "[diagnostics]") { + GIVEN("A program with a nonexistent entry operator") { + std::string json_str = R"({ + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"type": "Output", "id": "output1", "portTypes": ["number"]} + ], + "connections": [ + {"from": "input1", "to": "output1"} + ], + "entryOperator": "does_not_exist", + "output": {"output1": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + THEN("Returns INVALID_ENTRY_OPERATOR error") { + REQUIRE_FALSE(result.valid); + bool found = false; + for (const auto& e : result.errors) { + if (e.code == DiagnosticCode::INVALID_ENTRY_OPERATOR) { + REQUIRE(e.path == "/entryOperator"); + REQUIRE_FALSE(e.suggestion.empty()); + found = true; + } + } + REQUIRE(found); + } + } +} + +// --------------------------------------------------------------------------- +// Phase 4d: Output mapping errors +// --------------------------------------------------------------------------- +SCENARIO("diagnoseProgram detects output mapping errors", "[diagnostics]") { + GIVEN("A program with a nonexistent output operator") { + std::string json_str = R"({ + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"type": "Output", "id": "output1", "portTypes": ["number"]} + ], + "connections": [ + {"from": "input1", "to": "output1"} + ], + "entryOperator": "input1", + "output": {"nonexistent_op": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + THEN("Returns INVALID_OUTPUT_MAPPING error") { + REQUIRE_FALSE(result.valid); + bool found = false; + for (const auto& e : result.errors) { + if (e.code == DiagnosticCode::INVALID_OUTPUT_MAPPING) { + REQUIRE(e.path.find("/output/") != std::string::npos); + found = true; + } + } + REQUIRE(found); + } + } +} + +// --------------------------------------------------------------------------- +// Valid program +// --------------------------------------------------------------------------- +SCENARIO("diagnoseProgram returns valid for correct programs", "[diagnostics]") { + GIVEN("A valid simple program") { + std::string json_str = R"({ + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"type": "MovingAverage", "id": "ma1", "window_size": 3}, + {"type": "Output", "id": "output1", "portTypes": ["number"]} + ], + "connections": [ + {"from": "input1", "to": "ma1", "fromPort": "o1", "toPort": "i1"}, + {"from": "ma1", "to": "output1", "fromPort": "o1", "toPort": "i1"} + ], + "entryOperator": "input1", + "output": {"output1": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + THEN("Returns valid with no errors") { + REQUIRE(result.valid); + REQUIRE(result.errors.empty()); + } + } +} + +// --------------------------------------------------------------------------- +// JSON output format +// --------------------------------------------------------------------------- +SCENARIO("diagnoseProgram returns correct JSON format", "[diagnostics]") { + GIVEN("An invalid program") { + std::string json_str = R"({ + "operators": [ + {"type": "Input", "id": "input1", "portTypes": ["number"]}, + {"type": "Output", "id": "output1", "portTypes": ["number"]} + ], + "connections": [ + {"from": "input1", "to": "nonexistent"} + ], + "entryOperator": "input1", + "output": {"output1": ["o1"]} + })"; + + auto result = ProgramDiagnostics::diagnose(json_str); + auto j = json(result); + + THEN("JSON contains valid, errors array with severity, path, message, code") { + REQUIRE(j.contains("valid")); + REQUIRE(j["valid"].is_boolean()); + REQUIRE(j.contains("errors")); + REQUIRE(j["errors"].is_array()); + REQUIRE(j["errors"].size() > 0); + + auto& first_error = j["errors"][0]; + REQUIRE(first_error.contains("severity")); + REQUIRE(first_error.contains("path")); + REQUIRE(first_error.contains("message")); + REQUIRE(first_error.contains("code")); + } + } +} diff --git a/libs/api/wasm/emscripten-bindings.cpp b/libs/api/wasm/emscripten-bindings.cpp index 67038527..a94e8849 100644 --- a/libs/api/wasm/emscripten-bindings.cpp +++ b/libs/api/wasm/emscripten-bindings.cpp @@ -83,6 +83,7 @@ EMSCRIPTEN_BINDINGS(RtBot) { function("deleteProgram", &rtbot::delete_program); function("validateProgram", &rtbot::validate_program); function("validateOperator", &rtbot::validate_operator); + function("diagnoseProgram", &rtbot::diagnose_program); // Message handling function("addToMessageBuffer", &addMessage); diff --git a/tools/BUILD.bazel b/tools/BUILD.bazel new file mode 100644 index 00000000..6a259607 --- /dev/null +++ b/tools/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_cc//cc:defs.bzl", "cc_binary") + +cc_binary( + name = "diagnose_file", + srcs = ["diagnose_file.cpp"], + deps = [ + "//libs/api:rtbot-api", + ], +) diff --git a/tools/diagnose_file.cpp b/tools/diagnose_file.cpp new file mode 100644 index 00000000..7b8a86bb --- /dev/null +++ b/tools/diagnose_file.cpp @@ -0,0 +1,31 @@ +#include +#include +#include +#include + +#include "rtbot/bindings.h" + +int main(int argc, char* argv[]) { + if (argc < 2) { + std::cerr << "Usage: diagnose_file " << std::endl; + return 1; + } + + std::ifstream file(argv[1]); + if (!file.is_open()) { + std::cerr << "Could not open file: " << argv[1] << std::endl; + return 1; + } + + std::stringstream buffer; + buffer << file.rdbuf(); + std::string json_str = buffer.str(); + + std::string result = rtbot::diagnose_program(json_str); + + // Pretty print the JSON result + auto j = nlohmann::json::parse(result); + std::cout << j.dump(2) << std::endl; + + return 0; +}