diff --git a/TARS/.gitignore b/TARS/.gitignore index 89cc49cb..3167aaa3 100644 --- a/TARS/.gitignore +++ b/TARS/.gitignore @@ -3,3 +3,4 @@ .vscode/c_cpp_properties.json .vscode/launch.json .vscode/ipch +sdlog.dat \ No newline at end of file diff --git a/TARS/data_parser.py b/TARS/data_parser.py index 8555a216..e57b3633 100644 --- a/TARS/data_parser.py +++ b/TARS/data_parser.py @@ -9,13 +9,18 @@ - There are numerous reasons this might not work out of the box. Some reasons I can think of right now include not having python on your path, having the wrong python on your path, not being in the right working directory, or not having installed the right lark. + - Outputting to a `.csv` file will automatically trigger a conversion from JSON to CSV at the end of the process. + +Authors: + Magilan Sendhil + Michael Karpov """ from __future__ import annotations import json from abc import ABC, abstractmethod -from typing import Callable +from typing import Callable, Any, NamedTuple, cast import lark from lark import Lark @@ -599,7 +604,13 @@ def render_bytes(n: int) -> str: return f"{n:#.4g}{endings[end_idx]}" -def get_arguments(): +class Arguments(NamedTuple): + header: pathlib.Path + raw: pathlib.Path + out: pathlib.Path + + +def get_arguments() -> Arguments: """ A helper function which parses the arguments passed to the file. @@ -617,10 +628,13 @@ def get_arguments(): if args.out is None: raw_path: pathlib.Path = args.raw args.out = raw_path.with_suffix(".json") - return args + return cast(Arguments, args) def preprocess(t: str) -> str: + """ + A very basic and very wrong C++ preprocessor, it just deletes any line which has a preprocessor directive + """ l = [] for line in t.splitlines(): if not line.startswith("#"): @@ -628,6 +642,46 @@ def preprocess(t: str) -> str: return "\n".join(l) +def construct_header_string(jsonobject: dict[str, Any], path: str = "") -> str: + """ + Given a json object that is representative of the whole file, recursively create a header string containing every + key in the json as a period-separated path. + + :param jsonobject: A JSON object to construct a header string for + :param path: The path this JSON object is at + :return: A string for the keys in this json object + """ + items: list[str] = [] + for key, value in jsonobject.items(): + if isinstance(value, dict): + # If it's a dictionary, continue recursively but including the trace to the datatype + items.append(construct_header_string(value, f"{path}.{key}")) + else: + items.append(f"{path}.{key}") + return ",".join(items) + + +def construct_data_string(jsonobject: dict[str, Any]): + """ + Construct a string turning a single JSON object into csv columns + + :param jsonobject: A JSON object to convert + :return: a string for the values in this json object as csv rows + """ + items: list[str] = [] + for key, value in jsonobject.items(): + if isinstance(value, dict): + items.append(construct_data_string(value)) + elif isinstance(value, list): + # If it's a list, take only the first value + # (Lists are only used for FSM, anyway, someone could alter this behavior though if they wish) + # todo make this less fragile, but it should be good for now + items.append(str(value[0])) + else: + items.append(value) + return ",".join(items) + + def main(): args = get_arguments() @@ -651,6 +705,7 @@ def main(): all_data: bytes = f_da.read() idx = 0 # a counter which keeps track of how far into the raw data we are + # open up the output file, which we write to incrementally with args.out.open("w") as f_out: f_out.write("[") @@ -676,6 +731,19 @@ def main(): print(f"\rParsed {render_bytes(idx)}/{render_bytes(len(all_data))} bytes", end="", flush=True) f_out.write("\n]") + # If file_type is .csv, we will convert it from JSON (It is easier to convert from JSON than from a binary format.) + if args.out.suffix == ".csv": + print("\nConverting .json output to .csv") + with args.out.open("r") as f_in: + data_json = json.load(f_in) + + header_string = construct_header_string(data_json[0]) + with args.out.open("w") as f_out: + # Now we can overwrite the data, since we have the json loaded. + f_out.write(header_string + "\n") + for data_object in data_json: + f_out.write(construct_data_string(data_object) + "\n") + if __name__ == '__main__': - main() \ No newline at end of file + main()