From 0ca727c253329958a4a8169895692cb3ba335e90 Mon Sep 17 00:00:00 2001 From: Eyal-Danieli Date: Thu, 1 Jan 2026 11:02:24 +0200 Subject: [PATCH] update function yamls --- cli/cli.py | 8 +- cli/common/generate_item_yaml.py | 19 +- cli/common/test_suite.py | 142 ++--- cli/common/update_readme.py | 27 +- cli/functions/function_to_item.py | 4 +- cli/functions/item_to_function.py | 59 +- cli/marketplace/build.py | 71 ++- cli/utils/helpers.py | 52 +- cli/utils/path_iterator.py | 6 +- functions/src/aggregate/aggregate.py | 142 +++-- functions/src/aggregate/function.yaml | 48 +- functions/src/aggregate/test_aggregate.py | 57 +- .../src/arc_to_parquet/arc_to_parquet.py | 76 +-- functions/src/arc_to_parquet/function.yaml | 35 +- .../src/arc_to_parquet/test_arc_to_parquet.py | 46 +- functions/src/auto_trainer/auto_trainer.py | 29 +- functions/src/auto_trainer/function.yaml | 48 +- .../src/auto_trainer/test_auto_trainer.py | 19 +- functions/src/azureml_serving/function.yaml | 60 +- functions/src/azureml_utils/azureml_utils.py | 50 +- functions/src/azureml_utils/function.yaml | 127 ++-- .../src/azureml_utils/test_azureml_utils.py | 10 +- .../src/batch_inference/batch_inference.py | 39 +- functions/src/batch_inference/function.yaml | 46 +- .../batch_inference/test_batch_inference.py | 3 +- .../batch_inference_v2/batch_inference_v2.py | 171 +++--- .../src/batch_inference_v2/function.yaml | 56 +- functions/src/batch_inference_v2/item.yaml | 2 +- .../test_batch_inference_v2.py | 128 ++-- functions/src/describe/describe.py | 17 +- functions/src/describe/function.yaml | 34 +- functions/src/describe/test_describe.py | 6 +- functions/src/describe_dask/describe_dask.py | 22 +- functions/src/describe_dask/function.yaml | 35 +- .../src/describe_dask/test_describe_dask.py | 40 +- .../src/describe_spark/describe_spark.py | 551 +++++++++++------- functions/src/describe_spark/function.yaml | 320 +++++----- .../feature_selection/feature_selection.py | 7 +- functions/src/feature_selection/function.yaml | 43 +- functions/src/feature_selection/item.yaml | 2 +- functions/src/gen_class_data/function.yaml | 33 +- .../src/gen_class_data/gen_class_data.py | 30 +- .../src/gen_class_data/test_gen_class_data.py | 14 +- functions/src/github_utils/function.yaml | 54 +- functions/src/github_utils/github_utils.py | 12 +- .../src/hugging_face_serving/function.yaml | 27 +- .../hugging_face_serving.py | 6 +- .../test_hugging_face_serving.py | 6 +- functions/src/load_dataset/function.yaml | 64 +- functions/src/mlflow_utils/function.yaml | 37 +- functions/src/mlflow_utils/mlflow_utils.py | 7 +- .../src/mlflow_utils/test_mlflow_utils.py | 17 +- functions/src/model_server/function.yaml | 33 +- functions/src/model_server/model_server.py | 11 +- .../src/model_server/test_model_server.py | 38 +- .../src/model_server_tester/function.yaml | 40 +- .../model_server_tester.py | 10 +- functions/src/noise_reduction/function.yaml | 135 ++--- .../src/noise_reduction/noise_reduction.py | 28 +- functions/src/onnx_utils/function.yaml | 69 +-- functions/src/onnx_utils/onnx_utils.py | 16 +- functions/src/open_archive/function.yaml | 31 +- functions/src/open_archive/item.yaml | 2 +- functions/src/open_archive/open_archive.py | 100 ++-- .../src/open_archive/test_open_archive.py | 54 +- functions/src/pii_recognizer/function.yaml | 73 ++- .../src/pii_recognizer/pii_recognizer.py | 48 +- .../src/pii_recognizer/test_pii_recognizer.py | 10 +- functions/src/pyannote_audio/function.yaml | 64 +- .../src/pyannote_audio/pyannote_audio.py | 18 +- .../src/question_answering/function.yaml | 142 +++-- .../question_answering/question_answering.py | 69 +-- .../test_question_answering.py | 16 +- functions/src/send_email/function.yaml | 49 +- functions/src/send_email/send_email.py | 10 +- functions/src/silero_vad/function.yaml | 147 +++-- functions/src/silero_vad/silero_vad.py | 49 +- .../src/sklearn_classifier/function.yaml | 43 +- .../sklearn_classifier/sklearn_classifier.py | 11 +- .../test_sklearn_classifier.py | 61 +- .../src/sklearn_classifier_dask/function.yaml | 45 +- .../sklearn_classifier_dask.py | 19 +- .../structured_data_generator/function.yaml | 41 +- .../structured_data_generator.py | 6 +- .../test_structured_data_generator.py | 11 +- functions/src/test_classifier/function.yaml | 60 +- .../src/test_classifier/test_classifier.py | 9 +- .../src/text_to_audio_generator/function.yaml | 58 +- .../test_text_to_audio_generator.py | 2 +- .../text_to_audio_generator.py | 43 +- functions/src/tf2_serving/function.yaml | 66 +-- functions/src/tf2_serving/tf2_serving.py | 19 +- functions/src/transcribe/test_transcribe.py | 8 +- functions/src/transcribe/transcribe.py | 170 +++--- functions/src/translate/function.yaml | 67 ++- functions/src/translate/item.yaml | 2 +- functions/src/translate/test_translate.py | 5 +- functions/src/translate/translate.py | 16 +- functions/src/v2_model_server/function.yaml | 96 +-- .../src/v2_model_server/v2_model_server.py | 11 +- functions/src/v2_model_tester/function.yaml | 40 +- .../src/v2_model_tester/v2_model_tester.py | 9 +- modules/src/agent_deployer/agent_deployer.py | 17 +- .../src/agent_deployer/test_agent_deployer.py | 26 +- modules/src/count_events/count_events.py | 13 +- modules/src/count_events/item.yaml | 2 +- modules/src/count_events/test_count_events.py | 15 +- modules/src/evidently_iris/evidently_iris.py | 25 +- modules/src/evidently_iris/item.yaml | 2 +- .../src/evidently_iris/test_evidently_iris.py | 4 +- .../histogram_data_drift.py | 23 +- modules/src/histogram_data_drift/item.yaml | 2 +- .../test_histogram_data_drift.py | 47 +- .../src/openai_proxy_app/openai_proxy_app.py | 31 +- .../openai_proxy_app/test_openai_proxy_app.py | 10 +- modules/src/vllm_module/test_vllm_module.py | 7 +- modules/src/vllm_module/vllm_module.py | 63 +- pyproject.toml | 8 +- steps/src/verify_schema/test_verify_schema.py | 23 +- steps/src/verify_schema/verify_schema.py | 9 +- 120 files changed, 2755 insertions(+), 2716 deletions(-) diff --git a/cli/cli.py b/cli/cli.py index e8e6922fe..8d31ad38f 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -14,17 +14,19 @@ # import click +from cli.common.generate_item_yaml import generate_item_yaml +from cli.common.test_suite import test_suite +from cli.common.update_readme import update_readme from cli.functions.function_to_item import function_to_item_cli from cli.functions.item_to_function import item_to_function_cli from cli.marketplace.build import build_marketplace_cli -from cli.common.test_suite import test_suite -from cli.common.update_readme import update_readme -from cli.common.generate_item_yaml import generate_item_yaml + @click.group() def cli(): pass + cli.add_command(generate_item_yaml, name="generate-item-yaml") cli.add_command(item_to_function_cli, name="item-to-function") cli.add_command(function_to_item_cli, name="function-to-item") diff --git a/cli/common/generate_item_yaml.py b/cli/common/generate_item_yaml.py index e97089ad3..093d19fac 100644 --- a/cli/common/generate_item_yaml.py +++ b/cli/common/generate_item_yaml.py @@ -1,6 +1,7 @@ import sys -from pathlib import Path from datetime import datetime, timezone +from pathlib import Path + import click from jinja2 import Environment, FileSystemLoader @@ -14,14 +15,18 @@ @click.command() @click.argument("type", type=click.Choice(list(TEMPLATES.keys()))) @click.argument("name") -@click.option("--overwrite", is_flag=True, help="Replace existing file instead of raising an error.") +@click.option( + "--overwrite", + is_flag=True, + help="Replace existing file instead of raising an error.", +) def generate_item_yaml(type: str, name: str, overwrite: bool = False): """ - Generate an item.yaml file from a template. + Generate an item.yaml file from a template. -type: one of the supported types (currently only `function` or `module`) -name: the function/module name (also used as the directory name) -overwrite: whether to overwrite existing item.yaml file + type: one of the supported types (currently only `function` or `module`) + name: the function/module name (also used as the directory name) + overwrite: whether to overwrite existing item.yaml file """ # Construct the target path path = Path(f"{type}s/src/{name}").resolve() @@ -53,4 +58,4 @@ def generate_item_yaml(type: str, name: str, overwrite: bool = False): if __name__ == "__main__": - generate_item_yaml() \ No newline at end of file + generate_item_yaml() diff --git a/cli/common/test_suite.py b/cli/common/test_suite.py index 52dc1c5ae..9e1e7b983 100644 --- a/cli/common/test_suite.py +++ b/cli/common/test_suite.py @@ -12,23 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import re import subprocess +import sys from abc import ABC, abstractmethod from dataclasses import dataclass, field from pathlib import Path from subprocess import CompletedProcess -from typing import List, Union, Optional -import sys + import click import yaml -import re from cli.utils.helpers import ( - is_item_dir, + get_item_yaml_values, install_pipenv, install_python, install_requirements, - get_item_yaml_values, + is_item_dir, ) from cli.utils.path_iterator import PathIterator @@ -45,11 +45,13 @@ default=False, help="When true, test suite will stop running after the first test ran", ) -def test_suite(root_directory: str, - suite: str, - stop_on_failure: bool, - multi_processing: bool = False, - function_name: str = None): +def test_suite( + root_directory: str, + suite: str, + stop_on_failure: bool, + multi_processing: bool = False, + function_name: str = None, +): if not suite: click.echo("-s/--suite is required") exit(1) @@ -101,25 +103,19 @@ def test_example(root_dir="."): @dataclass class TestResult: status: str - status_code: Optional[int] + status_code: int | None meta_data: dict = field(default_factory=dict) @classmethod - def passed( - cls, status_code: Optional[int] = None, meta_data: Optional[dict] = None - ): + def passed(cls, status_code: int | None = None, meta_data: dict | None = None): return cls(status="Passed", status_code=status_code, meta_data=meta_data) @classmethod - def failed( - cls, status_code: Optional[int] = None, meta_data: Optional[dict] = None - ): + def failed(cls, status_code: int | None = None, meta_data: dict | None = None): return cls(status="Failed", status_code=status_code, meta_data=meta_data) @classmethod - def ignored( - cls, status_code: Optional[int] = None, meta_data: Optional[dict] = None - ): + def ignored(cls, status_code: int | None = None, meta_data: dict | None = None): return cls(status="Ignored", status_code=status_code, meta_data=meta_data) @@ -129,11 +125,11 @@ def __init__(self, stop_on_failure: bool = True): self.test_results = [] @abstractmethod - def discover(self, path: Union[str, Path]) -> List[str]: + def discover(self, path: str | Path) -> list[str]: pass @abstractmethod - def run(self, path: Union[str, Path]) -> TestResult: + def run(self, path: str | Path) -> TestResult: pass @abstractmethod @@ -145,27 +141,32 @@ def after_run(self): pass @abstractmethod - def before_each(self, path: Union[str, Path]): + def before_each(self, path: str | Path): pass @abstractmethod - def after_each(self, path: Union[str, Path], test_result: TestResult): + def after_each(self, path: str | Path, test_result: TestResult): pass - def _run(self, path: Union[str, Path], multiprocess, function_name): + def _run(self, path: str | Path, multiprocess, function_name): import multiprocessing as mp + process_count = 1 if multiprocess: process_count = mp.cpu_count() - 1 - print("running tests with {} process".format(process_count)) + print(f"running tests with {process_count} process") discovered_functions = self.discover(path) if function_name is not None: - click.echo("running test with name {}".format(function_name)) - discovered_functions = [fn for fn in discovered_functions if Path(function_name).stem == Path(fn).stem] + click.echo(f"running test with name {function_name}") + discovered_functions = [ + fn + for fn in discovered_functions + if Path(function_name).stem == Path(fn).stem + ] for path in discovered_functions: if re.match(".+/test_*", path): discovered_functions.remove(path) - print("a function name cannot start with test, please rename {} ".format(path)) + print(f"a function name cannot start with test, please rename {path} ") self.before_run() @@ -191,7 +192,7 @@ def __init__(self, stop_on_failure: bool = True, clean_env_artifacts: bool = Tru self.clean_env_artifacts = clean_env_artifacts self.results = [] - def discover(self, path: Union[str, Path]) -> List[str]: + def discover(self, path: str | Path) -> list[str]: path = Path(path) testable = [] item_yaml_path = path / "item.yaml" @@ -228,15 +229,21 @@ def discover(self, path: Union[str, Path]) -> List[str]: def before_run(self): install_pipenv() - def before_each(self, path: Union[str, Path]): + def before_each(self, path: str | Path): pass - def run(self, path: Union[str, Path]): - print("PY run path {}".format(path)) + def run(self, path: str | Path): + print(f"PY run path {path}") install_python(path) - item_requirements = list(get_item_yaml_values(path, 'requirements')['requirements']) - mlrun_version = list(get_item_yaml_values(path, "mlrunVersion")["mlrunVersion"])[0] - install_requirements(path, ["pytest", f"mlrun=={mlrun_version}"] + item_requirements) + item_requirements = list( + get_item_yaml_values(path, "requirements")["requirements"] + ) + mlrun_version = list( + get_item_yaml_values(path, "mlrunVersion")["mlrunVersion"] + )[0] + install_requirements( + path, ["pytest", f"mlrun=={mlrun_version}"] + item_requirements + ) click.echo(f"Running tests for {path}...") completed_process: CompletedProcess = subprocess.run( f"cd {path} ; pipenv run python -m pytest", @@ -256,7 +263,7 @@ def run(self, path: Union[str, Path]): meta_data=meta_data, ) - def after_each(self, path: Union[str, Path], test_result: TestResult): + def after_each(self, path: str | Path, test_result: TestResult): if self.clean_env_artifacts: clean_pipenv(path) @@ -314,11 +321,11 @@ def after_run(self): sys.exit(1) @staticmethod - def is_test_py(path: Union[str, Path]) -> bool: + def is_test_py(path: str | Path) -> bool: return ( - path.is_file() - and path.name.startswith("test_") - and path.name.endswith(".py") + path.is_file() + and path.name.startswith("test_") + and path.name.endswith(".py") ) @@ -328,7 +335,7 @@ def __init__(self, stop_on_failure: bool = True, clean_env_artifacts: bool = Tru self.clean_env_artifacts = clean_env_artifacts self.results = [] - def discover(self, path: Union[str, Path]) -> List[str]: + def discover(self, path: str | Path) -> list[str]: path = Path(path) testables = [] @@ -357,34 +364,34 @@ def discover(self, path: Union[str, Path]) -> List[str]: ) exit(0) testables.sort() - click.echo( - "tests list " + str(testables) - ) + click.echo("tests list " + str(testables)) return testables def before_run(self): install_pipenv() - def before_each(self, path: Union[str, Path]): + def before_each(self, path: str | Path): pass # def run(self, path: Union[str, Path]) -> TestResult: - def run(self, path: Union[str, Path]) -> TestResult: - print("IPYNB run path {}".format(path)) + def run(self, path: str | Path) -> TestResult: + print(f"IPYNB run path {path}") install_python(path) - item_requirements = list(get_item_yaml_values(path, 'requirements')['requirements']) + item_requirements = list( + get_item_yaml_values(path, "requirements")["requirements"] + ) install_requirements(path, ["papermill"] + item_requirements) click.echo(f"Running tests for {path}...") running_ipynb = Path(path).name + ".ipynb" click.echo(f"Running notebook {running_ipynb}") - command = f'pipenv run papermill {running_ipynb} out.ipynb --log-output' + command = f"pipenv run papermill {running_ipynb} out.ipynb --log-output" completed_process: CompletedProcess = subprocess.run( f"cd {path} ;echo {command} ; {command}", stdout=sys.stdout, stderr=subprocess.PIPE, cwd=path, - shell=True + shell=True, ) meta_data = {"completed_process": completed_process, "test_path": path} @@ -438,7 +445,7 @@ def after_run(self): if failed_tests: exit(1) - def after_each(self, path: Union[str, Path], test_result: TestResult): + def after_each(self, path: str | Path, test_result: TestResult): if self.clean_env_artifacts: clean_pipenv(path) @@ -454,22 +461,19 @@ def after_each(self, path: Union[str, Path], test_result: TestResult): click.echo(complete_subprocess.stderr.decode("utf-8")) exit(test_result.status_code) - def _run(self, path: Union[str, Path], multi_processing, function_name): + def _run(self, path: str | Path, multi_processing, function_name): super()._run(path, multi_processing, function_name) @staticmethod def is_test_ipynb(path: Path): - return ( - path.is_file() - and path.name.endswith(".ipynb") - ) + return path.is_file() and path.name.endswith(".ipynb") class TestItemYamls(TestSuite): def __init__(self, stop_on_failure: bool = True): super().__init__(stop_on_failure) - def discover(self, path: Union[str, Path]) -> List[str]: + def discover(self, path: str | Path) -> list[str]: path = Path(path) testables = [] @@ -493,9 +497,9 @@ def discover(self, path: Union[str, Path]) -> List[str]: return testables - def run(self, path: Union[str, Path]) -> TestResult: + def run(self, path: str | Path) -> TestResult: path = Path(path) - item = yaml.full_load(open(path, "r")) + item = yaml.full_load(open(path)) directory = path.parent if item.get("spec")["filename"]: @@ -572,10 +576,10 @@ def after_run(self): if failed_tests: exit(1) - def before_each(self, path: Union[str, Path]): + def before_each(self, path: str | Path): pass - def after_each(self, path: Union[str, Path], test_result: TestResult): + def after_each(self, path: str | Path, test_result: TestResult): if self.stop_on_failure: if test_result.status == "Failed": message = test_result.meta_data["message"] @@ -583,7 +587,7 @@ def after_each(self, path: Union[str, Path], test_result: TestResult): click.echo(f"Error: {message}") exit(1) - def _run(self, path: Union[str, Path]): + def _run(self, path: str | Path): super()._run(path) @@ -599,20 +603,24 @@ def clean_pipenv(directory: str): # load item yaml def load_item(path): - with open(path, 'r') as stream: + with open(path) as stream: data = yaml.load(stream=stream, Loader=yaml.FullLoader) return data def is_test_valid_by_item(item_posix_path): - full_path = str(item_posix_path.absolute())+'/item.yaml' + full_path = str(item_posix_path.absolute()) + "/item.yaml" data = load_item(full_path) if data.get("test_valid") is not None: test_valid = data.get("test_valid") test_name = data.get("name") if not test_valid: - click.echo("==================== Test {} Not valid ====================".format(test_name)) - click.echo("==================== enable test_valid in item.yaml ====================") + click.echo( + f"==================== Test {test_name} Not valid ====================" + ) + click.echo( + "==================== enable test_valid in item.yaml ====================" + ) return test_valid else: return True diff --git a/cli/common/update_readme.py b/cli/common/update_readme.py index f6e582bb6..7816ebaa5 100644 --- a/cli/common/update_readme.py +++ b/cli/common/update_readme.py @@ -14,8 +14,8 @@ import sys +from collections.abc import Iterable from pathlib import Path -from typing import Iterable, List, Tuple import click import yaml @@ -28,6 +28,7 @@ "steps": ("Name", "Description", "Class Name", "Categories"), } + @click.command("update-readme") @click.option("-c", "--channel", default="master", help="Name of build channel") @click.option( @@ -35,12 +36,14 @@ multiple=True, required=True, help="Asset types to process (e.g: functions). " - "Pass multiple: --asset functions --asset modules", + "Pass multiple: --asset functions --asset modules", +) +@click.option( + "--check", + is_flag=True, + help="Do not write; exit non‑zero if README(s) would change.", ) -@click.option("--check", is_flag=True, - help="Do not write; exit non‑zero if README(s) would change.") -def update_readme(channel: str, asset: Iterable[str], - check: bool) -> None: +def update_readme(channel: str, asset: Iterable[str], check: bool) -> None: """ Regenerate the README tables for asset types from their item.yaml files. """ @@ -102,7 +105,11 @@ def _rows_for_asset_type(channel: str, asset_dir: Path, columns) -> list: kind = (data.get("spec", {}).get("kind", "")).strip() class_name = (data.get("className", "")).strip() cats = data.get("categories") or [] - cats_str = ", ".join(c.strip() for c in cats) if isinstance(cats, list) else str(cats).strip() + cats_str = ( + ", ".join(c.strip() for c in cats) + if isinstance(cats, list) + else str(cats).strip() + ) # Link the name to its source directory # Construct the relative path from the repo root for the asset rel_path = asset_dir.relative_to(Path(".").resolve()) @@ -135,7 +142,11 @@ def _build_table_md(rows, columns) -> str: "| " + " | ".join("---" for _ in columns) + " |", ] for r in rows: - lines.append("| " + " | ".join((cell or "").replace("\n", " ").strip() for cell in r) + " |") + lines.append( + "| " + + " | ".join((cell or "").replace("\n", " ").strip() for cell in r) + + " |" + ) return "\n".join(lines) diff --git a/cli/functions/function_to_item.py b/cli/functions/function_to_item.py index c3c870d75..e31364961 100644 --- a/cli/functions/function_to_item.py +++ b/cli/functions/function_to_item.py @@ -14,7 +14,6 @@ # from datetime import datetime from pathlib import Path -from typing import Union import click import yaml @@ -70,8 +69,7 @@ def function_to_item(path: str): exit(0) -def function_yaml_to_item(function_path: Union[str, Path]) -> dict: - +def function_yaml_to_item(function_path: str | Path) -> dict: function_path = Path(function_path) function_yaml = yaml.full_load(open(function_path)) diff --git a/cli/functions/item_to_function.py b/cli/functions/item_to_function.py index be84c0dce..80d95cd00 100644 --- a/cli/functions/item_to_function.py +++ b/cli/functions/item_to_function.py @@ -13,12 +13,11 @@ # limitations under the License. # from pathlib import Path -from typing import Optional, Union import click import semver import yaml -from black import format_str, FileMode +from black import FileMode, format_str from mlrun import code_to_function from yaml import full_load @@ -55,17 +54,21 @@ help="If -b/--bump_version is enabled, increase the minor version in the item.yaml file", ) def item_to_function_cli( - item_path: str, output_path: Optional[str], code_output: bool, format_code: bool, bump_version: bool + item_path: str, + output_path: str | None, + code_output: bool, + format_code: bool, + bump_version: bool, ): item_to_function(item_path, output_path, code_output, format_code, bump_version) def item_to_function( - item_path: str, - output_path: Optional[str] = None, - code_output: bool = False, - format_code: bool = True, - bump_version: bool = False, + item_path: str, + output_path: str | None = None, + code_output: bool = False, + format_code: bool = True, + bump_version: bool = False, ): item_path = Path(item_path) if item_path.is_dir(): @@ -74,17 +77,21 @@ def item_to_function( # That means we are in a specific item directory if item_path.exists(): _output_path = output_path or item_path.parent / "function.yaml" - create_function_yaml(item_path, _output_path, code_output, format_code, bump_version) + create_function_yaml( + item_path, _output_path, code_output, format_code, bump_version + ) # That means we need to search for items inside this direcotry else: for inner_dir in PathIterator( - root=item_path.parent, - rule=is_item_dir, - as_path=True, + root=item_path.parent, + rule=is_item_dir, + as_path=True, ): try: _output_path = output_path or (inner_dir / "function.yaml") - create_function_yaml(inner_dir, _output_path, code_output, format_code, bump_version) + create_function_yaml( + inner_dir, _output_path, code_output, format_code, bump_version + ) except Exception as e: print(e) click.echo(f"{inner_dir.name}: Failed to generate function.yaml") @@ -114,16 +121,16 @@ def _get_item_yaml(item_path: Path) -> dict: elif not item_path.exists(): raise FileNotFoundError(f"{item_path} not found") - item_yaml = full_load(open(item_path, "r")) + item_yaml = full_load(open(item_path)) return item_path, item_yaml def create_function_yaml( - item_path: Union[str, Path], - output_path: Optional[str] = None, - code_output: bool = False, - format_code: bool = True, - bump_version: bool = False, + item_path: str | Path, + output_path: str | None = None, + code_output: bool = False, + format_code: bool = True, + bump_version: bool = False, ): item_path = Path(item_path) if bump_version: @@ -157,11 +164,15 @@ def create_function_yaml( labels=item_yaml.get("labels", {}), with_doc=True, ) + + # Store only the file name in the function spec for portability. + function_object.spec.filename = Path(filename).name + function_object.metadata.project = "" # remove build info from object - function_object.spec.build.code_origin = '' - function_object.spec.build.origin_filename = '' - if 'state_thresholds' not in spec: + function_object.spec.build.code_origin = "" + function_object.spec.build.origin_filename = "" + if "state_thresholds" not in spec: function_object.spec.state_thresholds = None custom_fields = spec.get("customFields", {}) @@ -194,7 +205,7 @@ def create_function_yaml( function_object.export(target=str(output_path.resolve())) if code_output and format_code: - with open(_code_output, "r") as file: + with open(_code_output) as file: code = file.read() code = format_str(code, mode=FileMode()) with open(_code_output, "w") as file: @@ -206,5 +217,5 @@ def bump_function_yaml_version(item_path: Path): item_ver = item_yaml.get("version", "0.0.0") new_ver = semver.Version.parse(item_ver).bump_minor() item_yaml["version"] = str(new_ver) - with open(item_path, 'w') as file: + with open(item_path, "w") as file: yaml.safe_dump(item_yaml, file, default_flow_style=False) diff --git a/cli/marketplace/build.py b/cli/marketplace/build.py index 206886631..0d65dacce 100644 --- a/cli/marketplace/build.py +++ b/cli/marketplace/build.py @@ -18,7 +18,6 @@ import subprocess import uuid from pathlib import Path -from typing import Dict, List, Optional, Set, Union import click import yaml @@ -26,9 +25,14 @@ from sphinx.cmd.build import main as sphinx_build_cmd from sphinx.ext.apidoc import main as sphinx_apidoc_cmd -from cli.utils.helpers import (PROJECT_ROOT, get_item_yaml_values, - get_mock_requirements, is_item_dir, render_jinja) from cli.marketplace.changelog import ChangeLog +from cli.utils.helpers import ( + PROJECT_ROOT, + get_item_yaml_values, + get_mock_requirements, + is_item_dir, + render_jinja, +) from cli.utils.path_iterator import PathIterator _verbose = False @@ -192,7 +196,7 @@ def build_marketplace( write_change_log(marketplace_root / "README.md", change_log) -def print_file_tree(title: str, path: Union[str, Path]): +def print_file_tree(title: str, path: str | Path): click.echo(f"\n\n -- {title}:") path = Path(path) lines = ["---------------------------------", f"\t{path.resolve()}"] @@ -210,7 +214,7 @@ def print_file_tree(title: str, path: Union[str, Path]): def write_change_log(readme_path: Path, change_log: ChangeLog): readme_path.touch(exist_ok=True) - content = open(readme_path, "r").read() + content = open(readme_path).read() if change_log.changes_available: with open(readme_path, "w") as f: compiled_change_log = change_log.compile() @@ -218,7 +222,7 @@ def write_change_log(readme_path: Path, change_log: ChangeLog): f.write(content) -def write_index_html(marketplace_root: Union[str, Path]): +def write_index_html(marketplace_root: str | Path): marketplace_root = Path(marketplace_root) index_path = marketplace_root / "index.html" template_path = PROJECT_ROOT / "cli" / "marketplace" / "index.html" @@ -238,7 +242,12 @@ def copy_resources(marketplace_dir, temp_docs): def update_or_create_items( - source_dir, source_name, marketplace_dir, temp_docs, change_log, force_update: bool = False + source_dir, + source_name, + marketplace_dir, + temp_docs, + change_log, + force_update: bool = False, ): click.echo("Creating items...") for item_dir in PathIterator(root=source_dir, rule=is_item_dir, as_path=True): @@ -248,9 +257,9 @@ def update_or_create_items( def build_catalog_json( - marketplace_dir: Union[str, Path], - source_directory: Union[str, Path], - catalog_path: Union[str, Path], + marketplace_dir: str | Path, + source_directory: str | Path, + catalog_path: str | Path, change_log: ChangeLog, in_channel_directory: bool = True, with_assets: bool = False, @@ -275,7 +284,7 @@ def build_catalog_json( channel = marketplace_dir.name source = marketplace_dir.parent.name - catalog = json.load(open(catalog_path, "r")) if catalog_path.exists() else {} + catalog = json.load(open(catalog_path)) if catalog_path.exists() else {} funcs = catalog if in_channel_directory: @@ -325,7 +334,7 @@ def update_item_in_catalog(directory: Path, with_assets: bool) -> dict: """ source_yaml_path = directory / "src" / "item.yaml" - item_yaml = yaml.full_load(open(source_yaml_path, "r")) + item_yaml = yaml.full_load(open(source_yaml_path)) item_yaml["generationDate"] = str(item_yaml["generationDate"]) if with_assets: add_assets(item_yaml) @@ -360,7 +369,7 @@ def update_or_create_item( force_update: bool = False, ): # Copy source directories to target directories, if target already has the directory, archive previous version - item_yaml = yaml.full_load(open(item_dir / "item.yaml", "r")) + item_yaml = yaml.full_load(open(item_dir / "item.yaml")) source_version = item_yaml["version"] relative_path = "../../../" @@ -369,9 +378,7 @@ def update_or_create_item( target_version = marketplace_item / source_version if target_version.exists() and not force_update: - latest_item_yaml = yaml.full_load( - open(target_latest / "src" / "item.yaml", "r") - ) + latest_item_yaml = yaml.full_load(open(target_latest / "src" / "item.yaml")) if item_yaml["hidden"] == latest_item_yaml.get("hidden"): click.echo("Source version already exists in target directory!") return @@ -432,8 +439,7 @@ def update_or_create_item( source_py_name = item_yaml.get("spec", {}).get("filename", "") if source_py_name.endswith(".py") and (item_dir / source_py_name).exists(): - - with open((item_dir / source_py_name), "r") as f: + with open(item_dir / source_py_name) as f: source_code = f.read() render_jinja( @@ -447,7 +453,7 @@ def update_or_create_item( {"source_code": source_code}, ) - with open((item_dir / "item.yaml"), "r") as f: + with open(item_dir / "item.yaml") as f: source_code = f.read() render_jinja( @@ -466,7 +472,7 @@ def update_or_create_item( asset_yaml_path = item_dir / f"{asset_name}.yaml" if asset_yaml_path.exists(): - with open(asset_yaml_path, "r") as f: + with open(asset_yaml_path) as f: source_code = f.read() render_jinja( templates / "yaml.html", @@ -490,7 +496,7 @@ def update_html_resource_paths( item_name: str = None, ): if html_path.exists(): - with open(html_path, "r", encoding="utf8") as html: + with open(html_path, encoding="utf8") as html: parsed = BeautifulSoup(html.read(), features="html.parser") # Update back to docs link (from source page) @@ -516,9 +522,9 @@ def update_html_resource_paths( nodes = parsed.find_all(lambda node: "_sources" in node.get("href", "")) for node in nodes: # fix path and remove example from name: - node[ - "href" - ] = f'../{node["href"].replace("_sources", "src").replace("_example", "")}' + node["href"] = ( + f"../{node['href'].replace('_sources', 'src').replace('_example', '')}" + ) else: # Removing download option from documentation: nodes = parsed.find_all( @@ -551,7 +557,7 @@ def patch_temp_docs(source_dir, temp_docs): for directory in PathIterator(root=source_dir, rule=is_item_dir): directory = Path(directory) - with open(directory / "item.yaml", "r") as f: + with open(directory / "item.yaml") as f: item = yaml.full_load(f) example_file = directory / item["example"] @@ -576,7 +582,7 @@ def build_temp_project(source_dir, temp_root): item_count += 1 click.echo(f"[Temporary project] Now processing: {directory / 'item.yaml'}") - with open(directory / "item.yaml", "r") as f: + with open(directory / "item.yaml") as f: item = yaml.full_load(f) filename = item.get("spec")["filename"] @@ -594,8 +600,8 @@ def build_temp_project(source_dir, temp_root): def collect_values_from_items( - source_dir: Union[Path, str], tags_set: Set[str] -) -> Dict[str, List[str]]: + source_dir: Path | str, tags_set: set[str] +) -> dict[str, list[str]]: """ Collecting all tags values from item.yaml files. If the `with_requirements` flag is on than also collecting requirements from ite.yaml and requirements.txt files. @@ -626,9 +632,7 @@ def collect_values_from_items( return tags -def sphinx_quickstart( - temp_root: Union[str, Path], requirements: Optional[List[str]] = None -): +def sphinx_quickstart(temp_root: str | Path, requirements: list[str] | None = None): """ Generate required files for a Sphinx project. sphinx-quickstart is an interactive tool that asks some questions about your project and then @@ -694,5 +698,8 @@ def build_temp_docs(temp_root, temp_docs, source_dir): sphinx_apidoc_cmd(cmd.split(" ")) - shutil.copytree(PROJECT_ROOT / "cli" / "marketplace" / "_static" / "css", temp_docs / '_static/css') + shutil.copytree( + PROJECT_ROOT / "cli" / "marketplace" / "_static" / "css", + temp_docs / "_static/css", + ) click.echo("[Sphinx] Done autodoc") diff --git a/cli/utils/helpers.py b/cli/utils/helpers.py index fabccbf7a..df67c8c3e 100644 --- a/cli/utils/helpers.py +++ b/cli/utils/helpers.py @@ -15,10 +15,10 @@ import os import pathlib import subprocess -from pathlib import Path -from typing import Union, List, Set, Dict import sys from glob import iglob +from pathlib import Path + import yaml from jinja2 import Template @@ -35,13 +35,11 @@ def is_function_dir(path: Path) -> bool: # dir_name = path.name # ipynb_found = any((f.name.endswith(".ipynb") for f in path.iterdir())) # py_found = any((f.name.endswith(".py") for f in path.iterdir())) - return any((f.name == "function.yaml" for f in path.iterdir())) + return any(f.name == "function.yaml" for f in path.iterdir()) -def render_jinja( - template_path: Union[str, Path], output_path: Union[str, Path], data: dict -): - with open(template_path, "r") as t: +def render_jinja(template_path: str | Path, output_path: str | Path, data: dict): + with open(template_path) as t: template_text = t.read() template = Template(template_text) @@ -54,7 +52,7 @@ def render_jinja( def install_pipenv(): print("Installing pipenv...") pipenv_install: subprocess.CompletedProcess = subprocess.run( - f"export PIP_NO_INPUT=1;pip install pipenv==2023.10.24", + "export PIP_NO_INPUT=1;pip install pipenv==2023.10.24", stdout=sys.stdout, stderr=subprocess.PIPE, shell=True, @@ -62,12 +60,16 @@ def install_pipenv(): exit_on_non_zero_return(pipenv_install) -def install_python(directory: Union[str, Path]): +def install_python(directory: str | Path): print(f"Installing python for {directory} ...") - install_command = f"pipenv --rm;pipenv --python 3.10.17" - if (os.environ.get('CONDA_DEFAULT_ENV') is not None) and (os.environ.get('CONDA_PREFIX') is not None): + install_command = "pipenv --rm;pipenv --python 3.10.17" + if (os.environ.get("CONDA_DEFAULT_ENV") is not None) and ( + os.environ.get("CONDA_PREFIX") is not None + ): print("conda env detected using conda to get pipenv python version") - install_command = f"pipenv --rm;pipenv --python=$(conda run which python) --site-packages" + install_command = ( + "pipenv --rm;pipenv --python=$(conda run which python) --site-packages" + ) python_install: subprocess.CompletedProcess = subprocess.run( install_command, stdout=sys.stdout, @@ -81,7 +83,7 @@ def install_python(directory: Union[str, Path]): stderr = python_install.stderr.decode("utf8") stderr = stderr.split("\n") python_location = [l for l in stderr if "Virtualenv location: " in l] - if python_location.count(python_location)>0: + if python_location.count(python_location) > 0: python_location = ( python_location[0].split("Virtualenv location: ")[-1] + "bin/python" ) @@ -90,7 +92,7 @@ def install_python(directory: Union[str, Path]): return python_location -def _run_subprocess(cmd: List[str], directory): +def _run_subprocess(cmd: list[str], directory): completed_process: subprocess.CompletedProcess = subprocess.run( cmd, stdout=sys.stdout, @@ -103,14 +105,14 @@ def _run_subprocess(cmd: List[str], directory): def install_requirements( directory: str, - requirements: Union[List[str], Set[str]], + requirements: list[str] | set[str], ): """ Installing requirements from a requirements list/set and from a requirements.txt file if found in directory :param directory: The relevant directory were the requirements are installed and collected :param requirements: Requirement list/set with or without bounds """ - requirements_file = Path(directory) / 'requirements.txt' + requirements_file = Path(directory) / "requirements.txt" if not requirements and not requirements_file.exists(): print(f"No requirements found for {directory}...") @@ -120,7 +122,7 @@ def install_requirements( print(f"Installing requirements from {requirements_file}...") cmd = ["pipenv", "install", "--skip-lock", "-r", str(requirements_file)] _run_subprocess(cmd, directory) - with open(requirements_file, "r") as f: + with open(requirements_file) as f: mlrun_version = [l.replace("\n", "") for l in f.readlines() if "mlrun" in l] # remove mlrun from requirements if installed with version limits: if mlrun_version and any([c in mlrun_version[0] for c in "<>=~"]): @@ -133,8 +135,8 @@ def install_requirements( def get_item_yaml_values( - item_path: pathlib.Path, keys: Union[str, Set[str]] -) -> Dict[str, Set[str]]: + item_path: pathlib.Path, keys: str | set[str] +) -> dict[str, set[str]]: """ Getting value from item.yaml requested field. @@ -153,7 +155,7 @@ def get_item_yaml_values( item_path = Path(item_path) if item_path.is_dir(): item_path = item_path / "item.yaml" - with open(item_path, "r") as f: + with open(item_path) as f: item = yaml.full_load(f) if key in item: values = item.get(key, "") @@ -174,7 +176,7 @@ def get_item_yaml_values( return values_dict -def get_mock_requirements(source_dir: Union[str, Path]) -> List[str]: +def get_mock_requirements(source_dir: str | Path) -> list[str]: """ Getting all requirements from .py files inside all the subdirectories of the given source dir. Only the files with the same name as their parent directory are taken in consideration. @@ -197,13 +199,13 @@ def get_mock_requirements(source_dir: Union[str, Path]) -> List[str]: # Skipping test files continue # Getting all packages: - with open(filename, 'r') as f: + with open(filename) as f: lines = list(filter(None, f.read().split("\n"))) for line in lines: - words = line.split(' ') + words = line.split(" ") words = [w for w in words if w] - if words and (words[0] == 'from' or words[0] == 'import'): - mock_reqs.add(words[1].split('.')[0]) + if words and (words[0] == "from" or words[0] == "import"): + mock_reqs.add(words[1].split(".")[0]) return sorted(mock_reqs) diff --git a/cli/utils/path_iterator.py b/cli/utils/path_iterator.py index 0aaccc2b7..2ea62588d 100644 --- a/cli/utils/path_iterator.py +++ b/cli/utils/path_iterator.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from collections.abc import Callable from pathlib import Path -from typing import Optional, Callable, Union class PathIterator: @@ -27,8 +27,8 @@ class PathIterator: def __init__( self, - root: Union[str, Path], - rule: Optional[Callable[[Path], bool]] = None, + root: str | Path, + rule: Callable[[Path], bool] | None = None, recursive: bool = False, absolute: bool = True, as_path: bool = False, diff --git a/functions/src/aggregate/aggregate.py b/functions/src/aggregate/aggregate.py index 1e9d8502d..f3f555569 100644 --- a/functions/src/aggregate/aggregate.py +++ b/functions/src/aggregate/aggregate.py @@ -15,51 +15,52 @@ # Generated by nuclio.export.NuclioExporter import os + import pandas as pd from mlrun.datastore import DataItem -from typing import Union - - -def aggregate(context, - df_artifact: Union[DataItem, pd.core.frame.DataFrame], - save_to: str = 'aggregated-df.pq', - keys: list = None, - metrics: list = None, - labels: list = None, - metric_aggregations: list = ['mean'], - label_aggregations: list = ['max'], - suffix: str = '', - window: int = 3, - center: bool = False, - inplace: bool = False, - drop_na: bool = True, - files_to_select: int = 1): + +def aggregate( + context, + df_artifact: DataItem | pd.core.frame.DataFrame, + save_to: str = "aggregated-df.pq", + keys: list = None, + metrics: list = None, + labels: list = None, + metric_aggregations: list = ["mean"], + label_aggregations: list = ["max"], + suffix: str = "", + window: int = 3, + center: bool = False, + inplace: bool = False, + drop_na: bool = True, + files_to_select: int = 1, +): """Time-series aggregation function - + Will perform a rolling aggregation on {df_artifact}, over {window} by the selected {keys} applying {metric_aggregations} on {metrics} and {label_aggregations} on {labels}. adding {suffix} to the feature names. - + if not {inplace}, will return the original {df_artifact}, joined by the aggregated result. :param context: After running a job, you need to be able to track it. To gain the maximum value, MLRun uses the job context object inside the code. This provides access to job metadata, parameters, inputs, secrets, and API for logging and monitoring the results, as well as log text, files, artifacts, and labels. - - :param df_artifact: MLRun input pointing to pandas dataframe (csv/parquet file path) or a + + :param df_artifact: MLRun input pointing to pandas dataframe (csv/parquet file path) or a directory containing parquet files. * When given a directory the latest {files_to_select} will be selected :param save_to: Where to save the result dataframe. * If relative will add to the {artifact_path} :param keys: Subset of indexes from the source dataframe to aggregate by (default=all) - :param metrics: Array containing a list of metrics to run the aggregations on. (default=None) - :param labels: Array containing a list of labels to run the aggregations on. (default=None) + :param metrics: Array containing a list of metrics to run the aggregations on. (default=None) + :param labels: Array containing a list of labels to run the aggregations on. (default=None) :param metric_aggregations: Array containing a list of aggregation function names to run on {metrics}. (Ex: 'mean', 'std') (default='mean') :param label_aggregations: Array containing a list of aggregation function names to run on {metrics}. - (Ex: 'max', 'min') (default='max') + (Ex: 'max', 'min') (default='max') :param suffix: Suffix to add to the feature name, E.g: __ (Ex: 'last_60_minutes') (default='') :param window: Window size to perform the rolling aggregate on. (default=3) @@ -70,70 +71,99 @@ def aggregate(context, :param drop_na: Will drop na lines due to the Rolling. :param files_to_select: Specifies the number of *latest* files to select (and concat) for aggregation. """ - + from_model = type(df_artifact) == pd.DataFrame if from_model: - context.logger.info('Aggregating from Buffer') + context.logger.info("Aggregating from Buffer") input_df = df_artifact else: - if df_artifact.url.endswith('/'): # is a directory? - mpath = [os.path.join(df_artifact.url, file) for file in df_artifact.listdir() if file.endswith(('parquet', 'pq'))] + if df_artifact.url.endswith("/"): # is a directory? + mpath = [ + os.path.join(df_artifact.url, file) + for file in df_artifact.listdir() + if file.endswith(("parquet", "pq")) + ] files_by_updated = sorted(mpath, key=os.path.getmtime, reverse=True) context.logger.info(files_by_updated) latest = files_by_updated[:files_to_select] - context.logger.info(f'Aggregating {latest}') + context.logger.info(f"Aggregating {latest}") input_df = pd.concat([context.get_dataitem(df).as_df() for df in latest]) else: # A regular artifact - context.logger.info(f'Aggregating {df_artifact.url}') + context.logger.info(f"Aggregating {df_artifact.url}") input_df = df_artifact.as_df() - + if not (metrics or labels): - raise ValueError('please specify metrics or labels param') - + raise ValueError("please specify metrics or labels param") + if keys: current_index = input_df.index.names indexes_to_drop = [col for col in input_df.index.names if col not in keys] df = input_df.reset_index(level=indexes_to_drop) else: df = input_df - + if metrics: - metrics_df = df.loc[:, metrics].rolling(window=window, center=center).aggregate(metric_aggregations) - metrics_df.columns = ['_'.join(col).strip() for col in metrics_df.columns.values] - + metrics_df = ( + df.loc[:, metrics] + .rolling(window=window, center=center) + .aggregate(metric_aggregations) + ) + metrics_df.columns = [ + "_".join(col).strip() for col in metrics_df.columns.values + ] + if suffix: - metrics_df.columns = [f'{metric}_{suffix}' for metric in metrics_df.columns] - + metrics_df.columns = [f"{metric}_{suffix}" for metric in metrics_df.columns] + if not inplace: - final_df = pd.merge(input_df, metrics_df, suffixes=('', suffix), left_index=True, right_index=True) + final_df = pd.merge( + input_df, + metrics_df, + suffixes=("", suffix), + left_index=True, + right_index=True, + ) else: final_df = metrics_df if labels: - labels_df = df.loc[:, labels].rolling(window=window, - center=center).aggregate(label_aggregations) - labels_df.columns = ['_'.join(col).strip() for col in labels_df.columns.values] - + labels_df = ( + df.loc[:, labels] + .rolling(window=window, center=center) + .aggregate(label_aggregations) + ) + labels_df.columns = ["_".join(col).strip() for col in labels_df.columns.values] + if suffix: - labels_df.columns = [f'{label}_{suffix}' for label in labels_df.columns] - + labels_df.columns = [f"{label}_{suffix}" for label in labels_df.columns] + if metrics: - final_df = pd.merge(final_df, labels_df, suffixes=('', suffix), left_index=True, right_index=True) + final_df = pd.merge( + final_df, + labels_df, + suffixes=("", suffix), + left_index=True, + right_index=True, + ) else: if not inplace: - final_df = pd.merge(input_df, labels_df, suffixes=('', suffix), left_index=True, right_index=True) + final_df = pd.merge( + input_df, + labels_df, + suffixes=("", suffix), + left_index=True, + right_index=True, + ) else: final_df = labels_df - + if drop_na: final_df = final_df.dropna() - - context.logger.info('Logging artifact') + + context.logger.info("Logging artifact") if not from_model: - context.log_dataset(key='aggregate', - df=final_df, - format='parquet', - local_path=save_to) + context.log_dataset( + key="aggregate", df=final_df, format="parquet", local_path=save_to + ) else: return final_df - diff --git a/functions/src/aggregate/function.yaml b/functions/src/aggregate/function.yaml index 4782ee7ea..ba8b0656a 100644 --- a/functions/src/aggregate/function.yaml +++ b/functions/src/aggregate/function.yaml @@ -1,4 +1,18 @@ +metadata: + tag: '' + name: aggregate + categories: + - data-preparation +verbose: false +kind: job spec: + image: mlrun/mlrun + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IG9zCgppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQoKCmRlZiBhZ2dyZWdhdGUoCiAgICBjb250ZXh0LAogICAgZGZfYXJ0aWZhY3Q6IERhdGFJdGVtIHwgcGQuY29yZS5mcmFtZS5EYXRhRnJhbWUsCiAgICBzYXZlX3RvOiBzdHIgPSAiYWdncmVnYXRlZC1kZi5wcSIsCiAgICBrZXlzOiBsaXN0ID0gTm9uZSwKICAgIG1ldHJpY3M6IGxpc3QgPSBOb25lLAogICAgbGFiZWxzOiBsaXN0ID0gTm9uZSwKICAgIG1ldHJpY19hZ2dyZWdhdGlvbnM6IGxpc3QgPSBbIm1lYW4iXSwKICAgIGxhYmVsX2FnZ3JlZ2F0aW9uczogbGlzdCA9IFsibWF4Il0sCiAgICBzdWZmaXg6IHN0ciA9ICIiLAogICAgd2luZG93OiBpbnQgPSAzLAogICAgY2VudGVyOiBib29sID0gRmFsc2UsCiAgICBpbnBsYWNlOiBib29sID0gRmFsc2UsCiAgICBkcm9wX25hOiBib29sID0gVHJ1ZSwKICAgIGZpbGVzX3RvX3NlbGVjdDogaW50ID0gMSwKKToKICAgICIiIlRpbWUtc2VyaWVzIGFnZ3JlZ2F0aW9uIGZ1bmN0aW9uCgogICAgV2lsbCBwZXJmb3JtIGEgcm9sbGluZyBhZ2dyZWdhdGlvbiBvbiB7ZGZfYXJ0aWZhY3R9LCBvdmVyIHt3aW5kb3d9IGJ5IHRoZSBzZWxlY3RlZCB7a2V5c30KICAgIGFwcGx5aW5nIHttZXRyaWNfYWdncmVnYXRpb25zfSBvbiB7bWV0cmljc30gYW5kIHtsYWJlbF9hZ2dyZWdhdGlvbnN9IG9uIHtsYWJlbHN9LiBhZGRpbmcge3N1ZmZpeH0gdG8gdGhlCiAgICBmZWF0dXJlIG5hbWVzLgoKICAgIGlmIG5vdCB7aW5wbGFjZX0sIHdpbGwgcmV0dXJuIHRoZSBvcmlnaW5hbCB7ZGZfYXJ0aWZhY3R9LCBqb2luZWQgYnkgdGhlIGFnZ3JlZ2F0ZWQgcmVzdWx0LgoKICAgIDpwYXJhbSBjb250ZXh0OiBBZnRlciBydW5uaW5nIGEgam9iLCB5b3UgbmVlZCB0byBiZSBhYmxlIHRvIHRyYWNrIGl0LiBUbyBnYWluIHRoZSBtYXhpbXVtIHZhbHVlLCBNTFJ1biB1c2VzIHRoZQogICAgICAgICAgICAgICAgICAgIGpvYiBjb250ZXh0IG9iamVjdCBpbnNpZGUgdGhlIGNvZGUuIFRoaXMgcHJvdmlkZXMgYWNjZXNzIHRvIGpvYiBtZXRhZGF0YSwgcGFyYW1ldGVycywKICAgICAgICAgICAgICAgICAgICBpbnB1dHMsIHNlY3JldHMsIGFuZCBBUEkgZm9yIGxvZ2dpbmcgYW5kIG1vbml0b3JpbmcgdGhlIHJlc3VsdHMsIGFzIHdlbGwgYXMgbG9nIHRleHQsIGZpbGVzLAogICAgICAgICAgICAgICAgICAgIGFydGlmYWN0cywgYW5kIGxhYmVscy4KCiAgICA6cGFyYW0gZGZfYXJ0aWZhY3Q6IE1MUnVuIGlucHV0IHBvaW50aW5nIHRvIHBhbmRhcyBkYXRhZnJhbWUgKGNzdi9wYXJxdWV0IGZpbGUgcGF0aCkgb3IgYQogICAgICAgICAgICAgICAgICAgICAgICBkaXJlY3RvcnkgY29udGFpbmluZyBwYXJxdWV0IGZpbGVzLgogICAgICAgICAgICAgICAgICAgICAgICAqIFdoZW4gZ2l2ZW4gYSBkaXJlY3RvcnkgdGhlIGxhdGVzdCB7ZmlsZXNfdG9fc2VsZWN0fSB3aWxsIGJlIHNlbGVjdGVkCiAgICA6cGFyYW0gc2F2ZV90bzogICAgIFdoZXJlIHRvIHNhdmUgdGhlIHJlc3VsdCBkYXRhZnJhbWUuCiAgICAgICAgICAgICAgICAgICAgICAgICogSWYgcmVsYXRpdmUgd2lsbCBhZGQgdG8gdGhlIHthcnRpZmFjdF9wYXRofQogICAgOnBhcmFtIGtleXM6ICAgICAgICBTdWJzZXQgb2YgaW5kZXhlcyBmcm9tIHRoZSBzb3VyY2UgZGF0YWZyYW1lIHRvIGFnZ3JlZ2F0ZSBieSAoZGVmYXVsdD1hbGwpCiAgICA6cGFyYW0gbWV0cmljczogICAgIEFycmF5IGNvbnRhaW5pbmcgYSBsaXN0IG9mIG1ldHJpY3MgdG8gcnVuIHRoZSBhZ2dyZWdhdGlvbnMgb24uIChkZWZhdWx0PU5vbmUpCiAgICA6cGFyYW0gbGFiZWxzOiAgICAgIEFycmF5IGNvbnRhaW5pbmcgYSBsaXN0IG9mIGxhYmVscyB0byBydW4gdGhlIGFnZ3JlZ2F0aW9ucyBvbi4gKGRlZmF1bHQ9Tm9uZSkKICAgIDpwYXJhbSBtZXRyaWNfYWdncmVnYXRpb25zOiBBcnJheSBjb250YWluaW5nIGEgbGlzdCBvZiBhZ2dyZWdhdGlvbiBmdW5jdGlvbiBuYW1lcyB0byBydW4gb24ge21ldHJpY3N9LgogICAgICAgICAgICAgICAgICAgICAgICAoRXg6ICdtZWFuJywgJ3N0ZCcpIChkZWZhdWx0PSdtZWFuJykKICAgIDpwYXJhbSBsYWJlbF9hZ2dyZWdhdGlvbnM6ICBBcnJheSBjb250YWluaW5nIGEgbGlzdCBvZiBhZ2dyZWdhdGlvbiBmdW5jdGlvbiBuYW1lcyB0byBydW4gb24ge21ldHJpY3N9LgogICAgICAgICAgICAgICAgICAgICAgICAoRXg6ICdtYXgnLCAnbWluJykgKGRlZmF1bHQ9J21heCcpCiAgICA6cGFyYW0gc3VmZml4OiAgICAgIFN1ZmZpeCB0byBhZGQgdG8gdGhlIGZlYXR1cmUgbmFtZSwgRS5nOiA8RmVhdHVyZV9OYW1lPl88QWdnX0Z1bmN0aW9uPl88U3VmZml4PgogICAgICAgICAgICAgICAgICAgICAgICAoRXg6ICdsYXN0XzYwX21pbnV0ZXMnKSAoZGVmYXVsdD0nJykKICAgIDpwYXJhbSB3aW5kb3c6ICAgICAgV2luZG93IHNpemUgdG8gcGVyZm9ybSB0aGUgcm9sbGluZyBhZ2dyZWdhdGUgb24uIChkZWZhdWx0PTMpCiAgICA6cGFyYW0gY2VudGVyOiAgICAgIElmIFRydWUsIFNldHMgdGhlIHZhbHVlIGZvciB0aGUgY2VudHJhbCBzYW1wbGUgaW4gdGhlIHdpbmRvdywKICAgICAgICAgICAgICAgICAgICAgICAgSWYgRmFsc2UsIHdpbGwgc2V0IHRoZSB2YWx1ZSB0byB0aGUgbGFzdCBzYW1wbGUuIChkZWZhdWx0PUZhbHNlKQogICAgOnBhcmFtIGlucGxhY2U6ICAgICBJZiBUcnVlLCB3aWxsIHJldHVybiBvbmx5IHRoZSBhZ2dyZWdhdGVkIHJlc3VsdHMuCiAgICAgICAgICAgICAgICAgICAgICAgIElmIEZhbHNlLCB3aWxsIGpvaW4gdGhlIGFnZ3JlZ2F0ZWQgcmVzdWx0cyB3aXRoIHRoZSBvcmlnaW5hbCBkYXRhZnJhbWUKICAgIDpwYXJhbSBkcm9wX25hOiAgICAgV2lsbCBkcm9wIG5hIGxpbmVzIGR1ZSB0byB0aGUgUm9sbGluZy4KICAgIDpwYXJhbSBmaWxlc190b19zZWxlY3Q6IFNwZWNpZmllcyB0aGUgbnVtYmVyIG9mICpsYXRlc3QqIGZpbGVzIHRvIHNlbGVjdCAoYW5kIGNvbmNhdCkgZm9yIGFnZ3JlZ2F0aW9uLgogICAgIiIiCgogICAgZnJvbV9tb2RlbCA9IHR5cGUoZGZfYXJ0aWZhY3QpID09IHBkLkRhdGFGcmFtZQogICAgaWYgZnJvbV9tb2RlbDoKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJBZ2dyZWdhdGluZyBmcm9tIEJ1ZmZlciIpCiAgICAgICAgaW5wdXRfZGYgPSBkZl9hcnRpZmFjdAogICAgZWxzZToKICAgICAgICBpZiBkZl9hcnRpZmFjdC51cmwuZW5kc3dpdGgoIi8iKTogICMgaXMgYSBkaXJlY3Rvcnk/CiAgICAgICAgICAgIG1wYXRoID0gWwogICAgICAgICAgICAgICAgb3MucGF0aC5qb2luKGRmX2FydGlmYWN0LnVybCwgZmlsZSkKICAgICAgICAgICAgICAgIGZvciBmaWxlIGluIGRmX2FydGlmYWN0Lmxpc3RkaXIoKQogICAgICAgICAgICAgICAgaWYgZmlsZS5lbmRzd2l0aCgoInBhcnF1ZXQiLCAicHEiKSkKICAgICAgICAgICAgXQogICAgICAgICAgICBmaWxlc19ieV91cGRhdGVkID0gc29ydGVkKG1wYXRoLCBrZXk9b3MucGF0aC5nZXRtdGltZSwgcmV2ZXJzZT1UcnVlKQogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGZpbGVzX2J5X3VwZGF0ZWQpCiAgICAgICAgICAgIGxhdGVzdCA9IGZpbGVzX2J5X3VwZGF0ZWRbOmZpbGVzX3RvX3NlbGVjdF0KICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIkFnZ3JlZ2F0aW5nIHtsYXRlc3R9IikKICAgICAgICAgICAgaW5wdXRfZGYgPSBwZC5jb25jYXQoW2NvbnRleHQuZ2V0X2RhdGFpdGVtKGRmKS5hc19kZigpIGZvciBkZiBpbiBsYXRlc3RdKQogICAgICAgIGVsc2U6ICAjIEEgcmVndWxhciBhcnRpZmFjdAogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiQWdncmVnYXRpbmcge2RmX2FydGlmYWN0LnVybH0iKQogICAgICAgICAgICBpbnB1dF9kZiA9IGRmX2FydGlmYWN0LmFzX2RmKCkKCiAgICBpZiBub3QgKG1ldHJpY3Mgb3IgbGFiZWxzKToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJwbGVhc2Ugc3BlY2lmeSBtZXRyaWNzIG9yIGxhYmVscyBwYXJhbSIpCgogICAgaWYga2V5czoKICAgICAgICBjdXJyZW50X2luZGV4ID0gaW5wdXRfZGYuaW5kZXgubmFtZXMKICAgICAgICBpbmRleGVzX3RvX2Ryb3AgPSBbY29sIGZvciBjb2wgaW4gaW5wdXRfZGYuaW5kZXgubmFtZXMgaWYgY29sIG5vdCBpbiBrZXlzXQogICAgICAgIGRmID0gaW5wdXRfZGYucmVzZXRfaW5kZXgobGV2ZWw9aW5kZXhlc190b19kcm9wKQogICAgZWxzZToKICAgICAgICBkZiA9IGlucHV0X2RmCgogICAgaWYgbWV0cmljczoKICAgICAgICBtZXRyaWNzX2RmID0gKAogICAgICAgICAgICBkZi5sb2NbOiwgbWV0cmljc10KICAgICAgICAgICAgLnJvbGxpbmcod2luZG93PXdpbmRvdywgY2VudGVyPWNlbnRlcikKICAgICAgICAgICAgLmFnZ3JlZ2F0ZShtZXRyaWNfYWdncmVnYXRpb25zKQogICAgICAgICkKICAgICAgICBtZXRyaWNzX2RmLmNvbHVtbnMgPSBbCiAgICAgICAgICAgICJfIi5qb2luKGNvbCkuc3RyaXAoKSBmb3IgY29sIGluIG1ldHJpY3NfZGYuY29sdW1ucy52YWx1ZXMKICAgICAgICBdCgogICAgICAgIGlmIHN1ZmZpeDoKICAgICAgICAgICAgbWV0cmljc19kZi5jb2x1bW5zID0gW2Yie21ldHJpY31fe3N1ZmZpeH0iIGZvciBtZXRyaWMgaW4gbWV0cmljc19kZi5jb2x1bW5zXQoKICAgICAgICBpZiBub3QgaW5wbGFjZToKICAgICAgICAgICAgZmluYWxfZGYgPSBwZC5tZXJnZSgKICAgICAgICAgICAgICAgIGlucHV0X2RmLAogICAgICAgICAgICAgICAgbWV0cmljc19kZiwKICAgICAgICAgICAgICAgIHN1ZmZpeGVzPSgiIiwgc3VmZml4KSwKICAgICAgICAgICAgICAgIGxlZnRfaW5kZXg9VHJ1ZSwKICAgICAgICAgICAgICAgIHJpZ2h0X2luZGV4PVRydWUsCiAgICAgICAgICAgICkKICAgICAgICBlbHNlOgogICAgICAgICAgICBmaW5hbF9kZiA9IG1ldHJpY3NfZGYKCiAgICBpZiBsYWJlbHM6CiAgICAgICAgbGFiZWxzX2RmID0gKAogICAgICAgICAgICBkZi5sb2NbOiwgbGFiZWxzXQogICAgICAgICAgICAucm9sbGluZyh3aW5kb3c9d2luZG93LCBjZW50ZXI9Y2VudGVyKQogICAgICAgICAgICAuYWdncmVnYXRlKGxhYmVsX2FnZ3JlZ2F0aW9ucykKICAgICAgICApCiAgICAgICAgbGFiZWxzX2RmLmNvbHVtbnMgPSBbIl8iLmpvaW4oY29sKS5zdHJpcCgpIGZvciBjb2wgaW4gbGFiZWxzX2RmLmNvbHVtbnMudmFsdWVzXQoKICAgICAgICBpZiBzdWZmaXg6CiAgICAgICAgICAgIGxhYmVsc19kZi5jb2x1bW5zID0gW2Yie2xhYmVsfV97c3VmZml4fSIgZm9yIGxhYmVsIGluIGxhYmVsc19kZi5jb2x1bW5zXQoKICAgICAgICBpZiBtZXRyaWNzOgogICAgICAgICAgICBmaW5hbF9kZiA9IHBkLm1lcmdlKAogICAgICAgICAgICAgICAgZmluYWxfZGYsCiAgICAgICAgICAgICAgICBsYWJlbHNfZGYsCiAgICAgICAgICAgICAgICBzdWZmaXhlcz0oIiIsIHN1ZmZpeCksCiAgICAgICAgICAgICAgICBsZWZ0X2luZGV4PVRydWUsCiAgICAgICAgICAgICAgICByaWdodF9pbmRleD1UcnVlLAogICAgICAgICAgICApCiAgICAgICAgZWxzZToKICAgICAgICAgICAgaWYgbm90IGlucGxhY2U6CiAgICAgICAgICAgICAgICBmaW5hbF9kZiA9IHBkLm1lcmdlKAogICAgICAgICAgICAgICAgICAgIGlucHV0X2RmLAogICAgICAgICAgICAgICAgICAgIGxhYmVsc19kZiwKICAgICAgICAgICAgICAgICAgICBzdWZmaXhlcz0oIiIsIHN1ZmZpeCksCiAgICAgICAgICAgICAgICAgICAgbGVmdF9pbmRleD1UcnVlLAogICAgICAgICAgICAgICAgICAgIHJpZ2h0X2luZGV4PVRydWUsCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICBmaW5hbF9kZiA9IGxhYmVsc19kZgoKICAgIGlmIGRyb3BfbmE6CiAgICAgICAgZmluYWxfZGYgPSBmaW5hbF9kZi5kcm9wbmEoKQoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkxvZ2dpbmcgYXJ0aWZhY3QiKQogICAgaWYgbm90IGZyb21fbW9kZWw6CiAgICAgICAgY29udGV4dC5sb2dfZGF0YXNldCgKICAgICAgICAgICAga2V5PSJhZ2dyZWdhdGUiLCBkZj1maW5hbF9kZiwgZm9ybWF0PSJwYXJxdWV0IiwgbG9jYWxfcGF0aD1zYXZlX3RvCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXR1cm4gZmluYWxfZGYK + code_origin: '' + filename: aggregate.py entry_points: aggregate: parameters: @@ -8,10 +22,9 @@ spec: access to job metadata, parameters, inputs, secrets, and API for logging and monitoring the results, as well as log text, files, artifacts, and labels. - name: df_artifact - type: Union[DataItem, pd.core.frame.DataFrame] - doc: MLRun input pointing to pandas dataframe (csv/parquet file path) or a directory - containing parquet files. * When given a directory the latest {files_to_select} - will be selected + doc: MLRun input pointing to pandas dataframe (csv/parquet file path) or a + directory containing parquet files. * When given a directory the latest + {files_to_select} will be selected - name: save_to type: str doc: Where to save the result dataframe. * If relative will add to the {artifact_path} @@ -22,11 +35,11 @@ spec: default: null - name: metrics type: list - doc: 'Array containing a list of metrics to run the aggregations on. (default=None) ' + doc: Array containing a list of metrics to run the aggregations on. (default=None) default: null - name: labels type: list - doc: 'Array containing a list of labels to run the aggregations on. (default=None) ' + doc: Array containing a list of labels to run the aggregations on. (default=None) default: null - name: metric_aggregations type: list @@ -37,7 +50,7 @@ spec: - name: label_aggregations type: list doc: 'Array containing a list of aggregation function names to run on {metrics}. - (Ex: ''max'', ''min'') (default=''max'') ' + (Ex: ''max'', ''min'') (default=''max'')' default: - max - name: suffix @@ -67,6 +80,7 @@ spec: type: int doc: Specifies the number of *latest* files to select (and concat) for aggregation. default: 1 + name: aggregate doc: 'Time-series aggregation function @@ -81,23 +95,9 @@ spec: if not {inplace}, will return the original {df_artifact}, joined by the aggregated result.' - has_varargs: false - name: aggregate has_kwargs: false - lineno: 24 - disable_auto_mount: false + has_varargs: false + lineno: 23 + command: '' description: Rolling aggregation over Metrics and Lables according to specifications default_handler: aggregate - image: mlrun/mlrun - command: '' - build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IG9zCmltcG9ydCBwYW5kYXMgYXMgcGQKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCgpmcm9tIHR5cGluZyBpbXBvcnQgVW5pb24KCgpkZWYgYWdncmVnYXRlKGNvbnRleHQsCiAgICAgICAgICAgICAgZGZfYXJ0aWZhY3Q6IFVuaW9uW0RhdGFJdGVtLCBwZC5jb3JlLmZyYW1lLkRhdGFGcmFtZV0sCiAgICAgICAgICAgICAgc2F2ZV90bzogc3RyID0gJ2FnZ3JlZ2F0ZWQtZGYucHEnLAogICAgICAgICAgICAgIGtleXM6IGxpc3QgPSBOb25lLAogICAgICAgICAgICAgIG1ldHJpY3M6IGxpc3QgPSBOb25lLAogICAgICAgICAgICAgIGxhYmVsczogbGlzdCA9IE5vbmUsCiAgICAgICAgICAgICAgbWV0cmljX2FnZ3JlZ2F0aW9uczogbGlzdCA9IFsnbWVhbiddLAogICAgICAgICAgICAgIGxhYmVsX2FnZ3JlZ2F0aW9uczogbGlzdCA9IFsnbWF4J10sCiAgICAgICAgICAgICAgc3VmZml4OiBzdHIgPSAnJywKICAgICAgICAgICAgICB3aW5kb3c6IGludCA9IDMsCiAgICAgICAgICAgICAgY2VudGVyOiBib29sID0gRmFsc2UsCiAgICAgICAgICAgICAgaW5wbGFjZTogYm9vbCA9IEZhbHNlLAogICAgICAgICAgICAgIGRyb3BfbmE6IGJvb2wgPSBUcnVlLAogICAgICAgICAgICAgIGZpbGVzX3RvX3NlbGVjdDogaW50ID0gMSk6CiAgICAiIiJUaW1lLXNlcmllcyBhZ2dyZWdhdGlvbiBmdW5jdGlvbgogICAgCiAgICBXaWxsIHBlcmZvcm0gYSByb2xsaW5nIGFnZ3JlZ2F0aW9uIG9uIHtkZl9hcnRpZmFjdH0sIG92ZXIge3dpbmRvd30gYnkgdGhlIHNlbGVjdGVkIHtrZXlzfQogICAgYXBwbHlpbmcge21ldHJpY19hZ2dyZWdhdGlvbnN9IG9uIHttZXRyaWNzfSBhbmQge2xhYmVsX2FnZ3JlZ2F0aW9uc30gb24ge2xhYmVsc30uIGFkZGluZyB7c3VmZml4fSB0byB0aGUKICAgIGZlYXR1cmUgbmFtZXMuCiAgICAKICAgIGlmIG5vdCB7aW5wbGFjZX0sIHdpbGwgcmV0dXJuIHRoZSBvcmlnaW5hbCB7ZGZfYXJ0aWZhY3R9LCBqb2luZWQgYnkgdGhlIGFnZ3JlZ2F0ZWQgcmVzdWx0LgoKICAgIDpwYXJhbSBjb250ZXh0OiBBZnRlciBydW5uaW5nIGEgam9iLCB5b3UgbmVlZCB0byBiZSBhYmxlIHRvIHRyYWNrIGl0LiBUbyBnYWluIHRoZSBtYXhpbXVtIHZhbHVlLCBNTFJ1biB1c2VzIHRoZQogICAgICAgICAgICAgICAgICAgIGpvYiBjb250ZXh0IG9iamVjdCBpbnNpZGUgdGhlIGNvZGUuIFRoaXMgcHJvdmlkZXMgYWNjZXNzIHRvIGpvYiBtZXRhZGF0YSwgcGFyYW1ldGVycywKICAgICAgICAgICAgICAgICAgICBpbnB1dHMsIHNlY3JldHMsIGFuZCBBUEkgZm9yIGxvZ2dpbmcgYW5kIG1vbml0b3JpbmcgdGhlIHJlc3VsdHMsIGFzIHdlbGwgYXMgbG9nIHRleHQsIGZpbGVzLAogICAgICAgICAgICAgICAgICAgIGFydGlmYWN0cywgYW5kIGxhYmVscy4KICAgIAogICAgOnBhcmFtIGRmX2FydGlmYWN0OiBNTFJ1biBpbnB1dCBwb2ludGluZyB0byBwYW5kYXMgZGF0YWZyYW1lIChjc3YvcGFycXVldCBmaWxlIHBhdGgpIG9yIGEgCiAgICAgICAgICAgICAgICAgICAgICAgIGRpcmVjdG9yeSBjb250YWluaW5nIHBhcnF1ZXQgZmlsZXMuCiAgICAgICAgICAgICAgICAgICAgICAgICogV2hlbiBnaXZlbiBhIGRpcmVjdG9yeSB0aGUgbGF0ZXN0IHtmaWxlc190b19zZWxlY3R9IHdpbGwgYmUgc2VsZWN0ZWQKICAgIDpwYXJhbSBzYXZlX3RvOiAgICAgV2hlcmUgdG8gc2F2ZSB0aGUgcmVzdWx0IGRhdGFmcmFtZS4KICAgICAgICAgICAgICAgICAgICAgICAgKiBJZiByZWxhdGl2ZSB3aWxsIGFkZCB0byB0aGUge2FydGlmYWN0X3BhdGh9CiAgICA6cGFyYW0ga2V5czogICAgICAgIFN1YnNldCBvZiBpbmRleGVzIGZyb20gdGhlIHNvdXJjZSBkYXRhZnJhbWUgdG8gYWdncmVnYXRlIGJ5IChkZWZhdWx0PWFsbCkKICAgIDpwYXJhbSBtZXRyaWNzOiAgICAgQXJyYXkgY29udGFpbmluZyBhIGxpc3Qgb2YgbWV0cmljcyB0byBydW4gdGhlIGFnZ3JlZ2F0aW9ucyBvbi4gKGRlZmF1bHQ9Tm9uZSkgCiAgICA6cGFyYW0gbGFiZWxzOiAgICAgIEFycmF5IGNvbnRhaW5pbmcgYSBsaXN0IG9mIGxhYmVscyB0byBydW4gdGhlIGFnZ3JlZ2F0aW9ucyBvbi4gKGRlZmF1bHQ9Tm9uZSkgCiAgICA6cGFyYW0gbWV0cmljX2FnZ3JlZ2F0aW9uczogQXJyYXkgY29udGFpbmluZyBhIGxpc3Qgb2YgYWdncmVnYXRpb24gZnVuY3Rpb24gbmFtZXMgdG8gcnVuIG9uIHttZXRyaWNzfS4KICAgICAgICAgICAgICAgICAgICAgICAgKEV4OiAnbWVhbicsICdzdGQnKSAoZGVmYXVsdD0nbWVhbicpCiAgICA6cGFyYW0gbGFiZWxfYWdncmVnYXRpb25zOiAgQXJyYXkgY29udGFpbmluZyBhIGxpc3Qgb2YgYWdncmVnYXRpb24gZnVuY3Rpb24gbmFtZXMgdG8gcnVuIG9uIHttZXRyaWNzfS4KICAgICAgICAgICAgICAgICAgICAgICAgKEV4OiAnbWF4JywgJ21pbicpIChkZWZhdWx0PSdtYXgnKSAKICAgIDpwYXJhbSBzdWZmaXg6ICAgICAgU3VmZml4IHRvIGFkZCB0byB0aGUgZmVhdHVyZSBuYW1lLCBFLmc6IDxGZWF0dXJlX05hbWU+XzxBZ2dfRnVuY3Rpb24+XzxTdWZmaXg+CiAgICAgICAgICAgICAgICAgICAgICAgIChFeDogJ2xhc3RfNjBfbWludXRlcycpIChkZWZhdWx0PScnKQogICAgOnBhcmFtIHdpbmRvdzogICAgICBXaW5kb3cgc2l6ZSB0byBwZXJmb3JtIHRoZSByb2xsaW5nIGFnZ3JlZ2F0ZSBvbi4gKGRlZmF1bHQ9MykKICAgIDpwYXJhbSBjZW50ZXI6ICAgICAgSWYgVHJ1ZSwgU2V0cyB0aGUgdmFsdWUgZm9yIHRoZSBjZW50cmFsIHNhbXBsZSBpbiB0aGUgd2luZG93LAogICAgICAgICAgICAgICAgICAgICAgICBJZiBGYWxzZSwgd2lsbCBzZXQgdGhlIHZhbHVlIHRvIHRoZSBsYXN0IHNhbXBsZS4gKGRlZmF1bHQ9RmFsc2UpCiAgICA6cGFyYW0gaW5wbGFjZTogICAgIElmIFRydWUsIHdpbGwgcmV0dXJuIG9ubHkgdGhlIGFnZ3JlZ2F0ZWQgcmVzdWx0cy4KICAgICAgICAgICAgICAgICAgICAgICAgSWYgRmFsc2UsIHdpbGwgam9pbiB0aGUgYWdncmVnYXRlZCByZXN1bHRzIHdpdGggdGhlIG9yaWdpbmFsIGRhdGFmcmFtZQogICAgOnBhcmFtIGRyb3BfbmE6ICAgICBXaWxsIGRyb3AgbmEgbGluZXMgZHVlIHRvIHRoZSBSb2xsaW5nLgogICAgOnBhcmFtIGZpbGVzX3RvX3NlbGVjdDogU3BlY2lmaWVzIHRoZSBudW1iZXIgb2YgKmxhdGVzdCogZmlsZXMgdG8gc2VsZWN0IChhbmQgY29uY2F0KSBmb3IgYWdncmVnYXRpb24uCiAgICAiIiIKICAgIAogICAgZnJvbV9tb2RlbCA9IHR5cGUoZGZfYXJ0aWZhY3QpID09IHBkLkRhdGFGcmFtZQogICAgaWYgZnJvbV9tb2RlbDoKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCdBZ2dyZWdhdGluZyBmcm9tIEJ1ZmZlcicpCiAgICAgICAgaW5wdXRfZGYgPSBkZl9hcnRpZmFjdAogICAgZWxzZToKICAgICAgICBpZiBkZl9hcnRpZmFjdC51cmwuZW5kc3dpdGgoJy8nKTogICAjIGlzIGEgZGlyZWN0b3J5PwogICAgICAgICAgICBtcGF0aCA9IFtvcy5wYXRoLmpvaW4oZGZfYXJ0aWZhY3QudXJsLCBmaWxlKSBmb3IgZmlsZSBpbiBkZl9hcnRpZmFjdC5saXN0ZGlyKCkgaWYgZmlsZS5lbmRzd2l0aCgoJ3BhcnF1ZXQnLCAncHEnKSldCiAgICAgICAgICAgIGZpbGVzX2J5X3VwZGF0ZWQgPSBzb3J0ZWQobXBhdGgsIGtleT1vcy5wYXRoLmdldG10aW1lLCByZXZlcnNlPVRydWUpCiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZmlsZXNfYnlfdXBkYXRlZCkKICAgICAgICAgICAgbGF0ZXN0ID0gZmlsZXNfYnlfdXBkYXRlZFs6ZmlsZXNfdG9fc2VsZWN0XQogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYnQWdncmVnYXRpbmcge2xhdGVzdH0nKQogICAgICAgICAgICBpbnB1dF9kZiA9IHBkLmNvbmNhdChbY29udGV4dC5nZXRfZGF0YWl0ZW0oZGYpLmFzX2RmKCkgZm9yIGRmIGluIGxhdGVzdF0pCiAgICAgICAgZWxzZTogICMgQSByZWd1bGFyIGFydGlmYWN0CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZidBZ2dyZWdhdGluZyB7ZGZfYXJ0aWZhY3QudXJsfScpCiAgICAgICAgICAgIGlucHV0X2RmID0gZGZfYXJ0aWZhY3QuYXNfZGYoKQogICAgCiAgICBpZiBub3QgKG1ldHJpY3Mgb3IgbGFiZWxzKToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCdwbGVhc2Ugc3BlY2lmeSBtZXRyaWNzIG9yIGxhYmVscyBwYXJhbScpCiAgICAKICAgIGlmIGtleXM6CiAgICAgICAgY3VycmVudF9pbmRleCA9IGlucHV0X2RmLmluZGV4Lm5hbWVzCiAgICAgICAgaW5kZXhlc190b19kcm9wID0gW2NvbCBmb3IgY29sIGluIGlucHV0X2RmLmluZGV4Lm5hbWVzIGlmIGNvbCBub3QgaW4ga2V5c10KICAgICAgICBkZiA9IGlucHV0X2RmLnJlc2V0X2luZGV4KGxldmVsPWluZGV4ZXNfdG9fZHJvcCkKICAgIGVsc2U6CiAgICAgICAgZGYgPSBpbnB1dF9kZgogICAgICAgIAogICAgaWYgbWV0cmljczoKICAgICAgICBtZXRyaWNzX2RmID0gZGYubG9jWzosIG1ldHJpY3NdLnJvbGxpbmcod2luZG93PXdpbmRvdywgY2VudGVyPWNlbnRlcikuYWdncmVnYXRlKG1ldHJpY19hZ2dyZWdhdGlvbnMpCiAgICAgICAgbWV0cmljc19kZi5jb2x1bW5zID0gWydfJy5qb2luKGNvbCkuc3RyaXAoKSBmb3IgY29sIGluIG1ldHJpY3NfZGYuY29sdW1ucy52YWx1ZXNdCiAgICAgICAgCiAgICAgICAgaWYgc3VmZml4OgogICAgICAgICAgICBtZXRyaWNzX2RmLmNvbHVtbnMgPSBbZid7bWV0cmljfV97c3VmZml4fScgZm9yIG1ldHJpYyBpbiBtZXRyaWNzX2RmLmNvbHVtbnNdCiAgICAgICAgICAgIAogICAgICAgIGlmIG5vdCBpbnBsYWNlOgogICAgICAgICAgICBmaW5hbF9kZiA9IHBkLm1lcmdlKGlucHV0X2RmLCBtZXRyaWNzX2RmLCBzdWZmaXhlcz0oJycsIHN1ZmZpeCksIGxlZnRfaW5kZXg9VHJ1ZSwgcmlnaHRfaW5kZXg9VHJ1ZSkKICAgICAgICBlbHNlOgogICAgICAgICAgICBmaW5hbF9kZiA9IG1ldHJpY3NfZGYKCiAgICBpZiBsYWJlbHM6CiAgICAgICAgbGFiZWxzX2RmID0gZGYubG9jWzosIGxhYmVsc10ucm9sbGluZyh3aW5kb3c9d2luZG93LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgY2VudGVyPWNlbnRlcikuYWdncmVnYXRlKGxhYmVsX2FnZ3JlZ2F0aW9ucykKICAgICAgICBsYWJlbHNfZGYuY29sdW1ucyA9IFsnXycuam9pbihjb2wpLnN0cmlwKCkgZm9yIGNvbCBpbiBsYWJlbHNfZGYuY29sdW1ucy52YWx1ZXNdCiAgICAgICAgCiAgICAgICAgaWYgc3VmZml4OgogICAgICAgICAgICBsYWJlbHNfZGYuY29sdW1ucyA9IFtmJ3tsYWJlbH1fe3N1ZmZpeH0nIGZvciBsYWJlbCBpbiBsYWJlbHNfZGYuY29sdW1uc10KICAgICAgICAgICAgCiAgICAgICAgaWYgbWV0cmljczoKICAgICAgICAgICAgZmluYWxfZGYgPSBwZC5tZXJnZShmaW5hbF9kZiwgbGFiZWxzX2RmLCBzdWZmaXhlcz0oJycsIHN1ZmZpeCksIGxlZnRfaW5kZXg9VHJ1ZSwgcmlnaHRfaW5kZXg9VHJ1ZSkgICAKICAgICAgICBlbHNlOgogICAgICAgICAgICBpZiBub3QgaW5wbGFjZToKICAgICAgICAgICAgICAgIGZpbmFsX2RmID0gcGQubWVyZ2UoaW5wdXRfZGYsIGxhYmVsc19kZiwgc3VmZml4ZXM9KCcnLCBzdWZmaXgpLCBsZWZ0X2luZGV4PVRydWUsIHJpZ2h0X2luZGV4PVRydWUpICAgICAgCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICBmaW5hbF9kZiA9IGxhYmVsc19kZgogICAgICAgICAgICAgICAgCiAgICBpZiBkcm9wX25hOgogICAgICAgIGZpbmFsX2RmID0gZmluYWxfZGYuZHJvcG5hKCkKICAgICAgICAKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oJ0xvZ2dpbmcgYXJ0aWZhY3QnKQogICAgaWYgbm90IGZyb21fbW9kZWw6CiAgICAgICAgY29udGV4dC5sb2dfZGF0YXNldChrZXk9J2FnZ3JlZ2F0ZScsIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgZGY9ZmluYWxfZGYsIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgZm9ybWF0PSdwYXJxdWV0JywKICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxvY2FsX3BhdGg9c2F2ZV90bykKICAgIGVsc2U6CiAgICAgICAgcmV0dXJuIGZpbmFsX2RmCgo= - code_origin: '' - origin_filename: '' -verbose: false -metadata: - categories: - - data-preparation - name: aggregate - tag: '' -kind: job diff --git a/functions/src/aggregate/test_aggregate.py b/functions/src/aggregate/test_aggregate.py index 87248ac50..694da13ad 100644 --- a/functions/src/aggregate/test_aggregate.py +++ b/functions/src/aggregate/test_aggregate.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from pathlib import Path import os + from mlrun import code_to_function, import_function AGGREGATE_PATH = "artifacts/aggregate.pq" @@ -21,26 +21,27 @@ def test_run_local_aggregate(): - fn = code_to_function(name='code_to_function', - filename="aggregate.py", - handler="aggregate", - kind="local", - ) + fn = code_to_function( + name="code_to_function", + filename="aggregate.py", + handler="aggregate", + kind="local", + ) fn.run( params={ - 'metrics': ['cpu_utilization'], - 'labels': ['is_error'], - 'metric_aggs': ['mean', 'sum'], - 'label_aggs': ['max'], - 'suffix': 'daily', - 'inplace': False, - 'window': 5, - 'center': True, - 'save_to': AGGREGATE_PATH, - 'files_to_select': 2 + "metrics": ["cpu_utilization"], + "labels": ["is_error"], + "metric_aggs": ["mean", "sum"], + "label_aggs": ["max"], + "suffix": "daily", + "inplace": False, + "window": 5, + "center": True, + "save_to": AGGREGATE_PATH, + "files_to_select": 2, }, local=True, - inputs={'df_artifact': DATA} + inputs={"df_artifact": DATA}, ) assert os.path.exists("code-to-function-aggregate/0/aggregate.pq") == True @@ -49,18 +50,18 @@ def test_import_function_aggregate(): fn = import_function("function.yaml") fn.run( params={ - 'metrics': ['cpu_utilization'], - 'labels': ['is_error'], - 'metric_aggs': ['mean', 'sum'], - 'label_aggs': ['max'], - 'suffix': 'daily', - 'inplace': False, - 'window': 5, - 'center': True, - 'save_to': AGGREGATE_PATH, - 'files_to_select': 2, + "metrics": ["cpu_utilization"], + "labels": ["is_error"], + "metric_aggs": ["mean", "sum"], + "label_aggs": ["max"], + "suffix": "daily", + "inplace": False, + "window": 5, + "center": True, + "save_to": AGGREGATE_PATH, + "files_to_select": 2, }, local=True, - inputs={'df_artifact': DATA}, + inputs={"df_artifact": DATA}, ) assert os.path.exists("aggregate-aggregate/0/aggregate.pq") == True diff --git a/functions/src/arc_to_parquet/arc_to_parquet.py b/functions/src/arc_to_parquet/arc_to_parquet.py index d9275b7ca..ebae092a1 100644 --- a/functions/src/arc_to_parquet/arc_to_parquet.py +++ b/functions/src/arc_to_parquet/arc_to_parquet.py @@ -12,28 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os + +import numpy as np import pandas as pd -import pyarrow.parquet as pq import pyarrow as pa -import numpy as np - - -from mlrun.execution import MLClientCtx +import pyarrow.parquet as pq from mlrun.datastore import DataItem - -from typing import List -import os - +from mlrun.execution import MLClientCtx def _chunk_readwrite( - archive_url, - dest_path, - chunksize, - header, - encoding, - dtype, - dataset + archive_url, dest_path, chunksize, header, encoding, dtype, dataset ): """stream read and write archives @@ -46,9 +36,15 @@ def _chunk_readwrite( """ pqwriter = None header = [] - for i, df in enumerate(pd.read_csv(archive_url, chunksize=chunksize, - names=header, encoding=encoding, - dtype=dtype)): + for i, df in enumerate( + pd.read_csv( + archive_url, + chunksize=chunksize, + names=header, + encoding=encoding, + dtype=dtype, + ) + ): table = pa.Table.from_pandas(df) if i == 0: if dataset: @@ -56,7 +52,9 @@ def _chunk_readwrite( else: pqwriter = pq.ParquetWriter(dest_path, table.schema) if dataset: - pq.write_to_dataset(table, root_path=dest_path, partition_cols=partition_cols) + pq.write_to_dataset( + table, root_path=dest_path, partition_cols=partition_cols + ) else: pqwriter.write_table(table) if pqwriter: @@ -66,19 +64,19 @@ def _chunk_readwrite( def arc_to_parquet( - context: MLClientCtx, - archive_url: DataItem, - header: List[str] = [None], - chunksize: int = 0, - dtype=None, - encoding: str = "latin-1", - key: str = "data", - dataset: str = "None", - part_cols=[], - file_ext: str = "parquet", - index: bool = False, - refresh_data: bool = False, - stats: bool = False + context: MLClientCtx, + archive_url: DataItem, + header: list[str] = [None], + chunksize: int = 0, + dtype=None, + encoding: str = "latin-1", + key: str = "data", + dataset: str = "None", + part_cols=[], + file_ext: str = "parquet", + index: bool = False, + refresh_data: bool = False, + stats: bool = False, ) -> None: """Open a file/object archive and save as a parquet file or dataset @@ -123,12 +121,14 @@ def arc_to_parquet( if not exists: context.logger.info("destination file does not exist, downloading") if chunksize > 0: - header = _chunk_readwrite(archive_url, dest_path, chunksize, - encoding, dtype, dataset) - context.log_dataset(key=key, stats=stats, format='parquet', - target_path=dest_path) + header = _chunk_readwrite( + archive_url, dest_path, chunksize, encoding, dtype, dataset + ) + context.log_dataset( + key=key, stats=stats, format="parquet", target_path=dest_path + ) else: df = pd.read_csv(archive_url) context.log_dataset(key, df=df, format=file_ext, index=index) else: - context.logger.info("destination file already exists, nothing done") \ No newline at end of file + context.logger.info("destination file already exists, nothing done") diff --git a/functions/src/arc_to_parquet/function.yaml b/functions/src/arc_to_parquet/function.yaml index ca2c31921..7feaf4ece 100644 --- a/functions/src/arc_to_parquet/function.yaml +++ b/functions/src/arc_to_parquet/function.yaml @@ -1,17 +1,22 @@ +metadata: + tag: '' + name: arc-to-parquet + categories: + - utils +verbose: false kind: job spec: image: mlrun/mlrun + disable_auto_mount: false build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IHBhbmRhcyBhcyBwZAppbXBvcnQgcHlhcnJvdy5wYXJxdWV0IGFzIHBxCmltcG9ydCBweWFycm93IGFzIHBhCmltcG9ydCBudW1weSBhcyBucAoKCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmRhdGFzdG9yZSBpbXBvcnQgRGF0YUl0ZW0KCmZyb20gdHlwaW5nIGltcG9ydCBMaXN0CmltcG9ydCBvcwoKCgpkZWYgX2NodW5rX3JlYWR3cml0ZSgKICAgICAgICBhcmNoaXZlX3VybCwKICAgICAgICBkZXN0X3BhdGgsCiAgICAgICAgY2h1bmtzaXplLAogICAgICAgIGhlYWRlciwKICAgICAgICBlbmNvZGluZywKICAgICAgICBkdHlwZSwKICAgICAgICBkYXRhc2V0Cik6CiAgICAiIiJzdHJlYW0gcmVhZCBhbmQgd3JpdGUgYXJjaGl2ZXMKCiAgICBwYW5kYXMgcmVhZHMgYW5kIHBhcnF1ZXQgd3JpdGVzCgogICAgbm90ZXMKICAgIC0tLS0tCiAgICAqIGRlc3RfcGF0aCBjYW4gYmUgZWl0aGVyIGEgZmlsZS5wYXJxdWV0LCBvciBpbiBodGUgY2FzZSBvZiBwYXJ0aXRpb25lZCBwYXJxdWV0CiAgICAgIGl0IHdpbGwgYmUgb25seSB0aGUgZGVzdGluYXRpb24gZm9sZGVyIG9mIHRoZSBwYXJxdWV0IHBhcnRpdGlvbiBmaWxlcwogICAgIiIiCiAgICBwcXdyaXRlciA9IE5vbmUKICAgIGhlYWRlciA9IFtdCiAgICBmb3IgaSwgZGYgaW4gZW51bWVyYXRlKHBkLnJlYWRfY3N2KGFyY2hpdmVfdXJsLCBjaHVua3NpemU9Y2h1bmtzaXplLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBuYW1lcz1oZWFkZXIsIGVuY29kaW5nPWVuY29kaW5nLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkdHlwZT1kdHlwZSkpOgogICAgICAgIHRhYmxlID0gcGEuVGFibGUuZnJvbV9wYW5kYXMoZGYpCiAgICAgICAgaWYgaSA9PSAwOgogICAgICAgICAgICBpZiBkYXRhc2V0OgogICAgICAgICAgICAgICAgaGVhZGVyID0gbnAuY29weSh0YWJsZS5zY2hlbWEpCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICBwcXdyaXRlciA9IHBxLlBhcnF1ZXRXcml0ZXIoZGVzdF9wYXRoLCB0YWJsZS5zY2hlbWEpCiAgICAgICAgaWYgZGF0YXNldDoKICAgICAgICAgICAgcHEud3JpdGVfdG9fZGF0YXNldCh0YWJsZSwgcm9vdF9wYXRoPWRlc3RfcGF0aCwgcGFydGl0aW9uX2NvbHM9cGFydGl0aW9uX2NvbHMpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcHF3cml0ZXIud3JpdGVfdGFibGUodGFibGUpCiAgICBpZiBwcXdyaXRlcjoKICAgICAgICBwcXdyaXRlci5jbG9zZSgpCgogICAgcmV0dXJuIGhlYWRlcgoKCmRlZiBhcmNfdG9fcGFycXVldCgKICAgICAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgICAgICBhcmNoaXZlX3VybDogRGF0YUl0ZW0sCiAgICAgICAgaGVhZGVyOiBMaXN0W3N0cl0gPSBbTm9uZV0sCiAgICAgICAgY2h1bmtzaXplOiBpbnQgPSAwLAogICAgICAgIGR0eXBlPU5vbmUsCiAgICAgICAgZW5jb2Rpbmc6IHN0ciA9ICJsYXRpbi0xIiwKICAgICAgICBrZXk6IHN0ciA9ICJkYXRhIiwKICAgICAgICBkYXRhc2V0OiBzdHIgPSAiTm9uZSIsCiAgICAgICAgcGFydF9jb2xzPVtdLAogICAgICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICAgICAgaW5kZXg6IGJvb2wgPSBGYWxzZSwKICAgICAgICByZWZyZXNoX2RhdGE6IGJvb2wgPSBGYWxzZSwKICAgICAgICBzdGF0czogYm9vbCA9IEZhbHNlCikgLT4gTm9uZToKICAgICIiIk9wZW4gYSBmaWxlL29iamVjdCBhcmNoaXZlIGFuZCBzYXZlIGFzIGEgcGFycXVldCBmaWxlIG9yIGRhdGFzZXQKCiAgICBOb3RlcwogICAgLS0tLS0KICAgICogdGhpcyBmdW5jdGlvbiBpcyB0eXBpY2FsbHkgZm9yIGxhcmdlIGZpbGVzLCBwbGVhc2UgYmUgc3VyZSB0byBjaGVjayBhbGwgc2V0dGluZ3MKICAgICogcGFydGl0aW9uaW5nIHJlcXVpcmVzIHByZWNpc2Ugc3BlY2lmaWNhdGlvbiBvZiBjb2x1bW4gdHlwZXMuCiAgICAqIHRoZSBhcmNoaXZlX3VybCBjYW4gYmUgYW55IGZpbGUgcmVhZGFibGUgYnkgcGFuZGFzIHJlYWRfY3N2LCB3aGljaCBpbmNsdWRlcyB0YXIgZmlsZXMKICAgICogaWYgdGhlIGBkYXRhc2V0YCBwYXJhbWV0ZXIgaXMgbm90IGVtcHR5LCB0aGVuIGEgcGFydGl0aW9uZWQgZGF0YXNldCB3aWxsIGJlIGNyZWF0ZWQKICAgIGluc3RlYWQgb2YgYSBzaW5nbGUgZmlsZSBpbiB0aGUgZm9sZGVyIGBkYXRhc2V0YAogICAgKiBpZiBhIGtleSBleGlzdHMgYWxyZWFkeSB0aGVuIGl0IHdpbGwgbm90IGJlIHJlLWFjcXVpcmVkIHVubGVzcyB0aGUgYHJlZnJlc2hfZGF0YWAgcGFyYW0KICAgIGlzIHNldCB0byBgVHJ1ZWAuICBUaGlzIGlzIGluIGNhc2UgdGhlIG9yaWdpbmFsIGZpbGUgaXMgY29ycnVwdCwgb3IgYSByZWZyZXNoIGlzCiAgICByZXF1aXJlZC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgIHRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gYXJjaGl2ZV91cmw6ICAgIE1MUnVuIGRhdGEgaW5wdXQgKERhdGFJdGVtIG9iamVjdCkKICAgIDpwYXJhbSBjaHVua3NpemU6ICAgICAgKDApIHdoZW4gPiAwLCByb3cgc2l6ZSAoY2h1bmspIHRvIHJldHJpZXZlCiAgICAgICAgICAgICAgICAgICAgICAgICAgIHBlciBpdGVyYXRpb24KICAgIDpwYXJhbSBkdHlwZSAgICAgICAgICAgZGVzdGluYXRpb24gZGF0YSB0eXBlIG9mIHNwZWNpZmllZCBjb2x1bW5zCiAgICA6cGFyYW0gZW5jb2RpbmcgICAgICAgICgibGF0aW4tOCIpIGZpbGUgZW5jb2RpbmcKICAgIDpwYXJhbSBrZXk6ICAgICAgICAgICAga2V5IGluIGFydGlmYWN0IHN0b3JlICh3aGVuIGxvZ19kYXRhPVRydWUpCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgIChOb25lKSBpZiBub3QgTm9uZSB0aGVuICJ0YXJnZXRfcGF0aC9kYXRhc2V0IgogICAgICAgICAgICAgICAgICAgICAgICAgICBpcyBmb2xkZXIgZm9yIHBhcnRpdGlvbmVkIGZpbGVzCiAgICA6cGFyYW0gcGFydF9jb2xzOiAgICAgIChbXSkgbGlzdCBvZiBwYXJ0aXRpb25pbmcgY29sdW1ucwogICAgOnBhcmFtIGZpbGVfZXh0OiAgICAgICAocGFycXVldCkgY3N2L3BhcnF1ZXQgZmlsZSBleHRlbnNpb24KICAgIDpwYXJhbSBpbmRleDogICAgICAgICAgKEZhbHNlKSBwYW5kYXMgc2F2ZSBpbmRleCBvcHRpb24KICAgIDpwYXJhbSByZWZyZXNoX2RhdGE6ICAgKEZhbHNlKSBvdmVyd3JpdGUgZXhpc3RpbmcgZGF0YSBhdCB0aGF0IGxvY2F0aW9uCiAgICA6cGFyYW0gc3RhdHM6ICAgICAgICAgIChOb25lKSBjYWxjdWxhdGUgdGFibGUgc3RhdHMgd2hlbiBsb2dnaW5nIGFydGlmYWN0CiAgICAiIiIKICAgIGJhc2VfcGF0aCA9IGNvbnRleHQuYXJ0aWZhY3RfcGF0aAogICAgb3MubWFrZWRpcnMoYmFzZV9wYXRoLCBleGlzdF9vaz1UcnVlKQoKICAgIGFyY2hpdmVfdXJsID0gYXJjaGl2ZV91cmwubG9jYWwoKQoKICAgIGlmIGRhdGFzZXQgaXMgbm90IE5vbmU6CiAgICAgICAgZGVzdF9wYXRoID0gb3MucGF0aC5qb2luKGJhc2VfcGF0aCwgZGF0YXNldCkKICAgICAgICBleGlzdHMgPSBvcy5wYXRoLmlzZGlyKGRlc3RfcGF0aCkKICAgIGVsc2U6CiAgICAgICAgZGVzdF9wYXRoID0gb3MucGF0aC5qb2luKGJhc2VfcGF0aCwga2V5ICsgZiIue2ZpbGVfZXh0fSIpCiAgICAgICAgZXhpc3RzID0gb3MucGF0aC5pc2ZpbGUoZGVzdF9wYXRoKQoKICAgIGlmIG5vdCBleGlzdHM6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygiZGVzdGluYXRpb24gZmlsZSBkb2VzIG5vdCBleGlzdCwgZG93bmxvYWRpbmciKQogICAgICAgIGlmIGNodW5rc2l6ZSA+IDA6CiAgICAgICAgICAgIGhlYWRlciA9IF9jaHVua19yZWFkd3JpdGUoYXJjaGl2ZV91cmwsIGRlc3RfcGF0aCwgY2h1bmtzaXplLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGVuY29kaW5nLCBkdHlwZSwgZGF0YXNldCkKICAgICAgICAgICAgY29udGV4dC5sb2dfZGF0YXNldChrZXk9a2V5LCBzdGF0cz1zdGF0cywgZm9ybWF0PSdwYXJxdWV0JywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0YXJnZXRfcGF0aD1kZXN0X3BhdGgpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgZGYgPSBwZC5yZWFkX2NzdihhcmNoaXZlX3VybCkKICAgICAgICAgICAgY29udGV4dC5sb2dfZGF0YXNldChrZXksIGRmPWRmLCBmb3JtYXQ9ZmlsZV9leHQsIGluZGV4PWluZGV4KQogICAgZWxzZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJkZXN0aW5hdGlvbiBmaWxlIGFscmVhZHkgZXhpc3RzLCBub3RoaW5nIGRvbmUiKQ== origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IG9zCgppbXBvcnQgbnVtcHkgYXMgbnAKaW1wb3J0IHBhbmRhcyBhcyBwZAppbXBvcnQgcHlhcnJvdyBhcyBwYQppbXBvcnQgcHlhcnJvdy5wYXJxdWV0IGFzIHBxCmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKCgpkZWYgX2NodW5rX3JlYWR3cml0ZSgKICAgIGFyY2hpdmVfdXJsLCBkZXN0X3BhdGgsIGNodW5rc2l6ZSwgaGVhZGVyLCBlbmNvZGluZywgZHR5cGUsIGRhdGFzZXQKKToKICAgICIiInN0cmVhbSByZWFkIGFuZCB3cml0ZSBhcmNoaXZlcwoKICAgIHBhbmRhcyByZWFkcyBhbmQgcGFycXVldCB3cml0ZXMKCiAgICBub3RlcwogICAgLS0tLS0KICAgICogZGVzdF9wYXRoIGNhbiBiZSBlaXRoZXIgYSBmaWxlLnBhcnF1ZXQsIG9yIGluIGh0ZSBjYXNlIG9mIHBhcnRpdGlvbmVkIHBhcnF1ZXQKICAgICAgaXQgd2lsbCBiZSBvbmx5IHRoZSBkZXN0aW5hdGlvbiBmb2xkZXIgb2YgdGhlIHBhcnF1ZXQgcGFydGl0aW9uIGZpbGVzCiAgICAiIiIKICAgIHBxd3JpdGVyID0gTm9uZQogICAgaGVhZGVyID0gW10KICAgIGZvciBpLCBkZiBpbiBlbnVtZXJhdGUoCiAgICAgICAgcGQucmVhZF9jc3YoCiAgICAgICAgICAgIGFyY2hpdmVfdXJsLAogICAgICAgICAgICBjaHVua3NpemU9Y2h1bmtzaXplLAogICAgICAgICAgICBuYW1lcz1oZWFkZXIsCiAgICAgICAgICAgIGVuY29kaW5nPWVuY29kaW5nLAogICAgICAgICAgICBkdHlwZT1kdHlwZSwKICAgICAgICApCiAgICApOgogICAgICAgIHRhYmxlID0gcGEuVGFibGUuZnJvbV9wYW5kYXMoZGYpCiAgICAgICAgaWYgaSA9PSAwOgogICAgICAgICAgICBpZiBkYXRhc2V0OgogICAgICAgICAgICAgICAgaGVhZGVyID0gbnAuY29weSh0YWJsZS5zY2hlbWEpCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICBwcXdyaXRlciA9IHBxLlBhcnF1ZXRXcml0ZXIoZGVzdF9wYXRoLCB0YWJsZS5zY2hlbWEpCiAgICAgICAgaWYgZGF0YXNldDoKICAgICAgICAgICAgcHEud3JpdGVfdG9fZGF0YXNldCgKICAgICAgICAgICAgICAgIHRhYmxlLCByb290X3BhdGg9ZGVzdF9wYXRoLCBwYXJ0aXRpb25fY29scz1wYXJ0aXRpb25fY29scwogICAgICAgICAgICApCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcHF3cml0ZXIud3JpdGVfdGFibGUodGFibGUpCiAgICBpZiBwcXdyaXRlcjoKICAgICAgICBwcXdyaXRlci5jbG9zZSgpCgogICAgcmV0dXJuIGhlYWRlcgoKCmRlZiBhcmNfdG9fcGFycXVldCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgYXJjaGl2ZV91cmw6IERhdGFJdGVtLAogICAgaGVhZGVyOiBsaXN0W3N0cl0gPSBbTm9uZV0sCiAgICBjaHVua3NpemU6IGludCA9IDAsCiAgICBkdHlwZT1Ob25lLAogICAgZW5jb2Rpbmc6IHN0ciA9ICJsYXRpbi0xIiwKICAgIGtleTogc3RyID0gImRhdGEiLAogICAgZGF0YXNldDogc3RyID0gIk5vbmUiLAogICAgcGFydF9jb2xzPVtdLAogICAgZmlsZV9leHQ6IHN0ciA9ICJwYXJxdWV0IiwKICAgIGluZGV4OiBib29sID0gRmFsc2UsCiAgICByZWZyZXNoX2RhdGE6IGJvb2wgPSBGYWxzZSwKICAgIHN0YXRzOiBib29sID0gRmFsc2UsCikgLT4gTm9uZToKICAgICIiIk9wZW4gYSBmaWxlL29iamVjdCBhcmNoaXZlIGFuZCBzYXZlIGFzIGEgcGFycXVldCBmaWxlIG9yIGRhdGFzZXQKCiAgICBOb3RlcwogICAgLS0tLS0KICAgICogdGhpcyBmdW5jdGlvbiBpcyB0eXBpY2FsbHkgZm9yIGxhcmdlIGZpbGVzLCBwbGVhc2UgYmUgc3VyZSB0byBjaGVjayBhbGwgc2V0dGluZ3MKICAgICogcGFydGl0aW9uaW5nIHJlcXVpcmVzIHByZWNpc2Ugc3BlY2lmaWNhdGlvbiBvZiBjb2x1bW4gdHlwZXMuCiAgICAqIHRoZSBhcmNoaXZlX3VybCBjYW4gYmUgYW55IGZpbGUgcmVhZGFibGUgYnkgcGFuZGFzIHJlYWRfY3N2LCB3aGljaCBpbmNsdWRlcyB0YXIgZmlsZXMKICAgICogaWYgdGhlIGBkYXRhc2V0YCBwYXJhbWV0ZXIgaXMgbm90IGVtcHR5LCB0aGVuIGEgcGFydGl0aW9uZWQgZGF0YXNldCB3aWxsIGJlIGNyZWF0ZWQKICAgIGluc3RlYWQgb2YgYSBzaW5nbGUgZmlsZSBpbiB0aGUgZm9sZGVyIGBkYXRhc2V0YAogICAgKiBpZiBhIGtleSBleGlzdHMgYWxyZWFkeSB0aGVuIGl0IHdpbGwgbm90IGJlIHJlLWFjcXVpcmVkIHVubGVzcyB0aGUgYHJlZnJlc2hfZGF0YWAgcGFyYW0KICAgIGlzIHNldCB0byBgVHJ1ZWAuICBUaGlzIGlzIGluIGNhc2UgdGhlIG9yaWdpbmFsIGZpbGUgaXMgY29ycnVwdCwgb3IgYSByZWZyZXNoIGlzCiAgICByZXF1aXJlZC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgIHRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gYXJjaGl2ZV91cmw6ICAgIE1MUnVuIGRhdGEgaW5wdXQgKERhdGFJdGVtIG9iamVjdCkKICAgIDpwYXJhbSBjaHVua3NpemU6ICAgICAgKDApIHdoZW4gPiAwLCByb3cgc2l6ZSAoY2h1bmspIHRvIHJldHJpZXZlCiAgICAgICAgICAgICAgICAgICAgICAgICAgIHBlciBpdGVyYXRpb24KICAgIDpwYXJhbSBkdHlwZSAgICAgICAgICAgZGVzdGluYXRpb24gZGF0YSB0eXBlIG9mIHNwZWNpZmllZCBjb2x1bW5zCiAgICA6cGFyYW0gZW5jb2RpbmcgICAgICAgICgibGF0aW4tOCIpIGZpbGUgZW5jb2RpbmcKICAgIDpwYXJhbSBrZXk6ICAgICAgICAgICAga2V5IGluIGFydGlmYWN0IHN0b3JlICh3aGVuIGxvZ19kYXRhPVRydWUpCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgIChOb25lKSBpZiBub3QgTm9uZSB0aGVuICJ0YXJnZXRfcGF0aC9kYXRhc2V0IgogICAgICAgICAgICAgICAgICAgICAgICAgICBpcyBmb2xkZXIgZm9yIHBhcnRpdGlvbmVkIGZpbGVzCiAgICA6cGFyYW0gcGFydF9jb2xzOiAgICAgIChbXSkgbGlzdCBvZiBwYXJ0aXRpb25pbmcgY29sdW1ucwogICAgOnBhcmFtIGZpbGVfZXh0OiAgICAgICAocGFycXVldCkgY3N2L3BhcnF1ZXQgZmlsZSBleHRlbnNpb24KICAgIDpwYXJhbSBpbmRleDogICAgICAgICAgKEZhbHNlKSBwYW5kYXMgc2F2ZSBpbmRleCBvcHRpb24KICAgIDpwYXJhbSByZWZyZXNoX2RhdGE6ICAgKEZhbHNlKSBvdmVyd3JpdGUgZXhpc3RpbmcgZGF0YSBhdCB0aGF0IGxvY2F0aW9uCiAgICA6cGFyYW0gc3RhdHM6ICAgICAgICAgIChOb25lKSBjYWxjdWxhdGUgdGFibGUgc3RhdHMgd2hlbiBsb2dnaW5nIGFydGlmYWN0CiAgICAiIiIKICAgIGJhc2VfcGF0aCA9IGNvbnRleHQuYXJ0aWZhY3RfcGF0aAogICAgb3MubWFrZWRpcnMoYmFzZV9wYXRoLCBleGlzdF9vaz1UcnVlKQoKICAgIGFyY2hpdmVfdXJsID0gYXJjaGl2ZV91cmwubG9jYWwoKQoKICAgIGlmIGRhdGFzZXQgaXMgbm90IE5vbmU6CiAgICAgICAgZGVzdF9wYXRoID0gb3MucGF0aC5qb2luKGJhc2VfcGF0aCwgZGF0YXNldCkKICAgICAgICBleGlzdHMgPSBvcy5wYXRoLmlzZGlyKGRlc3RfcGF0aCkKICAgIGVsc2U6CiAgICAgICAgZGVzdF9wYXRoID0gb3MucGF0aC5qb2luKGJhc2VfcGF0aCwga2V5ICsgZiIue2ZpbGVfZXh0fSIpCiAgICAgICAgZXhpc3RzID0gb3MucGF0aC5pc2ZpbGUoZGVzdF9wYXRoKQoKICAgIGlmIG5vdCBleGlzdHM6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygiZGVzdGluYXRpb24gZmlsZSBkb2VzIG5vdCBleGlzdCwgZG93bmxvYWRpbmciKQogICAgICAgIGlmIGNodW5rc2l6ZSA+IDA6CiAgICAgICAgICAgIGhlYWRlciA9IF9jaHVua19yZWFkd3JpdGUoCiAgICAgICAgICAgICAgICBhcmNoaXZlX3VybCwgZGVzdF9wYXRoLCBjaHVua3NpemUsIGVuY29kaW5nLCBkdHlwZSwgZGF0YXNldAogICAgICAgICAgICApCiAgICAgICAgICAgIGNvbnRleHQubG9nX2RhdGFzZXQoCiAgICAgICAgICAgICAgICBrZXk9a2V5LCBzdGF0cz1zdGF0cywgZm9ybWF0PSJwYXJxdWV0IiwgdGFyZ2V0X3BhdGg9ZGVzdF9wYXRoCiAgICAgICAgICAgICkKICAgICAgICBlbHNlOgogICAgICAgICAgICBkZiA9IHBkLnJlYWRfY3N2KGFyY2hpdmVfdXJsKQogICAgICAgICAgICBjb250ZXh0LmxvZ19kYXRhc2V0KGtleSwgZGY9ZGYsIGZvcm1hdD1maWxlX2V4dCwgaW5kZXg9aW5kZXgpCiAgICBlbHNlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oImRlc3RpbmF0aW9uIGZpbGUgYWxyZWFkeSBleGlzdHMsIG5vdGhpbmcgZG9uZSIpCg== code_origin: '' - command: '' - default_handler: arc_to_parquet - description: retrieve remote archive, open and save as parquet + filename: arc_to_parquet.py entry_points: arc_to_parquet: - name: arc_to_parquet - has_varargs: false + outputs: + - type: None parameters: - name: context type: MLClientCtx @@ -20,7 +25,7 @@ spec: type: DataItem doc: MLRun data input (DataItem object) - name: header - type: List[str] + type: list[str] default: - null - name: chunksize @@ -60,6 +65,7 @@ spec: type: bool doc: (None) calculate table stats when logging artifact default: false + name: arc_to_parquet doc: 'Open a file/object archive and save as a parquet file or dataset @@ -88,13 +94,8 @@ spec: required.' has_kwargs: false - outputs: - - type: None - lineno: 68 - disable_auto_mount: false -metadata: - categories: - - utils - name: arc-to-parquet - tag: '' -verbose: false + has_varargs: false + lineno: 66 + command: '' + description: retrieve remote archive, open and save as parquet + default_handler: arc_to_parquet diff --git a/functions/src/arc_to_parquet/test_arc_to_parquet.py b/functions/src/arc_to_parquet/test_arc_to_parquet.py index f0299f57c..ec990b66a 100644 --- a/functions/src/arc_to_parquet/test_arc_to_parquet.py +++ b/functions/src/arc_to_parquet/test_arc_to_parquet.py @@ -16,28 +16,36 @@ DATA_URL = "https://s3.wasabisys.com/iguazio/data/market-palce/arc_to_parquet/higgs-sample.csv.gz" + def test_run_arc_to_parquet(): - fn = code_to_function(name='test_arc_to_parquet', - filename="arc_to_parquet.py", - handler="arc_to_parquet", - kind="local", - ) - run = fn.run(params={"key": "higgs-sample"}, - handler="arc_to_parquet", - inputs={"archive_url": DATA_URL}, - artifact_path='artifacts', - local=False) - - assert(run.outputs['higgs-sample']) + fn = code_to_function( + name="test_arc_to_parquet", + filename="arc_to_parquet.py", + handler="arc_to_parquet", + kind="local", + ) + run = fn.run( + params={"key": "higgs-sample"}, + handler="arc_to_parquet", + inputs={"archive_url": DATA_URL}, + artifact_path="artifacts", + local=False, + ) + + assert run.outputs["higgs-sample"] + def test_run_local_arc_to_parquet(): import os + os.getcwd() fn = import_function("function.yaml") - run = fn.run(params={"key": "higgs-sample"}, - handler="arc_to_parquet", - inputs={"archive_url": DATA_URL}, - artifact_path=os.getcwd()+'/artifacts', - local=True) - - assert(run.outputs['higgs-sample']) \ No newline at end of file + run = fn.run( + params={"key": "higgs-sample"}, + handler="arc_to_parquet", + inputs={"archive_url": DATA_URL}, + artifact_path=os.getcwd() + "/artifacts", + local=True, + ) + + assert run.outputs["higgs-sample"] diff --git a/functions/src/auto_trainer/auto_trainer.py b/functions/src/auto_trainer/auto_trainer.py index 7b4764700..d9ad2c8e8 100755 --- a/functions/src/auto_trainer/auto_trainer.py +++ b/functions/src/auto_trainer/auto_trainer.py @@ -13,7 +13,7 @@ # limitations under the License. # from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Union import mlrun import mlrun.datastore @@ -23,7 +23,7 @@ from mlrun.datastore import DataItem from mlrun.execution import MLClientCtx from mlrun.frameworks.auto_mlrun import AutoMLRun -from mlrun.utils.helpers import create_class, create_function +from mlrun.utils.helpers import create_class from sklearn.model_selection import train_test_split PathType = Union[str, Path] @@ -35,7 +35,7 @@ class KWArgsPrefixes: TRAIN = "TRAIN_" -def _get_sub_dict_by_prefix(src: Dict, prefix_key: str) -> Dict[str, Any]: +def _get_sub_dict_by_prefix(src: dict, prefix_key: str) -> dict[str, Any]: """ Collect all the keys from the given dict that starts with the given prefix and creates a new dictionary with these keys. @@ -54,9 +54,9 @@ def _get_sub_dict_by_prefix(src: Dict, prefix_key: str) -> Dict[str, Any]: def _get_dataframe( context: MLClientCtx, dataset: DataItem, - label_columns: Optional[Union[str, List[str]]] = None, - drop_columns: Union[str, List[str], int, List[int]] = None, -) -> Tuple[pd.DataFrame, Optional[Union[str, List[str]]]]: + label_columns: str | list[str] | None = None, + drop_columns: str | list[str] | int | list[int] = None, +) -> tuple[pd.DataFrame, str | list[str] | None]: """ Getting the DataFrame of the dataset and drop the columns accordingly. @@ -122,8 +122,8 @@ def train( context: MLClientCtx, dataset: DataItem, model_class: str, - label_columns: Optional[Union[str, List[str]]] = None, - drop_columns: List[str] = None, + label_columns: str | list[str] | None = None, + drop_columns: list[str] = None, model_name: str = "model", tag: str = "", sample_set: DataItem = None, @@ -139,6 +139,7 @@ def train( example:: import mlrun + project = mlrun.get_or_create_project("my-project") project.set_function("hub://auto_trainer", "train") trainer_run = project.run( @@ -210,7 +211,7 @@ def train( # Getting the sample set: if sample_set is None: context.logger.info( - f"Sample set not given, using the whole training set as the sample set" + "Sample set not given, using the whole training set as the sample set" ) sample_set = dataset else: @@ -274,8 +275,8 @@ def evaluate( context: MLClientCtx, model: str, dataset: mlrun.DataItem, - drop_columns: List[str] = None, - label_columns: Optional[Union[str, List[str]]] = None, + drop_columns: list[str] = None, + label_columns: str | list[str] | None = None, **kwargs, ): """ @@ -328,9 +329,9 @@ def predict( context: MLClientCtx, model: str, dataset: mlrun.DataItem, - drop_columns: Union[str, List[str], int, List[int]] = None, - label_columns: Optional[Union[str, List[str]]] = None, - result_set: Optional[str] = None, + drop_columns: str | list[str] | int | list[int] = None, + label_columns: str | list[str] | None = None, + result_set: str | None = None, **kwargs, ): """ diff --git a/functions/src/auto_trainer/function.yaml b/functions/src/auto_trainer/function.yaml index 0920b1033..bb0f13ce8 100644 --- a/functions/src/auto_trainer/function.yaml +++ b/functions/src/auto_trainer/function.yaml @@ -1,22 +1,21 @@ metadata: + tag: '' + name: auto-trainer categories: - machine-learning - model-training - tag: '' - name: auto-trainer +verbose: false +kind: job spec: image: mlrun/mlrun + disable_auto_mount: false build: origin_filename: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICBzdG9yZV91cmlfcHJlZml4LCBfID0gbWxydW4uZGF0YXN0b3JlLnBhcnNlX3N0b3JlX3VyaShkYXRhc2V0LmFydGlmYWN0X3VybCkKCiAgICAjIEdldHRpbmcgdGhlIGRhdGFzZXQ6CiAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IGxhYmVsX2NvbHVtbnMgb3IgZGF0YXNldC5tZXRhLnN0YXR1cy5sYWJlbF9jb2x1bW4KICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYibGFiZWwgY29sdW1uczoge2xhYmVsX2NvbHVtbnN9IikKICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICB0cnk6CiAgICAgICAgICAgIGZ2ID0gbWxydW4uZGF0YXN0b3JlLmdldF9zdG9yZV9yZXNvdXJjZShkYXRhc2V0LmFydGlmYWN0X3VybCkKICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZXhjZXB0IEF0dHJpYnV0ZUVycm9yOgogICAgICAgICAgICAjIExlYXZlIGhlcmUgZm9yIGJhY2t3YXJkcyBjb21wYXRpYmlsaXR5CiAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgIGRhdGFzZXQubWV0YS51cmksIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMKICAgICAgICAgICAgKS50b19kYXRhZnJhbWUoKQoKICAgIGVsaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgImxhYmVsX2NvbHVtbnMgbm90IHByb3ZpZGVkLCBtYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBlbGlmIGlzaW5zdGFuY2UoZGF0YXNldCwgKGxpc3QsIGRpY3QpKToKICAgICAgICAjIGxpc3QvZGljdCBjYXNlOgogICAgICAgIGRhdGFzZXQgPSBwZC5EYXRhRnJhbWUoZGF0YXNldCkKICAgICAgICAjIENoZWNraW5nIGlmIGRyb3BfY29sdW1ucyBwcm92aWRlZCBieSBpbnRlZ2VyIHR5cGU6CiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICBpZiBpc2luc3RhbmNlKGRyb3BfY29sdW1ucywgc3RyKSBvciAoCiAgICAgICAgICAgICAgICBpc2luc3RhbmNlKGRyb3BfY29sdW1ucywgbGlzdCkKICAgICAgICAgICAgICAgIGFuZCBhbnkoaXNpbnN0YW5jZShjb2wsIHN0cikgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpCiAgICAgICAgICAgICk6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIG11c3QgYmUgYW4gaW50ZWdlci9saXN0IG9mIGludGVnZXJzIGlmIG5vdCBwcm92aWRlZCB3aXRoIGEgVVJJL0ZlYXR1cmVWZWN0b3IgZGF0YXNldCIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xLCBpbnBsYWNlPVRydWUpCgogICAgZWxzZToKICAgICAgICAjIHNpbXBsZSBVUkwgY2FzZToKICAgICAgICBkYXRhc2V0ID0gZGF0YXNldC5hc19kZigpCiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSkKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgIm5vdCBhbGwgb2YgdGhlIGNvbHVtbnMgdG8gZHJvcCBpbiB0aGUgZGF0YXNldCwgZHJvcCBjb2x1bW5zIHByb2Nlc3Mgc2tpcHBlZCIKICAgICAgICAgICAgICAgICkKCiAgICByZXR1cm4gZGF0YXNldCwgbGFiZWxfY29sdW1ucwoKCmRlZiB0cmFpbigKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBtb2RlbF9jbGFzczogc3RyLAogICAgbGFiZWxfY29sdW1uczogT3B0aW9uYWxbVW5pb25bc3RyLCBMaXN0W3N0cl1dXSA9IE5vbmUsCiAgICBkcm9wX2NvbHVtbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSAibW9kZWwiLAogICAgdGFnOiBzdHIgPSAiIiwKICAgIHNhbXBsZV9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRlc3Rfc2V0OiBEYXRhSXRlbSA9IE5vbmUsCiAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemU6IGZsb2F0ID0gTm9uZSwKICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gTm9uZSwKICAgIGxhYmVsczogZGljdCA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgVHJhaW5pbmcgYSBtb2RlbCB3aXRoIHRoZSBnaXZlbiBkYXRhc2V0LgoKICAgIGV4YW1wbGU6OgoKICAgICAgICBpbXBvcnQgbWxydW4KICAgICAgICBwcm9qZWN0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9wcm9qZWN0KCJteS1wcm9qZWN0IikKICAgICAgICBwcm9qZWN0LnNldF9mdW5jdGlvbigiaHViOi8vYXV0b190cmFpbmVyIiwgInRyYWluIikKICAgICAgICB0cmFpbmVyX3J1biA9IHByb2plY3QucnVuKAogICAgICAgICAgICBuYW1lPSJ0cmFpbiIsCiAgICAgICAgICAgIGhhbmRsZXI9InRyYWluIiwKICAgICAgICAgICAgaW5wdXRzPXsiZGF0YXNldCI6ICIuL3BhdGgvdG8vZGF0YXNldC5jc3YifSwKICAgICAgICAgICAgcGFyYW1zPXsKICAgICAgICAgICAgICAgICJtb2RlbF9jbGFzcyI6ICJza2xlYXJuLmxpbmVhcl9tb2RlbC5Mb2dpc3RpY1JlZ3Jlc3Npb24iLAogICAgICAgICAgICAgICAgImxhYmVsX2NvbHVtbnMiOiAibGFiZWwiLAogICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyI6ICJpZCIsCiAgICAgICAgICAgICAgICAibW9kZWxfbmFtZSI6ICJteS1tb2RlbCIsCiAgICAgICAgICAgICAgICAidGFnIjogInYxLjAuMCIsCiAgICAgICAgICAgICAgICAic2FtcGxlX3NldCI6ICIuL3BhdGgvdG8vc2FtcGxlX3NldC5jc3YiLAogICAgICAgICAgICAgICAgInRlc3Rfc2V0IjogIi4vcGF0aC90by90ZXN0X3NldC5jc3YiLAogICAgICAgICAgICAgICAgIkNMQVNTX3NvbHZlciI6ICJsaWJsaW5lYXIiLAogICAgICAgICAgICB9LAogICAgICAgICkKCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQKICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3RvcgogICAgOnBhcmFtIG1vZGVsX2NsYXNzOiAgICAgICAgICAgICBUaGUgY2xhc3Mgb2YgdGhlIG1vZGVsLCBlLmcuIGBza2xlYXJuLmxpbmVhcl9tb2RlbC5Mb2dpc3RpY1JlZ3Jlc3Npb25gCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uczogICAgICAgICAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2xhc3NpZmljYXRpb24gdGFza3MuIE1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyIG9yIGEgbGlzdCBvZiBzdHJpbmdzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW5zIHRvIGRyb3AKICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgICAgICAgICAgICAgVGhlIG1vZGVsJ3MgbmFtZSB0byB1c2UgZm9yIHN0b3JpbmcgdGhlIG1vZGVsIGFydGlmYWN0LCBkZWZhdWx0IHRvICdtb2RlbCcKICAgIDpwYXJhbSB0YWc6ICAgICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsJ3MgdGFnIHRvIGxvZyB3aXRoCiAgICA6cGFyYW0gc2FtcGxlX3NldDogICAgICAgICAgICAgIEEgc2FtcGxlIHNldCBvZiBpbnB1dHMgZm9yIHRoZSBtb2RlbCBmb3IgbG9nZ2luZyBpdHMgc3RhdHMgYWxvbmcgdGhlIG1vZGVsIGluIGZhdm91cgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvZiBtb2RlbCBtb25pdG9yaW5nLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3RvcgogICAgOnBhcmFtIHRlc3Rfc2V0OiAgICAgICAgICAgICAgICBUaGUgdGVzdCBzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIHdpdGguCiAgICA6cGFyYW0gdHJhaW5fdGVzdF9zcGxpdF9zaXplOiAgIGlmIHRlc3Rfc2V0IHdhcyBwcm92aWRlZCB0aGVuIHRoaXMgYXJndW1lbnQgaXMgaWdub3JlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgU2hvdWxkIGJlIGJldHdlZW4gMC4wIGFuZCAxLjAgYW5kIHJlcHJlc2VudCB0aGUgcHJvcG9ydGlvbiBvZiB0aGUgZGF0YXNldCB0byBpbmNsdWRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGluIHRoZSB0ZXN0IHNwbGl0LiBUaGUgc2l6ZSBvZiB0aGUgVHJhaW5pbmcgc2V0IGlzIHNldCB0byB0aGUgY29tcGxlbWVudCBvZiB0aGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHZhbHVlLiBEZWZhdWx0ID0gMC4yCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICAgICAgIFJlbGV2YW50IG9ubHkgd2hlbiB1c2luZyB0cmFpbl90ZXN0X3NwbGl0X3NpemUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIEEgcmFuZG9tIHN0YXRlIHNlZWQgdG8gc2h1ZmZsZSB0aGUgZGF0YS4gRm9yIG1vcmUgaW5mb3JtYXRpb24sIHNlZToKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaHR0cHM6Ly9zY2lraXQtbGVhcm4ub3JnL3N0YWJsZS9nbG9zc2FyeS5odG1sI3Rlcm0tcmFuZG9tX3N0YXRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE5vdGljZSB0aGF0IGhlcmUgd2Ugb25seSBwYXNzIGludGVnZXIgdmFsdWVzLgogICAgOnBhcmFtIGxhYmVsczogICAgICAgICAgICAgICAgICBMYWJlbHMgdG8gbG9nIHdpdGggdGhlIG1vZGVsCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHdpdGggcHJlZml4ZXMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoYXQgd2lsbCBiZSBwYXJzZWQgYW5kIHBhc3NlZCB0byB0aGUgcmVsZXZhbnQgZnVuY3Rpb24sIGJ5IHRoZSBmb2xsb3dpbmcgcHJlZml4ZXM6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYENMQVNTX2AgLSBmb3IgdGhlIG1vZGVsIGNsYXNzIGFyZ3VtZW50cwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAtIGBGSVRfYCAtIGZvciB0aGUgYGZpdGAgZnVuY3Rpb24gYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYFRSQUlOX2AgLSBmb3IgdGhlIGB0cmFpbmAgZnVuY3Rpb24gKGluIHhnYiBvciBsZ2JtIHRyYWluIGZ1bmN0aW9uIC0gZnV0dXJlKQoKICAgICIiIgogICAgIyBWYWxpZGF0ZSBpbnB1dHM6CiAgICAjIENoZWNrIGlmIGV4YWN0bHkgb25lIG9mIHRoZW0gaXMgc3VwcGxpZWQ6CiAgICBpZiB0ZXN0X3NldCBpcyBOb25lOgogICAgICAgIGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSBpcyBOb25lOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgInRlc3Rfc2V0IG9yIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSBhcmUgbm90IHByb3ZpZGVkLCBzZXR0aW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB0byAwLjIiCiAgICAgICAgICAgICkKICAgICAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gMC4yCgogICAgZWxpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgInRlc3Rfc2V0IHByb3ZpZGVkLCBpZ25vcmluZyBnaXZlbiB0cmFpbl90ZXN0X3NwbGl0X3NpemUgdmFsdWUiCiAgICAgICAgKQogICAgICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSA9IE5vbmUKCiAgICAjIEdldCBEYXRhRnJhbWUgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIEdldHRpbmcgdGhlIHNhbXBsZSBzZXQ6CiAgICBpZiBzYW1wbGVfc2V0IGlzIE5vbmU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgZiJTYW1wbGUgc2V0IG5vdCBnaXZlbiwgdXNpbmcgdGhlIHdob2xlIHRyYWluaW5nIHNldCBhcyB0aGUgc2FtcGxlIHNldCIKICAgICAgICApCiAgICAgICAgc2FtcGxlX3NldCA9IGRhdGFzZXQKICAgIGVsc2U6CiAgICAgICAgc2FtcGxlX3NldCwgXyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgICAgIGRhdGFzZXQ9c2FtcGxlX3NldCwKICAgICAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgICAgICkKCiAgICAjIFBhcnNpbmcga3dhcmdzOgogICAgIyBUT0RPOiBVc2UgaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24uCiAgICB0cmFpbl9rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeChzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLlRSQUlOKQogICAgZml0X2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuRklUKQogICAgbW9kZWxfY2xhc3Nfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoCiAgICAgICAgc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5NT0RFTF9DTEFTUwogICAgKQoKICAgICMgQ2hlY2sgaWYgbW9kZWwgb3IgZnVuY3Rpb246CiAgICBpZiBoYXNhdHRyKG1vZGVsX2NsYXNzLCAidHJhaW4iKToKICAgICAgICAjIFRPRE86IE5lZWQgdG8gY2FsbDogbW9kZWwoKSwgYWZ0ZXJ3YXJkcyB0byBzdGFydCB0aGUgdHJhaW4gZnVuY3Rpb24uCiAgICAgICAgIyBtb2RlbCA9IGNyZWF0ZV9mdW5jdGlvbihmInttb2RlbF9jbGFzc30udHJhaW4iKQogICAgICAgIHJhaXNlIE5vdEltcGxlbWVudGVkRXJyb3IKICAgIGVsc2U6CiAgICAgICAgIyBDcmVhdGluZyBtb2RlbCBpbnN0YW5jZToKICAgICAgICBtb2RlbCA9IGNyZWF0ZV9jbGFzcyhtb2RlbF9jbGFzcykoKiptb2RlbF9jbGFzc19rd2FyZ3MpCgogICAgeCA9IGRhdGFzZXQuZHJvcChsYWJlbF9jb2x1bW5zLCBheGlzPTEpCiAgICB5ID0gZGF0YXNldFtsYWJlbF9jb2x1bW5zXQogICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplOgogICAgICAgIHhfdHJhaW4sIHhfdGVzdCwgeV90cmFpbiwgeV90ZXN0ID0gdHJhaW5fdGVzdF9zcGxpdCgKICAgICAgICAgICAgeCwgeSwgdGVzdF9zaXplPXRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSwgcmFuZG9tX3N0YXRlPXJhbmRvbV9zdGF0ZQogICAgICAgICkKICAgIGVsc2U6CiAgICAgICAgeF90cmFpbiwgeV90cmFpbiA9IHgsIHkKCiAgICAgICAgdGVzdF9zZXQgPSB0ZXN0X3NldC5hc19kZigpCiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICB0ZXN0X3NldCA9IGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSkKCiAgICAgICAgeF90ZXN0LCB5X3Rlc3QgPSB0ZXN0X3NldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSksIHRlc3Rfc2V0W2xhYmVsX2NvbHVtbnNdCgogICAgQXV0b01MUnVuLmFwcGx5X21scnVuKAogICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgIG1vZGVsX25hbWU9bW9kZWxfbmFtZSwKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgdGFnPXRhZywKICAgICAgICBzYW1wbGVfc2V0PXNhbXBsZV9zZXQsCiAgICAgICAgeV9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgdGVzdF9zZXQ9dGVzdF9zZXQsCiAgICAgICAgeF90ZXN0PXhfdGVzdCwKICAgICAgICB5X3Rlc3Q9eV90ZXN0LAogICAgICAgIGFydGlmYWN0cz1jb250ZXh0LmFydGlmYWN0cywKICAgICAgICBsYWJlbHM9bGFiZWxzLAogICAgKQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInRyYWluaW5nICd7bW9kZWxfbmFtZX0nIikKICAgIG1vZGVsLmZpdCh4X3RyYWluLCB5X3RyYWluLCAqKmZpdF9rd2FyZ3MpCgoKZGVmIGV2YWx1YXRlKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBtb2RlbDogc3RyLAogICAgZGF0YXNldDogbWxydW4uRGF0YUl0ZW0sCiAgICBkcm9wX2NvbHVtbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBFdmFsdWF0aW5nIGEgbW9kZWwuIEFydGlmYWN0cyBnZW5lcmF0ZWQgYnkgdGhlIE1MSGFuZGxlci4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gbW9kZWw6ICAgICAgICAgICAgICAgICAgIFRoZSBtb2RlbCBTdG9yZSBwYXRoLgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byBldmFsdWF0ZSB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcC4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIFBhcnNpbmcgbGFiZWxfY29sdW1uczoKICAgIHBhcnNlZF9sYWJlbF9jb2x1bW5zID0gW10KICAgIGlmIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9ICgKICAgICAgICAgICAgbGFiZWxfY29sdW1ucyBpZiBpc2luc3RhbmNlKGxhYmVsX2NvbHVtbnMsIGxpc3QpIGVsc2UgW2xhYmVsX2NvbHVtbnNdCiAgICAgICAgKQogICAgICAgIGZvciBsYyBpbiBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICBpZiBmcy5jb21tb24uZmVhdHVyZV9zZXBhcmF0b3IgaW4gbGM6CiAgICAgICAgICAgICAgICBmZWF0dXJlX3NldF9uYW1lLCBsYWJlbF9uYW1lLCBhbGlhcyA9IGZzLmNvbW1vbi5wYXJzZV9mZWF0dXJlX3N0cmluZyhsYykKICAgICAgICAgICAgICAgIHBhcnNlZF9sYWJlbF9jb2x1bW5zLmFwcGVuZChhbGlhcyBvciBsYWJlbF9uYW1lKQogICAgICAgIGlmIHBhcnNlZF9sYWJlbF9jb2x1bW5zOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gcGFyc2VkX2xhYmVsX2NvbHVtbnMKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCgogICAgIyBMb2FkaW5nIHRoZSBtb2RlbCBhbmQgcHJlZGljdGluZzoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbCgKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQsIG1vZGVsX25hbWU9Im1vZGVsX0xpbmVhclJlZ3Jlc3Npb24iCiAgICApCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4obW9kZWxfaGFuZGxlci5tb2RlbCwgeV90ZXN0PXksIG1vZGVsX3BhdGg9bW9kZWwpCgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmImV2YWx1YXRpbmcgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoeCwgKiprd2FyZ3MpCgoKZGVmIHByZWRpY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIHJlc3VsdF9zZXQ6IE9wdGlvbmFsW3N0cl0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIFByZWRpY3RpbmcgZGF0YXNldCBieSBhIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIHByZWRpY3QgdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJLCBhIEZlYXR1cmVWZWN0b3Igb3IgYQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgaW4gYSBzaGFwZSBvZiBhIGxpc3QvZGljdC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV2hlbiBwYXNzaW5nIGEgc2FtcGxlLCBwYXNzIHRoZSBkYXRhc2V0IGFzIGEgZmllbGQgaW4gYHBhcmFtc2AgaW5zdGVhZCBvZiBgaW5wdXRzYC4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHRoZSBkYXRhc2V0IGlzIGEgbGlzdC9kaWN0IHRoaXMgcGFyYW1ldGVyIHNob3VsZCBiZSByZXByZXNlbnRlZCBieSBpbnRlZ2Vycy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIHJlc3VsdF9zZXQ6ICAgICAgICAgICAgICBUaGUgZGIga2V5IHRvIHNldCBuYW1lIG9mIHRoZSBwcmVkaWN0aW9uIHJlc3VsdCBhbmQgdGhlIGZpbGVuYW1lLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0IHRvICdwcmVkaWN0aW9uJy4KICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgdG8gdGhlIHByZWRpY3QgZnVuY3Rpb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKFBSRURJQ1RfIHByZWZpeCBpcyBub3QgcmVxdWlyZWQpLgogICAgIiIiCiAgICAjIEdldCBkYXRhc2V0IGJ5IFVSTCBvciBieSBGZWF0dXJlVmVjdG9yOgogICAgZGF0YXNldCwgbGFiZWxfY29sdW1ucyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICBkYXRhc2V0PWRhdGFzZXQsCiAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICApCgogICAgIyBsb2FkaW5nIHRoZSBtb2RlbCwgYW5kIGdldHRpbmcgdGhlIG1vZGVsIGhhbmRsZXI6CiAgICBtb2RlbF9oYW5kbGVyID0gQXV0b01MUnVuLmxvYWRfbW9kZWwobW9kZWxfcGF0aD1tb2RlbCwgY29udGV4dD1jb250ZXh0KQoKICAgICMgRHJvcHBpbmcgbGFiZWwgY29sdW1ucyBpZiBuZWNlc3Nhcnk6CiAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW10KICAgIGVsaWYgaXNpbnN0YW5jZShsYWJlbF9jb2x1bW5zLCBzdHIpOgogICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbbGFiZWxfY29sdW1uc10KCiAgICAjIFByZWRpY3Rpbmc6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYibWFraW5nIHByZWRpY3Rpb24gYnkgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICB5X3ByZWQgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoZGF0YXNldCwgKiprd2FyZ3MpCgogICAgIyBQcmVwYXJpbmcgYW5kIHZhbGlkYXRpbmcgbGFiZWwgY29sdW1ucyBmb3IgdGhlIGRhdGFmcmFtZSBvZiB0aGUgcHJlZGljdGlvbiByZXN1bHQ6CiAgICBudW1fcHJlZGljdGVkID0gMSBpZiBsZW4oeV9wcmVkLnNoYXBlKSA9PSAxIGVsc2UgeV9wcmVkLnNoYXBlWzFdCgogICAgaWYgbnVtX3ByZWRpY3RlZCA+IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBpZiBudW1fcHJlZGljdGVkID09IDE6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbInByZWRpY3RlZCBsYWJlbHMiXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMuZXh0ZW5kKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIGYicHJlZGljdGVkX2xhYmVsX3tpICsgMSArIGxlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgICAgICAgICAgICAgZm9yIGkgaW4gcmFuZ2UobnVtX3ByZWRpY3RlZCAtIGxlbihsYWJlbF9jb2x1bW5zKSkKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgZWxpZiBudW1fcHJlZGljdGVkIDwgbGVuKGxhYmVsX2NvbHVtbnMpOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIm51bWJlciBvZiBwcmVkaWN0ZWQgbGFiZWxzOiB7bnVtX3ByZWRpY3RlZH0gaXMgc21hbGxlciB0aGFuIG51bWJlciBvZiBsYWJlbCBjb2x1bW5zOiB7bGVuKGxhYmVsX2NvbHVtbnMpfSIKICAgICAgICApCiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcgoKICAgIGFydGlmYWN0X25hbWUgPSByZXN1bHRfc2V0IG9yICJwcmVkaWN0aW9uIgogICAgbGFiZWxzX2luc2lkZV9kZiA9IHNldChsYWJlbF9jb2x1bW5zKSAmIHNldChkYXRhc2V0LmNvbHVtbnMudG9saXN0KCkpCiAgICBpZiBsYWJlbHNfaW5zaWRlX2RmOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIlRoZSBsYWJlbHM6IHtsYWJlbHNfaW5zaWRlX2RmfSBhcmUgYWxyZWFkeSBleGlzdGVkIGluIHRoZSBkYXRhZnJhbWUiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgIHByZWRfZGYgPSBwZC5jb25jYXQoW2RhdGFzZXQsIHBkLkRhdGFGcmFtZSh5X3ByZWQsIGNvbHVtbnM9bGFiZWxfY29sdW1ucyldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KGFydGlmYWN0X25hbWUsIHByZWRfZGYsIGRiX2tleT1yZXN1bHRfc2V0KQo= + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIFVuaW9uCgppbXBvcnQgbWxydW4KaW1wb3J0IG1scnVuLmRhdGFzdG9yZQppbXBvcnQgbWxydW4udXRpbHMKaW1wb3J0IHBhbmRhcyBhcyBwZApmcm9tIG1scnVuIGltcG9ydCBmZWF0dXJlX3N0b3JlIGFzIGZzCmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLmF1dG9fbWxydW4gaW1wb3J0IEF1dG9NTFJ1bgpmcm9tIG1scnVuLnV0aWxzLmhlbHBlcnMgaW1wb3J0IGNyZWF0ZV9jbGFzcwpmcm9tIHNrbGVhcm4ubW9kZWxfc2VsZWN0aW9uIGltcG9ydCB0cmFpbl90ZXN0X3NwbGl0CgpQYXRoVHlwZSA9IFVuaW9uW3N0ciwgUGF0aF0KCgpjbGFzcyBLV0FyZ3NQcmVmaXhlczoKICAgIE1PREVMX0NMQVNTID0gIkNMQVNTXyIKICAgIEZJVCA9ICJGSVRfIgogICAgVFJBSU4gPSAiVFJBSU5fIgoKCmRlZiBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeChzcmM6IGRpY3QsIHByZWZpeF9rZXk6IHN0cikgLT4gZGljdFtzdHIsIEFueV06CiAgICAiIiIKICAgIENvbGxlY3QgYWxsIHRoZSBrZXlzIGZyb20gdGhlIGdpdmVuIGRpY3QgdGhhdCBzdGFydHMgd2l0aCB0aGUgZ2l2ZW4gcHJlZml4IGFuZCBjcmVhdGVzIGEgbmV3IGRpY3Rpb25hcnkgd2l0aCB0aGVzZQogICAga2V5cy4KCiAgICA6cGFyYW0gc3JjOiAgICAgICAgIFRoZSBzb3VyY2UgZGljdCB0byBleHRyYWN0IHRoZSB2YWx1ZXMgZnJvbS4KICAgIDpwYXJhbSBwcmVmaXhfa2V5OiAgT25seSBrZXlzIHdpdGggdGhpcyBwcmVmaXggd2lsbCBiZSByZXR1cm5lZC4gVGhlIGtleXMgaW4gdGhlIHJlc3VsdCBkaWN0IHdpbGwgYmUgd2l0aG91dCB0aGlzCiAgICAgICAgICAgICAgICAgICAgICAgIHByZWZpeC4KICAgICIiIgogICAgcmV0dXJuIHsKICAgICAgICBrZXkucmVwbGFjZShwcmVmaXhfa2V5LCAiIik6IHZhbAogICAgICAgIGZvciBrZXksIHZhbCBpbiBzcmMuaXRlbXMoKQogICAgICAgIGlmIGtleS5zdGFydHN3aXRoKHByZWZpeF9rZXkpCiAgICB9CgoKZGVmIF9nZXRfZGF0YWZyYW1lKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIGxhYmVsX2NvbHVtbnM6IHN0ciB8IGxpc3Rbc3RyXSB8IE5vbmUgPSBOb25lLAogICAgZHJvcF9jb2x1bW5zOiBzdHIgfCBsaXN0W3N0cl0gfCBpbnQgfCBsaXN0W2ludF0gPSBOb25lLAopIC0+IHR1cGxlW3BkLkRhdGFGcmFtZSwgc3RyIHwgbGlzdFtzdHJdIHwgTm9uZV06CiAgICAiIiIKICAgIEdldHRpbmcgdGhlIERhdGFGcmFtZSBvZiB0aGUgZGF0YXNldCBhbmQgZHJvcCB0aGUgY29sdW1ucyBhY2NvcmRpbmdseS4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgVGhlIGRhdGFzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIG9uLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2FuIGJlIGVpdGhlciBhIGxpc3Qgb2YgbGlzdHMsIGRpY3QsIFVSSSBvciBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uczogICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgIHN0ci9pbnQgb3IgYSBsaXN0IG9mIHN0cmluZ3MvaW50cyB0aGF0IHJlcHJlc2VudCB0aGUgY29sdW1uIG5hbWVzL2luZGljZXMgdG8gZHJvcC4KICAgICIiIgogICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGF0YXNldC5hcnRpZmFjdF91cmwpCgogICAgIyBHZXR0aW5nIHRoZSBkYXRhc2V0OgogICAgaWYgbWxydW4udXRpbHMuU3RvcmVQcmVmaXguRmVhdHVyZVZlY3RvciA9PSBzdG9yZV91cmlfcHJlZml4OgogICAgICAgIGxhYmVsX2NvbHVtbnMgPSBsYWJlbF9jb2x1bW5zIG9yIGRhdGFzZXQubWV0YS5zdGF0dXMubGFiZWxfY29sdW1uCiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbyhmImxhYmVsIGNvbHVtbnM6IHtsYWJlbF9jb2x1bW5zfSIpCiAgICAgICAgIyBGZWF0dXJlVmVjdG9yIGNhc2U6CiAgICAgICAgdHJ5OgogICAgICAgICAgICBmdiA9IG1scnVuLmRhdGFzdG9yZS5nZXRfc3RvcmVfcmVzb3VyY2UoZGF0YXNldC5hcnRpZmFjdF91cmwpCiAgICAgICAgICAgIGRhdGFzZXQgPSBmdi5nZXRfb2ZmbGluZV9mZWF0dXJlcyhkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zKS50b19kYXRhZnJhbWUoKQogICAgICAgIGV4Y2VwdCBBdHRyaWJ1dGVFcnJvcjoKICAgICAgICAgICAgIyBMZWF2ZSBoZXJlIGZvciBiYWNrd2FyZHMgY29tcGF0aWJpbGl0eQogICAgICAgICAgICBkYXRhc2V0ID0gZnMuZ2V0X29mZmxpbmVfZmVhdHVyZXMoCiAgICAgICAgICAgICAgICBkYXRhc2V0Lm1ldGEudXJpLCBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zCiAgICAgICAgICAgICkudG9fZGF0YWZyYW1lKCkKCiAgICBlbGlmIG5vdCBsYWJlbF9jb2x1bW5zOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICJsYWJlbF9jb2x1bW5zIG5vdCBwcm92aWRlZCwgbWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yIgogICAgICAgICkKICAgICAgICByYWlzZSBWYWx1ZUVycm9yCgogICAgZWxpZiBpc2luc3RhbmNlKGRhdGFzZXQsIChsaXN0LCBkaWN0KSk6CiAgICAgICAgIyBsaXN0L2RpY3QgY2FzZToKICAgICAgICBkYXRhc2V0ID0gcGQuRGF0YUZyYW1lKGRhdGFzZXQpCiAgICAgICAgIyBDaGVja2luZyBpZiBkcm9wX2NvbHVtbnMgcHJvdmlkZWQgYnkgaW50ZWdlciB0eXBlOgogICAgICAgIGlmIGRyb3BfY29sdW1uczoKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIHN0cikgb3IgKAogICAgICAgICAgICAgICAgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIGxpc3QpCiAgICAgICAgICAgICAgICBhbmQgYW55KGlzaW5zdGFuY2UoY29sLCBzdHIpIGZvciBjb2wgaW4gZHJvcF9jb2x1bW5zKQogICAgICAgICAgICApOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyBtdXN0IGJlIGFuIGludGVnZXIvbGlzdCBvZiBpbnRlZ2VycyBpZiBub3QgcHJvdmlkZWQgd2l0aCBhIFVSSS9GZWF0dXJlVmVjdG9yIGRhdGFzZXQiCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICAgICAgICAgIGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSwgaW5wbGFjZT1UcnVlKQoKICAgIGVsc2U6CiAgICAgICAgIyBzaW1wbGUgVVJMIGNhc2U6CiAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuYXNfZGYoKQogICAgICAgIGlmIGRyb3BfY29sdW1uczoKICAgICAgICAgICAgaWYgYWxsKGNvbCBpbiBkYXRhc2V0IGZvciBjb2wgaW4gZHJvcF9jb2x1bW5zKToKICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBkYXRhc2V0LmRyb3AoZHJvcF9jb2x1bW5zLCBheGlzPTEpCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgICAgICJub3QgYWxsIG9mIHRoZSBjb2x1bW5zIHRvIGRyb3AgaW4gdGhlIGRhdGFzZXQsIGRyb3AgY29sdW1ucyBwcm9jZXNzIHNraXBwZWQiCiAgICAgICAgICAgICAgICApCgogICAgcmV0dXJuIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMKCgpkZWYgdHJhaW4oCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIGRhdGFzZXQ6IERhdGFJdGVtLAogICAgbW9kZWxfY2xhc3M6IHN0ciwKICAgIGxhYmVsX2NvbHVtbnM6IHN0ciB8IGxpc3Rbc3RyXSB8IE5vbmUgPSBOb25lLAogICAgZHJvcF9jb2x1bW5zOiBsaXN0W3N0cl0gPSBOb25lLAogICAgbW9kZWxfbmFtZTogc3RyID0gIm1vZGVsIiwKICAgIHRhZzogc3RyID0gIiIsCiAgICBzYW1wbGVfc2V0OiBEYXRhSXRlbSA9IE5vbmUsCiAgICB0ZXN0X3NldDogRGF0YUl0ZW0gPSBOb25lLAogICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplOiBmbG9hdCA9IE5vbmUsCiAgICByYW5kb21fc3RhdGU6IGludCA9IE5vbmUsCiAgICBsYWJlbHM6IGRpY3QgPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIFRyYWluaW5nIGEgbW9kZWwgd2l0aCB0aGUgZ2l2ZW4gZGF0YXNldC4KCiAgICBleGFtcGxlOjoKCiAgICAgICAgaW1wb3J0IG1scnVuCgogICAgICAgIHByb2plY3QgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX3Byb2plY3QoIm15LXByb2plY3QiKQogICAgICAgIHByb2plY3Quc2V0X2Z1bmN0aW9uKCJodWI6Ly9hdXRvX3RyYWluZXIiLCAidHJhaW4iKQogICAgICAgIHRyYWluZXJfcnVuID0gcHJvamVjdC5ydW4oCiAgICAgICAgICAgIG5hbWU9InRyYWluIiwKICAgICAgICAgICAgaGFuZGxlcj0idHJhaW4iLAogICAgICAgICAgICBpbnB1dHM9eyJkYXRhc2V0IjogIi4vcGF0aC90by9kYXRhc2V0LmNzdiJ9LAogICAgICAgICAgICBwYXJhbXM9ewogICAgICAgICAgICAgICAgIm1vZGVsX2NsYXNzIjogInNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyI6ICJsYWJlbCIsCiAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIjogImlkIiwKICAgICAgICAgICAgICAgICJtb2RlbF9uYW1lIjogIm15LW1vZGVsIiwKICAgICAgICAgICAgICAgICJ0YWciOiAidjEuMC4wIiwKICAgICAgICAgICAgICAgICJzYW1wbGVfc2V0IjogIi4vcGF0aC90by9zYW1wbGVfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAidGVzdF9zZXQiOiAiLi9wYXRoL3RvL3Rlc3Rfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAiQ0xBU1Nfc29sdmVyIjogImxpYmxpbmVhciIsCiAgICAgICAgICAgIH0sCiAgICAgICAgKQoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dAogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICAgICAgICAgIFRoZSBjbGFzcyBvZiB0aGUgbW9kZWwsIGUuZy4gYHNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbmAKICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcAogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICAgICBUaGUgbW9kZWwncyBuYW1lIHRvIHVzZSBmb3Igc3RvcmluZyB0aGUgbW9kZWwgYXJ0aWZhY3QsIGRlZmF1bHQgdG8gJ21vZGVsJwogICAgOnBhcmFtIHRhZzogICAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwncyB0YWcgdG8gbG9nIHdpdGgKICAgIDpwYXJhbSBzYW1wbGVfc2V0OiAgICAgICAgICAgICAgQSBzYW1wbGUgc2V0IG9mIGlucHV0cyBmb3IgdGhlIG1vZGVsIGZvciBsb2dnaW5nIGl0cyBzdGF0cyBhbG9uZyB0aGUgbW9kZWwgaW4gZmF2b3VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9mIG1vZGVsIG1vbml0b3JpbmcuIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gdGVzdF9zZXQ6ICAgICAgICAgICAgICAgIFRoZSB0ZXN0IHNldCB0byB0cmFpbiB0aGUgbW9kZWwgd2l0aC4KICAgIDpwYXJhbSB0cmFpbl90ZXN0X3NwbGl0X3NpemU6ICAgaWYgdGVzdF9zZXQgd2FzIHByb3ZpZGVkIHRoZW4gdGhpcyBhcmd1bWVudCBpcyBpZ25vcmVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTaG91bGQgYmUgYmV0d2VlbiAwLjAgYW5kIDEuMCBhbmQgcmVwcmVzZW50IHRoZSBwcm9wb3J0aW9uIG9mIHRoZSBkYXRhc2V0IHRvIGluY2x1ZGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaW4gdGhlIHRlc3Qgc3BsaXQuIFRoZSBzaXplIG9mIHRoZSBUcmFpbmluZyBzZXQgaXMgc2V0IHRvIHRoZSBjb21wbGVtZW50IG9mIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWUuIERlZmF1bHQgPSAwLjIKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgUmVsZXZhbnQgb25seSB3aGVuIHVzaW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQSByYW5kb20gc3RhdGUgc2VlZCB0byBzaHVmZmxlIHRoZSBkYXRhLiBGb3IgbW9yZSBpbmZvcm1hdGlvbiwgc2VlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL3NjaWtpdC1sZWFybi5vcmcvc3RhYmxlL2dsb3NzYXJ5Lmh0bWwjdGVybS1yYW5kb21fc3RhdGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTm90aWNlIHRoYXQgaGVyZSB3ZSBvbmx5IHBhc3MgaW50ZWdlciB2YWx1ZXMuCiAgICA6cGFyYW0gbGFiZWxzOiAgICAgICAgICAgICAgICAgIExhYmVscyB0byBsb2cgd2l0aCB0aGUgbW9kZWwKICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgd2l0aCBwcmVmaXhlcywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCB3aWxsIGJlIHBhcnNlZCBhbmQgcGFzc2VkIHRvIHRoZSByZWxldmFudCBmdW5jdGlvbiwgYnkgdGhlIGZvbGxvd2luZyBwcmVmaXhlczoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgQ0xBU1NfYCAtIGZvciB0aGUgbW9kZWwgY2xhc3MgYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYEZJVF9gIC0gZm9yIHRoZSBgZml0YCBmdW5jdGlvbiBhcmd1bWVudHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgVFJBSU5fYCAtIGZvciB0aGUgYHRyYWluYCBmdW5jdGlvbiAoaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24gLSBmdXR1cmUpCgogICAgIiIiCiAgICAjIFZhbGlkYXRlIGlucHV0czoKICAgICMgQ2hlY2sgaWYgZXhhY3RseSBvbmUgb2YgdGhlbSBpcyBzdXBwbGllZDoKICAgIGlmIHRlc3Rfc2V0IGlzIE5vbmU6CiAgICAgICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGlzIE5vbmU6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAidGVzdF9zZXQgb3IgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGFyZSBub3QgcHJvdmlkZWQsIHNldHRpbmcgdHJhaW5fdGVzdF9zcGxpdF9zaXplIHRvIDAuMiIKICAgICAgICAgICAgKQogICAgICAgICAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemUgPSAwLjIKCiAgICBlbGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAidGVzdF9zZXQgcHJvdmlkZWQsIGlnbm9yaW5nIGdpdmVuIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB2YWx1ZSIKICAgICAgICApCiAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gTm9uZQoKICAgICMgR2V0IERhdGFGcmFtZSBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgR2V0dGluZyB0aGUgc2FtcGxlIHNldDoKICAgIGlmIHNhbXBsZV9zZXQgaXMgTm9uZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAiU2FtcGxlIHNldCBub3QgZ2l2ZW4sIHVzaW5nIHRoZSB3aG9sZSB0cmFpbmluZyBzZXQgYXMgdGhlIHNhbXBsZSBzZXQiCiAgICAgICAgKQogICAgICAgIHNhbXBsZV9zZXQgPSBkYXRhc2V0CiAgICBlbHNlOgogICAgICAgIHNhbXBsZV9zZXQsIF8gPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgICAgICBkYXRhc2V0PXNhbXBsZV9zZXQsCiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICAgICApCgogICAgIyBQYXJzaW5nIGt3YXJnczoKICAgICMgVE9ETzogVXNlIGluIHhnYiBvciBsZ2JtIHRyYWluIGZ1bmN0aW9uLgogICAgdHJhaW5fa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5UUkFJTikKICAgIGZpdF9rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeChzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLkZJVCkKICAgIG1vZGVsX2NsYXNzX2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KAogICAgICAgIHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuTU9ERUxfQ0xBU1MKICAgICkKCiAgICAjIENoZWNrIGlmIG1vZGVsIG9yIGZ1bmN0aW9uOgogICAgaWYgaGFzYXR0cihtb2RlbF9jbGFzcywgInRyYWluIik6CiAgICAgICAgIyBUT0RPOiBOZWVkIHRvIGNhbGw6IG1vZGVsKCksIGFmdGVyd2FyZHMgdG8gc3RhcnQgdGhlIHRyYWluIGZ1bmN0aW9uLgogICAgICAgICMgbW9kZWwgPSBjcmVhdGVfZnVuY3Rpb24oZiJ7bW9kZWxfY2xhc3N9LnRyYWluIikKICAgICAgICByYWlzZSBOb3RJbXBsZW1lbnRlZEVycm9yCiAgICBlbHNlOgogICAgICAgICMgQ3JlYXRpbmcgbW9kZWwgaW5zdGFuY2U6CiAgICAgICAgbW9kZWwgPSBjcmVhdGVfY2xhc3MobW9kZWxfY2xhc3MpKCoqbW9kZWxfY2xhc3Nfa3dhcmdzKQoKICAgIHggPSBkYXRhc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKQogICAgeSA9IGRhdGFzZXRbbGFiZWxfY29sdW1uc10KICAgIGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZToKICAgICAgICB4X3RyYWluLCB4X3Rlc3QsIHlfdHJhaW4sIHlfdGVzdCA9IHRyYWluX3Rlc3Rfc3BsaXQoCiAgICAgICAgICAgIHgsIHksIHRlc3Rfc2l6ZT10cmFpbl90ZXN0X3NwbGl0X3NpemUsIHJhbmRvbV9zdGF0ZT1yYW5kb21fc3RhdGUKICAgICAgICApCiAgICBlbHNlOgogICAgICAgIHhfdHJhaW4sIHlfdHJhaW4gPSB4LCB5CgogICAgICAgIHRlc3Rfc2V0ID0gdGVzdF9zZXQuYXNfZGYoKQogICAgICAgIGlmIGRyb3BfY29sdW1uczoKICAgICAgICAgICAgdGVzdF9zZXQgPSBkYXRhc2V0LmRyb3AoZHJvcF9jb2x1bW5zLCBheGlzPTEpCgogICAgICAgIHhfdGVzdCwgeV90ZXN0ID0gdGVzdF9zZXQuZHJvcChsYWJlbF9jb2x1bW5zLCBheGlzPTEpLCB0ZXN0X3NldFtsYWJlbF9jb2x1bW5zXQoKICAgIEF1dG9NTFJ1bi5hcHBseV9tbHJ1bigKICAgICAgICBtb2RlbD1tb2RlbCwKICAgICAgICBtb2RlbF9uYW1lPW1vZGVsX25hbWUsCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIHRhZz10YWcsCiAgICAgICAgc2FtcGxlX3NldD1zYW1wbGVfc2V0LAogICAgICAgIHlfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgIHRlc3Rfc2V0PXRlc3Rfc2V0LAogICAgICAgIHhfdGVzdD14X3Rlc3QsCiAgICAgICAgeV90ZXN0PXlfdGVzdCwKICAgICAgICBhcnRpZmFjdHM9Y29udGV4dC5hcnRpZmFjdHMsCiAgICAgICAgbGFiZWxzPWxhYmVscywKICAgICkKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJ0cmFpbmluZyAne21vZGVsX25hbWV9JyIpCiAgICBtb2RlbC5maXQoeF90cmFpbiwgeV90cmFpbiwgKipmaXRfa3dhcmdzKQoKCmRlZiBldmFsdWF0ZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWw6IHN0ciwKICAgIGRhdGFzZXQ6IG1scnVuLkRhdGFJdGVtLAogICAgZHJvcF9jb2x1bW5zOiBsaXN0W3N0cl0gPSBOb25lLAogICAgbGFiZWxfY29sdW1uczogc3RyIHwgbGlzdFtzdHJdIHwgTm9uZSA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgRXZhbHVhdGluZyBhIG1vZGVsLiBBcnRpZmFjdHMgZ2VuZXJhdGVkIGJ5IHRoZSBNTEhhbmRsZXIuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwgU3RvcmUgcGF0aC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gZXZhbHVhdGUgdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyIG9yIGEgbGlzdCBvZiBzdHJpbmdzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW5zIHRvIGRyb3AuCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uczogICAgICAgICAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2xhc3NpZmljYXRpb24gdGFza3MuIE1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgdG8gdGhlIHByZWRpY3QgZnVuY3Rpb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKFBSRURJQ1RfIHByZWZpeCBpcyBub3QgcmVxdWlyZWQpLgogICAgIiIiCiAgICAjIEdldCBkYXRhc2V0IGJ5IFVSTCBvciBieSBGZWF0dXJlVmVjdG9yOgogICAgZGF0YXNldCwgbGFiZWxfY29sdW1ucyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICBkYXRhc2V0PWRhdGFzZXQsCiAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICApCgogICAgIyBQYXJzaW5nIGxhYmVsX2NvbHVtbnM6CiAgICBwYXJzZWRfbGFiZWxfY29sdW1ucyA9IFtdCiAgICBpZiBsYWJlbF9jb2x1bW5zOgogICAgICAgIGxhYmVsX2NvbHVtbnMgPSAoCiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgaWYgaXNpbnN0YW5jZShsYWJlbF9jb2x1bW5zLCBsaXN0KSBlbHNlIFtsYWJlbF9jb2x1bW5zXQogICAgICAgICkKICAgICAgICBmb3IgbGMgaW4gbGFiZWxfY29sdW1uczoKICAgICAgICAgICAgaWYgZnMuY29tbW9uLmZlYXR1cmVfc2VwYXJhdG9yIGluIGxjOgogICAgICAgICAgICAgICAgZmVhdHVyZV9zZXRfbmFtZSwgbGFiZWxfbmFtZSwgYWxpYXMgPSBmcy5jb21tb24ucGFyc2VfZmVhdHVyZV9zdHJpbmcobGMpCiAgICAgICAgICAgICAgICBwYXJzZWRfbGFiZWxfY29sdW1ucy5hcHBlbmQoYWxpYXMgb3IgbGFiZWxfbmFtZSkKICAgICAgICBpZiBwYXJzZWRfbGFiZWxfY29sdW1uczoKICAgICAgICAgICAgbGFiZWxfY29sdW1ucyA9IHBhcnNlZF9sYWJlbF9jb2x1bW5zCgogICAgeCA9IGRhdGFzZXQuZHJvcChsYWJlbF9jb2x1bW5zLCBheGlzPTEpCiAgICB5ID0gZGF0YXNldFtsYWJlbF9jb2x1bW5zXQoKICAgICMgTG9hZGluZyB0aGUgbW9kZWwgYW5kIHByZWRpY3Rpbmc6CiAgICBtb2RlbF9oYW5kbGVyID0gQXV0b01MUnVuLmxvYWRfbW9kZWwoCiAgICAgICAgbW9kZWxfcGF0aD1tb2RlbCwgY29udGV4dD1jb250ZXh0LCBtb2RlbF9uYW1lPSJtb2RlbF9MaW5lYXJSZWdyZXNzaW9uIgogICAgKQogICAgQXV0b01MUnVuLmFwcGx5X21scnVuKG1vZGVsX2hhbmRsZXIubW9kZWwsIHlfdGVzdD15LCBtb2RlbF9wYXRoPW1vZGVsKQoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJldmFsdWF0aW5nICd7bW9kZWxfaGFuZGxlci5tb2RlbF9uYW1lfSciKQogICAgbW9kZWxfaGFuZGxlci5tb2RlbC5wcmVkaWN0KHgsICoqa3dhcmdzKQoKCmRlZiBwcmVkaWN0KAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBtb2RlbDogc3RyLAogICAgZGF0YXNldDogbWxydW4uRGF0YUl0ZW0sCiAgICBkcm9wX2NvbHVtbnM6IHN0ciB8IGxpc3Rbc3RyXSB8IGludCB8IGxpc3RbaW50XSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBzdHIgfCBsaXN0W3N0cl0gfCBOb25lID0gTm9uZSwKICAgIHJlc3VsdF9zZXQ6IHN0ciB8IE5vbmUgPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIFByZWRpY3RpbmcgZGF0YXNldCBieSBhIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIHByZWRpY3QgdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJLCBhIEZlYXR1cmVWZWN0b3Igb3IgYQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgaW4gYSBzaGFwZSBvZiBhIGxpc3QvZGljdC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV2hlbiBwYXNzaW5nIGEgc2FtcGxlLCBwYXNzIHRoZSBkYXRhc2V0IGFzIGEgZmllbGQgaW4gYHBhcmFtc2AgaW5zdGVhZCBvZiBgaW5wdXRzYC4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHRoZSBkYXRhc2V0IGlzIGEgbGlzdC9kaWN0IHRoaXMgcGFyYW1ldGVyIHNob3VsZCBiZSByZXByZXNlbnRlZCBieSBpbnRlZ2Vycy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIHJlc3VsdF9zZXQ6ICAgICAgICAgICAgICBUaGUgZGIga2V5IHRvIHNldCBuYW1lIG9mIHRoZSBwcmVkaWN0aW9uIHJlc3VsdCBhbmQgdGhlIGZpbGVuYW1lLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0IHRvICdwcmVkaWN0aW9uJy4KICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgdG8gdGhlIHByZWRpY3QgZnVuY3Rpb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKFBSRURJQ1RfIHByZWZpeCBpcyBub3QgcmVxdWlyZWQpLgogICAgIiIiCiAgICAjIEdldCBkYXRhc2V0IGJ5IFVSTCBvciBieSBGZWF0dXJlVmVjdG9yOgogICAgZGF0YXNldCwgbGFiZWxfY29sdW1ucyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICBkYXRhc2V0PWRhdGFzZXQsCiAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICApCgogICAgIyBsb2FkaW5nIHRoZSBtb2RlbCwgYW5kIGdldHRpbmcgdGhlIG1vZGVsIGhhbmRsZXI6CiAgICBtb2RlbF9oYW5kbGVyID0gQXV0b01MUnVuLmxvYWRfbW9kZWwobW9kZWxfcGF0aD1tb2RlbCwgY29udGV4dD1jb250ZXh0KQoKICAgICMgRHJvcHBpbmcgbGFiZWwgY29sdW1ucyBpZiBuZWNlc3Nhcnk6CiAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW10KICAgIGVsaWYgaXNpbnN0YW5jZShsYWJlbF9jb2x1bW5zLCBzdHIpOgogICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbbGFiZWxfY29sdW1uc10KCiAgICAjIFByZWRpY3Rpbmc6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYibWFraW5nIHByZWRpY3Rpb24gYnkgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICB5X3ByZWQgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoZGF0YXNldCwgKiprd2FyZ3MpCgogICAgIyBQcmVwYXJpbmcgYW5kIHZhbGlkYXRpbmcgbGFiZWwgY29sdW1ucyBmb3IgdGhlIGRhdGFmcmFtZSBvZiB0aGUgcHJlZGljdGlvbiByZXN1bHQ6CiAgICBudW1fcHJlZGljdGVkID0gMSBpZiBsZW4oeV9wcmVkLnNoYXBlKSA9PSAxIGVsc2UgeV9wcmVkLnNoYXBlWzFdCgogICAgaWYgbnVtX3ByZWRpY3RlZCA+IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBpZiBudW1fcHJlZGljdGVkID09IDE6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbInByZWRpY3RlZCBsYWJlbHMiXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMuZXh0ZW5kKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIGYicHJlZGljdGVkX2xhYmVsX3tpICsgMSArIGxlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgICAgICAgICAgICAgZm9yIGkgaW4gcmFuZ2UobnVtX3ByZWRpY3RlZCAtIGxlbihsYWJlbF9jb2x1bW5zKSkKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgZWxpZiBudW1fcHJlZGljdGVkIDwgbGVuKGxhYmVsX2NvbHVtbnMpOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIm51bWJlciBvZiBwcmVkaWN0ZWQgbGFiZWxzOiB7bnVtX3ByZWRpY3RlZH0gaXMgc21hbGxlciB0aGFuIG51bWJlciBvZiBsYWJlbCBjb2x1bW5zOiB7bGVuKGxhYmVsX2NvbHVtbnMpfSIKICAgICAgICApCiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcgoKICAgIGFydGlmYWN0X25hbWUgPSByZXN1bHRfc2V0IG9yICJwcmVkaWN0aW9uIgogICAgbGFiZWxzX2luc2lkZV9kZiA9IHNldChsYWJlbF9jb2x1bW5zKSAmIHNldChkYXRhc2V0LmNvbHVtbnMudG9saXN0KCkpCiAgICBpZiBsYWJlbHNfaW5zaWRlX2RmOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIlRoZSBsYWJlbHM6IHtsYWJlbHNfaW5zaWRlX2RmfSBhcmUgYWxyZWFkeSBleGlzdGVkIGluIHRoZSBkYXRhZnJhbWUiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgIHByZWRfZGYgPSBwZC5jb25jYXQoW2RhdGFzZXQsIHBkLkRhdGFGcmFtZSh5X3ByZWQsIGNvbHVtbnM9bGFiZWxfY29sdW1ucyldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KGFydGlmYWN0X25hbWUsIHByZWRfZGYsIGRiX2tleT1yZXN1bHRfc2V0KQo= code_origin: '' - description: Automatic train, evaluate and predict functions for the ML frameworks - - Scikit-Learn, XGBoost and LightGBM. - disable_auto_mount: false - default_handler: train + filename: auto_trainer.py entry_points: train: - lineno: 121 parameters: - name: context type: MLClientCtx @@ -28,12 +27,11 @@ spec: type: str doc: The class of the model, e.g. `sklearn.linear_model.LogisticRegression` - name: label_columns - type: Optional[Union[str, List[str]]] doc: The target label(s) of the column(s) in the dataset. for Regression or Classification tasks. Mandatory when dataset is not a FeatureVector. default: null - name: drop_columns - type: List[str] + type: list[str] doc: str or a list of strings that represent the columns to drop default: null - name: model_name @@ -70,11 +68,9 @@ spec: type: dict doc: Labels to log with the model default: null - has_varargs: false name: train - has_kwargs: true doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ - \ project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ + \n project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ hub://auto_trainer\", \"train\")\n trainer_run = project.run(\n \ \ name=\"train\",\n handler=\"train\",\n inputs={\"dataset\"\ : \"./path/to/dataset.csv\"},\n params={\n \"model_class\"\ @@ -83,8 +79,10 @@ spec: : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" + has_kwargs: true + has_varargs: false + lineno: 121 evaluate: - lineno: 273 parameters: - name: context type: MLClientCtx @@ -96,20 +94,19 @@ spec: type: DataItem doc: The dataset to evaluate the model on. Can be either a URI or a FeatureVector. - name: drop_columns - type: List[str] + type: list[str] doc: str or a list of strings that represent the columns to drop. default: null - name: label_columns - type: Optional[Union[str, List[str]]] doc: The target label(s) of the column(s) in the dataset. for Regression or Classification tasks. Mandatory when dataset is not a FeatureVector. default: null - has_varargs: false name: evaluate - has_kwargs: true doc: Evaluating a model. Artifacts generated by the MLHandler. + has_kwargs: true + has_varargs: false + lineno: 274 predict: - lineno: 327 parameters: - name: context type: MLClientCtx @@ -123,25 +120,24 @@ spec: or a sample in a shape of a list/dict. When passing a sample, pass the dataset as a field in `params` instead of `inputs`. - name: drop_columns - type: Union[str, List[str], int, List[int]] doc: str/int or a list of strings/ints that represent the column names/indices to drop. When the dataset is a list/dict this parameter should be represented by integers. default: null - name: label_columns - type: Optional[Union[str, List[str]]] doc: The target label(s) of the column(s) in the dataset. for Regression or Classification tasks. Mandatory when dataset is not a FeatureVector. default: null - name: result_set - type: Optional[str] doc: The db key to set name of the prediction result and the filename. Default to 'prediction'. default: null - has_varargs: false name: predict - has_kwargs: true doc: Predicting dataset by a model. + has_kwargs: true + has_varargs: false + lineno: 328 command: '' -kind: job -verbose: false + description: Automatic train, evaluate and predict functions for the ML frameworks + - Scikit-Learn, XGBoost and LightGBM. + default_handler: train diff --git a/functions/src/auto_trainer/test_auto_trainer.py b/functions/src/auto_trainer/test_auto_trainer.py index 9a1ff554c..49eb4101b 100644 --- a/functions/src/auto_trainer/test_auto_trainer.py +++ b/functions/src/auto_trainer/test_auto_trainer.py @@ -14,7 +14,6 @@ # import os import tempfile -from typing import Tuple import mlrun import pandas as pd @@ -78,7 +77,7 @@ def _assert_train_handler(train_run): @pytest.mark.parametrize("model", MODELS) -def test_train(model: Tuple[str, str]): +def test_train(model: tuple[str, str]): dataset, label_columns = _get_dataset(model[1]) is_test_passed = True @@ -115,7 +114,7 @@ def test_train(model: Tuple[str, str]): condition=not _validate_environment_variables(), reason="Project's environment variables are not set", ) -def test_train_evaluate(model: Tuple[str, str]): +def test_train_evaluate(model: tuple[str, str]): dataset, label_columns = _get_dataset(model[1]) is_test_passed = True # Importing function: @@ -156,9 +155,9 @@ def test_train_evaluate(model: Tuple[str, str]): is_test_passed = False assert is_test_passed, "The test failed" - assert ( - evaluate_run and "evaluation-test_set" in evaluate_run.outputs - ), "Missing fields in evaluate_run" + assert evaluate_run and "evaluation-test_set" in evaluate_run.outputs, ( + "Missing fields in evaluate_run" + ) @pytest.mark.parametrize("model", MODELS) @@ -166,7 +165,7 @@ def test_train_evaluate(model: Tuple[str, str]): condition=not _validate_environment_variables(), reason="Project's environment variables are not set", ) -def test_train_predict(model: Tuple[str, str]): +def test_train_predict(model: tuple[str, str]): is_test_passed = True dataset, label_columns = _get_dataset(model[1]) df = pd.read_csv(dataset) @@ -210,6 +209,6 @@ def test_train_predict(model: Tuple[str, str]): is_test_passed = False assert is_test_passed, "The test failed" - assert ( - predict_run and "prediction" in predict_run.outputs - ), "Prediction field must be in the output" + assert predict_run and "prediction" in predict_run.outputs, ( + "Prediction field must be in the output" + ) diff --git a/functions/src/azureml_serving/function.yaml b/functions/src/azureml_serving/function.yaml index 978806878..fd996b356 100644 --- a/functions/src/azureml_serving/function.yaml +++ b/functions/src/azureml_serving/function.yaml @@ -1,51 +1,31 @@ -kind: serving metadata: - name: azureml-serving tag: '' - hash: c0f404820b8f0fe92d2d1cfe9dbcc068be1a13bf - project: '' - labels: - author: Iguazio + name: azureml-serving categories: - machine-learning - model-serving +verbose: false +kind: serving spec: - command: '' - args: [] image: mlrun/mlrun - build: - commands: - - python -m pip install azureml-automl-runtime~=1.38.1 - code_origin: "" - origin_filename: "" - description: AzureML serving function disable_auto_mount: false - env: [] - priority_class_name: '' - preemption_mode: prevent + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBEbyBub3QgZGVsZXRlIQoKZnJvbSBtbHJ1bi5ydW50aW1lcyBpbXBvcnQgbnVjbGlvX2luaXRfaG9vawpkZWYgaW5pdF9jb250ZXh0KGNvbnRleHQpOgogICAgbnVjbGlvX2luaXRfaG9vayhjb250ZXh0LCBnbG9iYWxzKCksICdzZXJ2aW5nX3YyJykKCmRlZiBoYW5kbGVyKGNvbnRleHQsIGV2ZW50KToKICAgIHJldHVybiBjb250ZXh0Lm1scnVuX2hhbmRsZXIoY29udGV4dCwgZXZlbnQpCg== + requirements: + - azureml-automl-runtime~=1.38.1 + code_origin: '' + filename: azureml_serving.py + default_class: mlrun.frameworks.sklearn.PickleModelServer min_replicas: 1 - max_replicas: 4 - base_spec: - apiVersion: nuclio.io/v1 - kind: Function - metadata: - name: azureml-serving - labels: {} - annotations: - nuclio.io/generated_by: function generated from /Users/yonatanshelach/yoni/projects/functions/azureml_serving/azureml_serving.py - spec: - runtime: python - handler: azureml_serving:handler - env: [] - volumes: [] - build: - commands: [] - noBaseImagesPull: true - functionSourceCode: IyBEbyBub3QgZGVsZXRlIQoKZnJvbSBtbHJ1bi5ydW50aW1lcyBpbXBvcnQgbnVjbGlvX2luaXRfaG9vawpkZWYgaW5pdF9jb250ZXh0KGNvbnRleHQpOgogICAgbnVjbGlvX2luaXRfaG9vayhjb250ZXh0LCBnbG9iYWxzKCksICdzZXJ2aW5nX3YyJykKCmRlZiBoYW5kbGVyKGNvbnRleHQsIGV2ZW50KToKICAgIHJldHVybiBjb250ZXh0Lm1scnVuX2hhbmRsZXIoY29udGV4dCwgZXZlbnQpCg== + command: '' + default_handler: '' source: '' + max_replicas: 4 + base_image_pull: false + description: AzureML serving function function_kind: serving_v2 - default_class: mlrun.frameworks.sklearn.PickleModelServer - secret_sources: [] - affinity: null - tolerations: null -verbose: false \ No newline at end of file + function_handler: azureml-serving-nuclio:handler + env: + - name: MLRUN_HTTPDB__NUCLIO__EXPLICIT_ACK + value: enabled diff --git a/functions/src/azureml_utils/azureml_utils.py b/functions/src/azureml_utils/azureml_utils.py index 041af2b87..a8ac6bd7f 100644 --- a/functions/src/azureml_utils/azureml_utils.py +++ b/functions/src/azureml_utils/azureml_utils.py @@ -12,28 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import os import json import logging -from typing import Tuple, List +import os -from mlrun import MLClientCtx, DataItem, get_dataitem -import mlrun.feature_store as f_store import mlrun.datastore +import mlrun.feature_store as f_store import mlrun.utils -from mlrun.datastore.targets import ParquetTarget - from azureml.core.authentication import ServicePrincipalAuthentication -from azureml.core.workspace import Workspace -from azureml.core.experiment import Experiment +from azureml.core.compute import AmlCompute, ComputeTarget +from azureml.core.compute_target import ComputeTargetException from azureml.core.dataset import Dataset +from azureml.core.experiment import Experiment from azureml.core.model import Model -from azureml.core.compute import ComputeTarget, AmlCompute -from azureml.core.compute_target import ComputeTargetException from azureml.core.script_run import ScriptRun - +from azureml.core.workspace import Workspace from azureml.train.automl import AutoMLConfig from azureml.train.automl.run import AutoMLRun +from mlrun import DataItem, MLClientCtx, get_dataitem +from mlrun.datastore.targets import ParquetTarget def _env_or_secret(context, key): @@ -77,7 +74,7 @@ def _load_workspace(context: MLClientCtx) -> Workspace: def _init_experiment( context: MLClientCtx, experiment_name: str -) -> Tuple[Workspace, Experiment]: +) -> tuple[Workspace, Experiment]: """ Initialize workspace and experiment in Azure ML. Uses Service Principal authentication via environment variables. @@ -156,9 +153,9 @@ def register_dataset( """ # test for Azure storage connection environment variable or secret: - assert _env_or_secret( - context, "AZURE_STORAGE_CONNECTION_STRING" - ), "AZURE_STORAGE_CONNECTION_STRING secret not set" + assert _env_or_secret(context, "AZURE_STORAGE_CONNECTION_STRING"), ( + "AZURE_STORAGE_CONNECTION_STRING secret not set" + ) # Connect to AzureML experiment and datastore: context.logger.info("Connecting to AzureML experiment default datastore") @@ -177,7 +174,9 @@ def register_dataset( context.logger.info( f"Retrieving feature vector and uploading to Azure blob storage: {blob_path}" ) - f_store.get_offline_features(data.meta.uri, target=ParquetTarget(path=blob_path)) + f_store.get_offline_features( + data.meta.uri, target=ParquetTarget(path=blob_path) + ) else: blob_path += data.suffix # DataItem case: @@ -195,7 +194,7 @@ def register_dataset( ) else: context.logger.info( - f"OpenSSL version must be 1.1. Overriding the OpenSSL version to 1.1" + "OpenSSL version must be 1.1. Overriding the OpenSSL version to 1.1" ) # OpenSSL version must be 1.1 os.environ["CLR_OPENSSL_VERSION_OVERRIDE"] = "1.1" @@ -265,7 +264,7 @@ def upload_model( def _get_top_n_runs( remote_run: AutoMLRun, n: int = 5, primary_metric: str = "accuracy" -) -> List[ScriptRun]: +) -> list[ScriptRun]: """ Get top N complete runs from experiment sorted by primary metric. @@ -317,9 +316,9 @@ def _get_model_hp( return {} hp_dicts = spec_dict["objects"] # after training there are two hyper-parameters dicts inside the run object: - assert ( - len(hp_dicts) == 2 - ), "after training there are two hyper-parameters dicts inside the run object" + assert len(hp_dicts) == 2, ( + "after training there are two hyper-parameters dicts inside the run object" + ) result_dict = {} dict_keys = [ ["data_trans_class_name", "data_trans_module", "data_trans_spec_class"], @@ -336,7 +335,6 @@ def _get_model_hp( kwargs_prefix = "param_kwargs" for d, name, keys in zip(hp_dicts, ["data_trans", "train"], dict_keys): for key in keys: - if kwargs_prefix in key: result_dict[key] = d[kwargs_prefix][ key.replace(f"{name}_{kwargs_prefix}_", "") @@ -357,7 +355,7 @@ def submit_training_job( registered_dataset_name: str, automl_settings: dict, training_set: DataItem, - label_column_name: str = '', + label_column_name: str = "", save_n_models: int = 3, show_output: bool = True, ) -> None: @@ -390,7 +388,7 @@ def submit_training_job( if mlrun.utils.StorePrefix.FeatureVector == store_uri_prefix: feature_vector = training_set.meta.uri label_column_name = label_column_name or training_set.meta.status.label_column - context.logger.info(f'label column name: {label_column_name}') + context.logger.info(f"label column name: {label_column_name}") training_set = f_store.get_offline_features(feature_vector).to_dataframe() else: training_set = training_set.as_df() @@ -445,9 +443,7 @@ def submit_training_job( with context.get_child_context(**model_hp_dict) as child: model_key = f"model_{i + 1}_{model_hp_dict['data_trans_class_name'].lower()}_{model_hp_dict['train_class_name'].lower()}" # Log model: - context.logger.info( - f"Logging {model_key} model to MLRun" - ) + context.logger.info(f"Logging {model_key} model to MLRun") child.log_results(metrics) child.log_model( "model", diff --git a/functions/src/azureml_utils/function.yaml b/functions/src/azureml_utils/function.yaml index f14a6313f..fcd31ef59 100644 --- a/functions/src/azureml_utils/function.yaml +++ b/functions/src/azureml_utils/function.yaml @@ -1,32 +1,35 @@ +metadata: + tag: '' + name: azureml-utils + categories: + - model-serving + - utils verbose: false +kind: job spec: - command: '' + image: '' + disable_auto_mount: false build: - auto_build: true - code_origin: '' + origin_filename: '' with_mlrun: true + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IGpzb24KaW1wb3J0IGxvZ2dpbmcKaW1wb3J0IG9zCgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi5mZWF0dXJlX3N0b3JlIGFzIGZfc3RvcmUKaW1wb3J0IG1scnVuLnV0aWxzCmZyb20gYXp1cmVtbC5jb3JlLmF1dGhlbnRpY2F0aW9uIGltcG9ydCBTZXJ2aWNlUHJpbmNpcGFsQXV0aGVudGljYXRpb24KZnJvbSBhenVyZW1sLmNvcmUuY29tcHV0ZSBpbXBvcnQgQW1sQ29tcHV0ZSwgQ29tcHV0ZVRhcmdldApmcm9tIGF6dXJlbWwuY29yZS5jb21wdXRlX3RhcmdldCBpbXBvcnQgQ29tcHV0ZVRhcmdldEV4Y2VwdGlvbgpmcm9tIGF6dXJlbWwuY29yZS5kYXRhc2V0IGltcG9ydCBEYXRhc2V0CmZyb20gYXp1cmVtbC5jb3JlLmV4cGVyaW1lbnQgaW1wb3J0IEV4cGVyaW1lbnQKZnJvbSBhenVyZW1sLmNvcmUubW9kZWwgaW1wb3J0IE1vZGVsCmZyb20gYXp1cmVtbC5jb3JlLnNjcmlwdF9ydW4gaW1wb3J0IFNjcmlwdFJ1bgpmcm9tIGF6dXJlbWwuY29yZS53b3Jrc3BhY2UgaW1wb3J0IFdvcmtzcGFjZQpmcm9tIGF6dXJlbWwudHJhaW4uYXV0b21sIGltcG9ydCBBdXRvTUxDb25maWcKZnJvbSBhenVyZW1sLnRyYWluLmF1dG9tbC5ydW4gaW1wb3J0IEF1dG9NTFJ1bgpmcm9tIG1scnVuIGltcG9ydCBEYXRhSXRlbSwgTUxDbGllbnRDdHgsIGdldF9kYXRhaXRlbQpmcm9tIG1scnVuLmRhdGFzdG9yZS50YXJnZXRzIGltcG9ydCBQYXJxdWV0VGFyZ2V0CgoKZGVmIF9lbnZfb3Jfc2VjcmV0KGNvbnRleHQsIGtleSk6CiAgICBpZiBrZXkgaW4gb3MuZW52aXJvbjoKICAgICAgICByZXR1cm4gb3MuZW52aXJvbltrZXldCiAgICByZXR1cm4gY29udGV4dC5nZXRfc2VjcmV0KGtleSkKCgpkZWYgX2xvYWRfd29ya3NwYWNlKGNvbnRleHQ6IE1MQ2xpZW50Q3R4KSAtPiBXb3Jrc3BhY2U6CiAgICAiIiIKICAgIExvYWRpbmcgQXp1cmVNTCBXb3Jrc3BhY2Ugd2l0aCBBenVyZSBzZWNyZXRzLgoKICAgIDpwYXJhbSBjb250ZXh0OiBNTFJ1biBjb250ZXh0LgogICAgOnJldHVybnM6ICAgICAgIEF6dXJlTUwgV29ya3NwYWNlCiAgICAiIiIKCiAgICBpZiBoYXNhdHRyKGNvbnRleHQsICJfYXp1cmVfd29ya3NwYWNlIik6CiAgICAgICAgcmV0dXJuIGNvbnRleHQuX2F6dXJlX3dvcmtzcGFjZQoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkxvYWRpbmcgQXp1cmVNTCBXb3Jrc3BhY2UiKQogICAgIyBBenVyZSBzZXJ2aWNlIGF1dGhlbnRpY2F0aW9uOgogICAgc2VydmljZV9hdXRoZW50aWNhdGlvbiA9IFNlcnZpY2VQcmluY2lwYWxBdXRoZW50aWNhdGlvbigKICAgICAgICB0ZW5hbnRfaWQ9X2Vudl9vcl9zZWNyZXQoY29udGV4dCwgIkFaVVJFX1RFTkFOVF9JRCIpLAogICAgICAgIHNlcnZpY2VfcHJpbmNpcGFsX2lkPV9lbnZfb3Jfc2VjcmV0KGNvbnRleHQsICJBWlVSRV9TRVJWSUNFX1BSSU5DSVBBTF9JRCIpLAogICAgICAgIHNlcnZpY2VfcHJpbmNpcGFsX3Bhc3N3b3JkPV9lbnZfb3Jfc2VjcmV0KAogICAgICAgICAgICBjb250ZXh0LCAiQVpVUkVfU0VSVklDRV9QUklOQ0lQQUxfUEFTU1dPUkQiCiAgICAgICAgKSwKICAgICkKCiAgICAjIExvYWRpbmcgQXp1cmUgd29ya3NwYWNlOgogICAgd29ya3NwYWNlID0gV29ya3NwYWNlKAogICAgICAgIHN1YnNjcmlwdGlvbl9pZD1fZW52X29yX3NlY3JldChjb250ZXh0LCAiQVpVUkVfU1VCU0NSSVBUSU9OX0lEIiksCiAgICAgICAgcmVzb3VyY2VfZ3JvdXA9X2Vudl9vcl9zZWNyZXQoY29udGV4dCwgIkFaVVJFX1JFU09VUkNFX0dST1VQIiksCiAgICAgICAgd29ya3NwYWNlX25hbWU9X2Vudl9vcl9zZWNyZXQoY29udGV4dCwgIkFaVVJFX1dPUktTUEFDRV9OQU1FIiksCiAgICAgICAgYXV0aD1zZXJ2aWNlX2F1dGhlbnRpY2F0aW9uLAogICAgKQoKICAgIGNvbnRleHQuX2F6dXJlX3dvcmtzcGFjZSA9IHdvcmtzcGFjZQogICAgcmV0dXJuIHdvcmtzcGFjZQoKCmRlZiBfaW5pdF9leHBlcmltZW50KAogICAgY29udGV4dDogTUxDbGllbnRDdHgsIGV4cGVyaW1lbnRfbmFtZTogc3RyCikgLT4gdHVwbGVbV29ya3NwYWNlLCBFeHBlcmltZW50XToKICAgICIiIgogICAgSW5pdGlhbGl6ZSB3b3Jrc3BhY2UgYW5kIGV4cGVyaW1lbnQgaW4gQXp1cmUgTUwuIFVzZXMgU2VydmljZQogICAgUHJpbmNpcGFsIGF1dGhlbnRpY2F0aW9uIHZpYSBlbnZpcm9ubWVudCB2YXJpYWJsZXMuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBleHBlcmltZW50X25hbWU6IE5hbWUgb2YgZXhwZXJpbWVudCB0byBjcmVhdGUgaW4gQXp1cmUgTUwuCiAgICA6cmV0dXJuczogICAgICAgICAgICAgICBBenVyZSBNTCBXb3Jrc3BhY2UgYW5kIEV4cGVyaW1lbnQuCiAgICAiIiIKCiAgICAjIEluaXRpYWxpemUgZXhwZXJpbWVudCB2aWEgU2VydmljZSBQcmluY2lwYWwgQXV0aGVudGljYXRpb246CiAgICAjIGh0dHBzOi8vZG9jcy5taWNyb3NvZnQuY29tL2VuLXVzL2F6dXJlL21hY2hpbmUtbGVhcm5pbmcvaG93LXRvLXNldHVwLWF1dGhlbnRpY2F0aW9uI3VzZS1zZXJ2aWNlLXByaW5jaXBhbC1hdXRoZW50aWNhdGlvbgoKICAgIHdvcmtzcGFjZSA9IF9sb2FkX3dvcmtzcGFjZShjb250ZXh0KQoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJJbml0aWFsaXppbmcgQXp1cmVNTCBleHBlcmltZW50IHtleHBlcmltZW50X25hbWV9IikKICAgICMgQ3JlYXRpbmcgZXhwZXJpbWVudDoKICAgIGV4cGVyaW1lbnQgPSBFeHBlcmltZW50KHdvcmtzcGFjZSwgZXhwZXJpbWVudF9uYW1lKQoKICAgIHJldHVybiB3b3Jrc3BhY2UsIGV4cGVyaW1lbnQKCgpkZWYgaW5pdF9jb21wdXRlKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBjcHVfY2x1c3Rlcl9uYW1lOiBzdHIsCiAgICB2bV9zaXplOiBzdHIgPSAiU1RBTkRBUkRfRDJfVjIiLAogICAgbWF4X25vZGVzOiBpbnQgPSAxLAopIC0+IENvbXB1dGVUYXJnZXQ6CiAgICAiIiIKICAgIEluaXRpYWxpemUgQXp1cmUgTUwgY29tcHV0ZSB0YXJnZXQgdG8gcnVuIGV4cGVyaW1lbnQuIENoZWNrcyBmb3IKICAgIGV4aXN0aW5nIGNvbXB1dGUgdGFyZ2V0IGFuZCBjcmVhdGVzIG5ldyBpZiBkb2VzIG5vdCBleGlzdC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBjcHVfY2x1c3Rlcl9uYW1lOiBOYW1lIG9mIEF6dXJlIE1MIGNvbXB1dGUgdGFyZ2V0LiBDcmVhdGVkIGlmIGRvZXMgbm90IGV4aXN0LgogICAgOnBhcmFtIHZtX3NpemU6ICAgICAgICAgIEF6dXJlIG1hY2hpbmUgdHlwZSBmb3IgY29tcHV0ZSB0YXJnZXQuCiAgICA6cGFyYW0gbWF4X25vZGVzOiAgICAgICAgTWF4aW11bSBudW1iZXIgb2YgY29uY3VycmVudCBjb21wdXRlIHRhcmdldHMuCiAgICA6cmV0dXJuczogICAgICAgICAgICAgICAgQXp1cmUgTUwgQ29tcHV0ZSBUYXJnZXQuCiAgICAiIiIKCiAgICB3b3Jrc3BhY2UgPSBfbG9hZF93b3Jrc3BhY2UoY29udGV4dCkKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJJbml0aWFsaXppbmcgQXp1cmVNTCBjb21wdXRlIHRhcmdldCB7Y3B1X2NsdXN0ZXJfbmFtZX0iKQoKICAgICMgVmVyaWZ5IHRoYXQgY2x1c3RlciBkb2VzIG5vdCBleGlzdCBhbHJlYWR5OgogICAgdHJ5OgogICAgICAgIGNvbXB1dGVfdGFyZ2V0ID0gQ29tcHV0ZVRhcmdldCh3b3Jrc3BhY2U9d29ya3NwYWNlLCBuYW1lPWNwdV9jbHVzdGVyX25hbWUpCiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygiRm91bmQgZXhpc3RpbmcgY2x1c3Rlciwgd2lsbCB1c2UgaXQuIikKICAgIGV4Y2VwdCBDb21wdXRlVGFyZ2V0RXhjZXB0aW9uOgogICAgICAgIGNvbXB1dGVfY29uZmlnID0gQW1sQ29tcHV0ZS5wcm92aXNpb25pbmdfY29uZmlndXJhdGlvbigKICAgICAgICAgICAgdm1fc2l6ZT12bV9zaXplLCBtYXhfbm9kZXM9bWF4X25vZGVzCiAgICAgICAgKQogICAgICAgIGNvbXB1dGVfdGFyZ2V0ID0gQ29tcHV0ZVRhcmdldC5jcmVhdGUoCiAgICAgICAgICAgIHdvcmtzcGFjZSwgY3B1X2NsdXN0ZXJfbmFtZSwgY29tcHV0ZV9jb25maWcKICAgICAgICApCgogICAgY29tcHV0ZV90YXJnZXQud2FpdF9mb3JfY29tcGxldGlvbihzaG93X291dHB1dD1UcnVlKQogICAgcmV0dXJuIGNvbXB1dGVfdGFyZ2V0CgoKZGVmIHJlZ2lzdGVyX2RhdGFzZXQoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIGRhdGFzZXRfbmFtZTogc3RyLAogICAgZGF0YXNldF9kZXNjcmlwdGlvbjogc3RyLAogICAgZGF0YTogRGF0YUl0ZW0sCiAgICBjcmVhdGVfbmV3X3ZlcnNpb246IGJvb2wgPSBGYWxzZSwKKToKICAgICIiIgogICAgUmVnaXN0ZXIgZGF0YXNldCBvYmplY3QgKGNhbiBiZSBhbHNvIGFuIElndWF6aW8gRmVhdHVyZVZlY3RvcikgaW4gQXp1cmUgTUwuCiAgICBVcGxvYWRzIHBhcnF1ZXQgZmlsZSB0byBBenVyZSBibG9iIHN0b3JhZ2UgYW5kIHJlZ2lzdGVycwogICAgdGhhdCBmaWxlIGFzIGEgZGF0YXNldCBpbiBBenVyZSBNTC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIGRhdGFzZXRfbmFtZTogICAgICAgICAgTmFtZSBvZiBBenVyZSBkYXRhc2V0IHRvIHJlZ2lzdGVyLgogICAgOnBhcmFtIGRhdGFzZXRfZGVzY3JpcHRpb246ICAgRGVzY3JpcHRpb24gb2YgQXp1cmUgZGF0YXNldCB0byByZWdpc3Rlci4KICAgIDpwYXJhbSBkYXRhOiAgICAgICAgICAgICAgICAgIE1MUnVuIEZlYXR1cmVWZWN0b3Igb3IgZGF0YXNldCBvYmplY3QgdG8gdXBsb2FkLgogICAgOnBhcmFtIGNyZWF0ZV9uZXdfdmVyc2lvbjogICAgUmVnaXN0ZXIgQXp1cmUgZGF0YXNldCBhcyBuZXcgdmVyc2lvbi4gTXVzdCBiZSB1c2VkIHdoZW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGlmeWluZyBkYXRhc2V0IHNjaGVtYS4KICAgICIiIgoKICAgICMgdGVzdCBmb3IgQXp1cmUgc3RvcmFnZSBjb25uZWN0aW9uIGVudmlyb25tZW50IHZhcmlhYmxlIG9yIHNlY3JldDoKICAgIGFzc2VydCBfZW52X29yX3NlY3JldChjb250ZXh0LCAiQVpVUkVfU1RPUkFHRV9DT05ORUNUSU9OX1NUUklORyIpLCAoCiAgICAgICAgIkFaVVJFX1NUT1JBR0VfQ09OTkVDVElPTl9TVFJJTkcgc2VjcmV0IG5vdCBzZXQiCiAgICApCgogICAgIyBDb25uZWN0IHRvIEF6dXJlTUwgZXhwZXJpbWVudCBhbmQgZGF0YXN0b3JlOgogICAgY29udGV4dC5sb2dnZXIuaW5mbygiQ29ubmVjdGluZyB0byBBenVyZU1MIGV4cGVyaW1lbnQgZGVmYXVsdCBkYXRhc3RvcmUiKQoKICAgIHdvcmtzcGFjZSA9IF9sb2FkX3dvcmtzcGFjZShjb250ZXh0KQogICAgZGF0YXN0b3JlID0gd29ya3NwYWNlLmdldF9kZWZhdWx0X2RhdGFzdG9yZSgpCgogICAgIyBBenVyZSBibG9iIHBhdGggKGRlZmF1bHQgZGF0YXN0b3JlIGZvciB3b3Jrc3BhY2UpOgogICAgYmxvYl9wYXRoID0gZiJhejovL3tkYXRhc3RvcmUuY29udGFpbmVyX25hbWV9L3tkYXRhc2V0X25hbWV9IgoKICAgIHN0b3JlX3VyaV9wcmVmaXgsIF8gPSBtbHJ1bi5kYXRhc3RvcmUucGFyc2Vfc3RvcmVfdXJpKGRhdGEuYXJ0aWZhY3RfdXJsKQogICAgZmVhdHVyZV92ZWN0b3JfY2FzZSA9IG1scnVuLnV0aWxzLlN0b3JlUHJlZml4LkZlYXR1cmVWZWN0b3IgPT0gc3RvcmVfdXJpX3ByZWZpeAogICAgIyBSZXRyaWV2ZSBkYXRhIHNvdXJjZSBhcyBkYXRhZnJhbWU6CiAgICBpZiBmZWF0dXJlX3ZlY3Rvcl9jYXNlOgogICAgICAgICMgRmVhdHVyZVZlY3RvciBjYXNlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgIGYiUmV0cmlldmluZyBmZWF0dXJlIHZlY3RvciBhbmQgdXBsb2FkaW5nIHRvIEF6dXJlIGJsb2Igc3RvcmFnZToge2Jsb2JfcGF0aH0iCiAgICAgICAgKQogICAgICAgIGZfc3RvcmUuZ2V0X29mZmxpbmVfZmVhdHVyZXMoCiAgICAgICAgICAgIGRhdGEubWV0YS51cmksIHRhcmdldD1QYXJxdWV0VGFyZ2V0KHBhdGg9YmxvYl9wYXRoKQogICAgICAgICkKICAgIGVsc2U6CiAgICAgICAgYmxvYl9wYXRoICs9IGRhdGEuc3VmZml4CiAgICAgICAgIyBEYXRhSXRlbSBjYXNlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgIGYiUmV0cmlldmluZyBmZWF0dXJlIHZlY3RvciBhbmQgdXBsb2FkaW5nIHRvIEF6dXJlIGJsb2Igc3RvcmFnZToge2Jsb2JfcGF0aH0iCiAgICAgICAgKQogICAgICAgIGRhdGFfaW5fYnl0ZXMgPSBkYXRhLmdldCgpCiAgICAgICAgZ2V0X2RhdGFpdGVtKGJsb2JfcGF0aCkucHV0KGRhdGFfaW5fYnl0ZXMpCgogICAgIyBSZWdpc3RlciBkYXRhc2V0IGluIEF6dXJlTUw6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiUmVnaXN0ZXJpbmcgZGF0YXNldCB7ZGF0YXNldF9uYW1lfSBpbiBBenVyZSBNTCIpCiAgICBpZiBkYXRhLnN1ZmZpeCA9PSAiLnBhcnF1ZXQiIG9yIGZlYXR1cmVfdmVjdG9yX2Nhc2U6CiAgICAgICAgZGF0YXNldCA9IERhdGFzZXQuVGFidWxhci5mcm9tX3BhcnF1ZXRfZmlsZXMoCiAgICAgICAgICAgIHBhdGg9KGRhdGFzdG9yZSwgZiJ7ZGF0YXNldF9uYW1lfS5wYXJxdWV0IiksIHZhbGlkYXRlPUZhbHNlCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAiT3BlblNTTCB2ZXJzaW9uIG11c3QgYmUgMS4xLiBPdmVycmlkaW5nIHRoZSBPcGVuU1NMIHZlcnNpb24gdG8gMS4xIgogICAgICAgICkKICAgICAgICAjIE9wZW5TU0wgdmVyc2lvbiBtdXN0IGJlIDEuMQogICAgICAgIG9zLmVudmlyb25bIkNMUl9PUEVOU1NMX1ZFUlNJT05fT1ZFUlJJREUiXSA9ICIxLjEiCiAgICAgICAgZGF0YXNldCA9IERhdGFzZXQuVGFidWxhci5mcm9tX2RlbGltaXRlZF9maWxlcygKICAgICAgICAgICAgcGF0aD0oZGF0YXN0b3JlLCBmIntkYXRhc2V0X25hbWV9e2RhdGEuc3VmZml4fSIpLCB2YWxpZGF0ZT1GYWxzZQogICAgICAgICkKCiAgICBkYXRhc2V0LnJlZ2lzdGVyKAogICAgICAgIHdvcmtzcGFjZT13b3Jrc3BhY2UsCiAgICAgICAgbmFtZT1kYXRhc2V0X25hbWUsCiAgICAgICAgZGVzY3JpcHRpb249ZGF0YXNldF9kZXNjcmlwdGlvbiwKICAgICAgICBjcmVhdGVfbmV3X3ZlcnNpb249Y3JlYXRlX25ld192ZXJzaW9uLAogICAgKQoKICAgICMgT3V0cHV0IHJlZ2lzdGVyZWQgZGF0YXNldCBuYW1lIGluIEF6dXJlOgogICAgY29udGV4dC5sb2dfcmVzdWx0KCJkYXRhc2V0X2Jsb2JfcGF0aCIsIGJsb2JfcGF0aCkKCgpkZWYgZG93bmxvYWRfbW9kZWwoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsX25hbWU6IHN0ciwKICAgIG1vZGVsX3ZlcnNpb246IGludCwKICAgIHRhcmdldF9kaXI6IHN0ciA9ICIuIiwKKSAtPiBOb25lOgogICAgIiIiCiAgICBEb3dubG9hZCB0cmFpbmVkIG1vZGVsIGZyb20gQXp1cmUgTUwgdG8gbG9jYWwgZmlsZXN5c3RlbS4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgICBOYW1lIG9mIHRyYWluZWQgYW5kIHJlZ2lzdGVyZWQgbW9kZWwuCiAgICA6cGFyYW0gbW9kZWxfdmVyc2lvbjogVmVyc2lvbiBvZiBtb2RlbCB0byBkb3dubG9hZC4KICAgIDpwYXJhbSB0YXJnZXRfZGlyOiAgICBUYXJnZXQgZGlyZWN0b3J5IHRvIGRvd25sb2FkIG1vZGVsLgogICAgIiIiCiAgICAjIExvYWRpbmcgd29ya3NwYWNlIGlmIG5vdCBwcm92aWRlZDoKICAgIHdvcmtzcGFjZSA9IF9sb2FkX3dvcmtzcGFjZShjb250ZXh0KQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIkRvd25sb2FkaW5nIG1vZGVsIHttb2RlbF9uYW1lfTp7bW9kZWxfdmVyc2lvbn0iKQogICAgbW9kZWwgPSBNb2RlbCh3b3Jrc3BhY2UsIG1vZGVsX25hbWUsIHZlcnNpb249bW9kZWxfdmVyc2lvbikKICAgIG1vZGVsLmRvd25sb2FkKHRhcmdldF9kaXI9dGFyZ2V0X2RpciwgZXhpc3Rfb2s9VHJ1ZSkKCgpkZWYgdXBsb2FkX21vZGVsKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBtb2RlbF9uYW1lOiBzdHIsCiAgICBtb2RlbF9wYXRoOiBzdHIsCiAgICBtb2RlbF9kZXNjcmlwdGlvbjogc3RyID0gTm9uZSwKICAgIG1vZGVsX3RhZ3M6IGRpY3QgPSBOb25lLAopIC0+IE5vbmU6CiAgICAiIiIKICAgIFVwbG9hZCBwcmUtdHJhaW5lZCBtb2RlbCBmcm9tIGxvY2FsIGZpbGVzeXN0ZW0gdG8gQXp1cmUgTUwuCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gbW9kZWxfbmFtZTogICAgICAgIE5hbWUgb2YgdHJhaW5lZCBhbmQgcmVnaXN0ZXJlZCBtb2RlbC4KICAgIDpwYXJhbSBtb2RlbF9wYXRoOiAgICAgICAgUGF0aCB0byBmaWxlIG9uIGxvY2FsIGZpbGVzeXN0ZW0uCiAgICA6cGFyYW0gbW9kZWxfZGVzY3JpcHRpb246IERlc2NyaXB0aW9uIG9mIG1vZGVscy4KICAgIDpwYXJhbSBtb2RlbF90YWdzOiAgICAgICAgS1YgcGFpcnMgb2YgbW9kZWwgdGFncy4KICAgICIiIgogICAgIyBMb2FkaW5nIHdvcmtzcGFjZSBpZiBub3QgcHJvdmlkZWQ6CiAgICB3b3Jrc3BhY2UgPSBfbG9hZF93b3Jrc3BhY2UoY29udGV4dCkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVXBsb2FkIG1vZGVsIHttb2RlbF9uYW1lfSBmcm9tIHttb2RlbF9wYXRofSIpCiAgICBNb2RlbC5yZWdpc3RlcigKICAgICAgICB3b3Jrc3BhY2U9d29ya3NwYWNlLAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWxfcGF0aCwKICAgICAgICBtb2RlbF9uYW1lPW1vZGVsX25hbWUsCiAgICAgICAgZGVzY3JpcHRpb249bW9kZWxfZGVzY3JpcHRpb24sCiAgICAgICAgdGFncz1tb2RlbF90YWdzLAogICAgKQoKCmRlZiBfZ2V0X3RvcF9uX3J1bnMoCiAgICByZW1vdGVfcnVuOiBBdXRvTUxSdW4sIG46IGludCA9IDUsIHByaW1hcnlfbWV0cmljOiBzdHIgPSAiYWNjdXJhY3kiCikgLT4gbGlzdFtTY3JpcHRSdW5dOgogICAgIiIiCiAgICBHZXQgdG9wIE4gY29tcGxldGUgcnVucyBmcm9tIGV4cGVyaW1lbnQgc29ydGVkIGJ5IHByaW1hcnkgbWV0cmljLgoKICAgIDpwYXJhbSByZW1vdGVfcnVuOiAgICAgQXp1cmUgTUwgUnVuLgogICAgOnBhcmFtIG46ICAgICAgICAgICAgICBOdW1iZXIgb2YgdG9wIHJ1bnMgdG8gcmV0dXJuLgogICAgOnBhcmFtIHByaW1hcnlfbWV0cmljOiBNZXRyaWMgdG8gc29ydCBieS4KCiAgICA6cmV0dXJuczogICAgICAgICAgICAgIExpc3Qgb2YgdG9wIE4gcnVucyBzb3J0ZWQgYnkgcHJpbWFyeSBtZXRyaWMuCiAgICAiIiIKICAgICMgQ29sbGVjdCBhbGwgbW9kZWxzOgogICAgY29tcGxldGVfcnVucyA9IFsKICAgICAgICBydW4KICAgICAgICBmb3IgcnVuIGluIHJlbW90ZV9ydW4uZ2V0X2NoaWxkcmVuKHN0YXR1cz0iQ29tcGxldGVkIikKICAgICAgICBpZiBub3QgYW55KHMgaW4gcnVuLmlkIGZvciBzIGluIFsic2V0dXAiLCAid29ya2VyIl0pCiAgICBdCgogICAgIyBDaGVja2luZyB0aGF0IHRoZSByZXF1aXJlZCBudW1iZXIgb2YgcnVucyBhcmUgZG9uZToKICAgIGlmIGxlbihjb21wbGV0ZV9ydW5zKSA8IG46CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcihmIkV4cGVjdGVkIHtufSBydW5zIGJ1dCBvbmx5IHJlY2VpdmVkIHtsZW4oY29tcGxldGVfcnVucyl9IikKCiAgICAjIFNvcnRpbmcgYnkgdGhlIHByaW1hcnkgbWV0cmljOgogICAgc29ydGVkX3J1bnMgPSBzb3J0ZWQoCiAgICAgICAgY29tcGxldGVfcnVucywga2V5PWxhbWJkYSBydW46IHJ1bi5nZXRfbWV0cmljcygpW3ByaW1hcnlfbWV0cmljXSwgcmV2ZXJzZT1UcnVlCiAgICApCiAgICByZXR1cm4gc29ydGVkX3J1bnNbOm5dCgoKZGVmIF9nZXRfbW9kZWxfaHAoCiAgICBydW46IFNjcmlwdFJ1biwKKSAtPiBkaWN0OgogICAgIiIiCiAgICBHZXQgaHlwZXItcGFyYW1ldGVycyBvZiB0cmFpbmVkIEF6dXJlTUwgbW9kZWwuCiAgICBDb21iaW5lIHRoZSBoeXBlci1wYXJhbWV0ZXJzIG9mIHRoZSBkYXRhIHRyYW5zZm9ybWF0aW9uIGFuZCB0cmFpbmluZyB0byBhIGRpY3Rpb25hcnkuCiAgICBUaGUgcHJlZml4IG9mIHRoZSBkaWN0aW9uYXJ5IGtleXMgY29ycmVzcG9uZHMgdG8gJ2RhdGEgdHJhbnNmb3JtYXRpb24nIGFuZCAndHJhaW5pbmcnLgoKICAgIDpwYXJhbSBydW46IFJ1biBvYmplY3Qgb2YgQXp1cmVNTCB0cmFpbmVkIG1vZGVsLgoKICAgIDpyZXR1cm5zOiAgICBBIGRpY3Rpb25hcnkgYXMgZGVzY3JpYmVkIGluIHRoZSBkb2NzdHJpbmcuCiAgICAiIiIKCiAgICBzcGVjX2ZpZWxkID0gInBpcGVsaW5lX3NwZWMiCiAgICBpZiBzcGVjX2ZpZWxkIG5vdCBpbiBydW4ucHJvcGVydGllczoKICAgICAgICByZXR1cm4ge30KICAgIHNwZWNfc3RyaW5nID0gcnVuLnByb3BlcnRpZXNbc3BlY19maWVsZF0KICAgIHNwZWNfZGljdCA9IGpzb24ubG9hZHMoc3BlY19zdHJpbmcpCgogICAgaWYgIm9iamVjdHMiIG5vdCBpbiBzcGVjX2RpY3Q6CiAgICAgICAgIyBObyBoeXBlci1wYXJhbXMKICAgICAgICByZXR1cm4ge30KICAgIGhwX2RpY3RzID0gc3BlY19kaWN0WyJvYmplY3RzIl0KICAgICMgYWZ0ZXIgdHJhaW5pbmcgdGhlcmUgYXJlIHR3byBoeXBlci1wYXJhbWV0ZXJzIGRpY3RzIGluc2lkZSB0aGUgcnVuIG9iamVjdDoKICAgIGFzc2VydCBsZW4oaHBfZGljdHMpID09IDIsICgKICAgICAgICAiYWZ0ZXIgdHJhaW5pbmcgdGhlcmUgYXJlIHR3byBoeXBlci1wYXJhbWV0ZXJzIGRpY3RzIGluc2lkZSB0aGUgcnVuIG9iamVjdCIKICAgICkKICAgIHJlc3VsdF9kaWN0ID0ge30KICAgIGRpY3Rfa2V5cyA9IFsKICAgICAgICBbImRhdGFfdHJhbnNfY2xhc3NfbmFtZSIsICJkYXRhX3RyYW5zX21vZHVsZSIsICJkYXRhX3RyYW5zX3NwZWNfY2xhc3MiXSwKICAgICAgICBbCiAgICAgICAgICAgICJ0cmFpbl9jbGFzc19uYW1lIiwKICAgICAgICAgICAgInRyYWluX21vZHVsZSIsCiAgICAgICAgICAgICJ0cmFpbl9wYXJhbV9rd2FyZ3NfQyIsCiAgICAgICAgICAgICJ0cmFpbl9wYXJhbV9rd2FyZ3NfY2xhc3Nfd2VpZ2h0IiwKICAgICAgICAgICAgInRyYWluX3NwZWNfY2xhc3MiLAogICAgICAgIF0sCiAgICBdCgogICAgIyBjcmVhdGluZyBoeXBlci1wYXJhbXMgZGljdCB3aXRoIGtleSBwcmVmaXhlcyBmb3IgZWFjaCBwYXJ0OgogICAga3dhcmdzX3ByZWZpeCA9ICJwYXJhbV9rd2FyZ3MiCiAgICBmb3IgZCwgbmFtZSwga2V5cyBpbiB6aXAoaHBfZGljdHMsIFsiZGF0YV90cmFucyIsICJ0cmFpbiJdLCBkaWN0X2tleXMpOgogICAgICAgIGZvciBrZXkgaW4ga2V5czoKICAgICAgICAgICAgaWYga3dhcmdzX3ByZWZpeCBpbiBrZXk6CiAgICAgICAgICAgICAgICByZXN1bHRfZGljdFtrZXldID0gZFtrd2FyZ3NfcHJlZml4XVsKICAgICAgICAgICAgICAgICAgICBrZXkucmVwbGFjZShmIntuYW1lfV97a3dhcmdzX3ByZWZpeH1fIiwgIiIpCiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICByZXN1bHRfZGljdFtrZXldID0gZFtrZXkucmVwbGFjZShmIntuYW1lfV8iLCAiIildCiAgICAgICAgICAgIGlmIG5vdCByZXN1bHRfZGljdFtrZXldOgogICAgICAgICAgICAgICAgcmVzdWx0X2RpY3Rba2V5XSA9ICIiCgogICAgcmV0dXJuIHJlc3VsdF9kaWN0CgoKZGVmIHN1Ym1pdF90cmFpbmluZ19qb2IoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIGV4cGVyaW1lbnQ6IEV4cGVyaW1lbnQsCiAgICBjb21wdXRlX3RhcmdldDogQ29tcHV0ZVRhcmdldCwKICAgIHJlZ2lzdGVyX21vZGVsX25hbWU6IHN0ciwKICAgIHJlZ2lzdGVyZWRfZGF0YXNldF9uYW1lOiBzdHIsCiAgICBhdXRvbWxfc2V0dGluZ3M6IGRpY3QsCiAgICB0cmFpbmluZ19zZXQ6IERhdGFJdGVtLAogICAgbGFiZWxfY29sdW1uX25hbWU6IHN0ciA9ICIiLAogICAgc2F2ZV9uX21vZGVsczogaW50ID0gMywKICAgIHNob3dfb3V0cHV0OiBib29sID0gVHJ1ZSwKKSAtPiBOb25lOgogICAgIiIiCiAgICBTdWJtaXQgdHJhaW5pbmcgam9iIHRvIEF6dXJlIEF1dG9NTCBhbmQgZG93bmxvYWQgdHJhaW5lZCBtb2RlbAogICAgd2hlbiBjb21wbGV0ZWQuIFVzZXMgcHJldmlvdXNseSByZWdpc3RlcmVkIGRhdGFzZXQgZm9yIHRyYWluaW5nLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBleHBlcmltZW50OiAgICAgICAgICAgICAgQXp1cmUgZXhwZXJpbWVudC4KICAgIDpwYXJhbSBjb21wdXRlX3RhcmdldDogICAgICAgICAgQXp1cmUgY29tcHV0ZSB0YXJnZXQuCiAgICA6cGFyYW0gcmVnaXN0ZXJfbW9kZWxfbmFtZTogICAgIE5hbWUgb2YgbW9kZWwgdG8gcmVnaXN0ZXIgaW4gQXp1cmUuCiAgICA6cGFyYW0gcmVnaXN0ZXJlZF9kYXRhc2V0X25hbWU6IE5hbWUgb2YgZGF0YXNldCByZWdpc3RlcmVkIGluIEF6dXJlIE1MLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbl9uYW1lOiAgICAgICBOYW1lIG9mIHRhcmdldCBjb2x1bW4gaW4gZGF0YXNldC4KICAgIDpwYXJhbSBhdXRvbWxfc2V0dGluZ3M6ICAgICAgICAgSlNPTiBzdHJpbmcgb2YgYWxsIEF6dXJlIEF1dG9NTCBzZXR0aW5ncy4KICAgIDpwYXJhbSB0cmFpbmluZ19zZXQ6ICAgICAgICAgICAgVHJhaW5pbmcgc2V0IHRvIGxvZyB3aXRoIG1vZGVsLiBGb3IgbW9kZWwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9uaXRvcmluZyBpbnRlZ3JhdGlvbi4KICAgIDpwYXJhbSBzaG93X291dHB1dDogICAgICAgICAgICAgRGlzcGxheWluZyBBenVyZSBsb2dzLgogICAgOnBhcmFtIHNhdmVfbl9tb2RlbHM6ICAgICAgICAgICBIb3cgbWFueSBvZiB0aGUgdG9wIHBlcmZvcm1pbmcgbW9kZWxzIHRvIGxvZy4KICAgICIiIgogICAgIyBMb2FkaW5nIHdvcmtzcGFjZSBpZiBub3QgcHJvdmlkZWQ6CiAgICB3b3Jrc3BhY2UgPSBfbG9hZF93b3Jrc3BhY2UoY29udGV4dCkKCiAgICAjIFNldHVwIGV4cGVyaW1lbnQ6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJTZXR0aW5nIHVwIGV4cGVyaW1lbnQgcGFyYW1ldGVycyIpCiAgICBkYXRhc2V0ID0gRGF0YXNldC5nZXRfYnlfbmFtZSh3b3Jrc3BhY2UsIG5hbWU9cmVnaXN0ZXJlZF9kYXRhc2V0X25hbWUpCgogICAgIyBHZXQgdHJhaW5pbmcgc2V0IHRvIGxvZyB3aXRoIG1vZGVsOgogICAgZmVhdHVyZV92ZWN0b3IgPSBOb25lCiAgICBzdG9yZV91cmlfcHJlZml4LCBfID0gbWxydW4uZGF0YXN0b3JlLnBhcnNlX3N0b3JlX3VyaSh0cmFpbmluZ19zZXQuYXJ0aWZhY3RfdXJsKQogICAgaWYgbWxydW4udXRpbHMuU3RvcmVQcmVmaXguRmVhdHVyZVZlY3RvciA9PSBzdG9yZV91cmlfcHJlZml4OgogICAgICAgIGZlYXR1cmVfdmVjdG9yID0gdHJhaW5pbmdfc2V0Lm1ldGEudXJpCiAgICAgICAgbGFiZWxfY29sdW1uX25hbWUgPSBsYWJlbF9jb2x1bW5fbmFtZSBvciB0cmFpbmluZ19zZXQubWV0YS5zdGF0dXMubGFiZWxfY29sdW1uCiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbyhmImxhYmVsIGNvbHVtbiBuYW1lOiB7bGFiZWxfY29sdW1uX25hbWV9IikKICAgICAgICB0cmFpbmluZ19zZXQgPSBmX3N0b3JlLmdldF9vZmZsaW5lX2ZlYXR1cmVzKGZlYXR1cmVfdmVjdG9yKS50b19kYXRhZnJhbWUoKQogICAgZWxzZToKICAgICAgICB0cmFpbmluZ19zZXQgPSB0cmFpbmluZ19zZXQuYXNfZGYoKQoKICAgIGF1dG9tbF9jb25maWcgPSBBdXRvTUxDb25maWcoCiAgICAgICAgY29tcHV0ZV90YXJnZXQ9Y29tcHV0ZV90YXJnZXQsCiAgICAgICAgdHJhaW5pbmdfZGF0YT1kYXRhc2V0LAogICAgICAgIHZlcmJvc2l0eT1sb2dnaW5nLklORk8sCiAgICAgICAgbGFiZWxfY29sdW1uX25hbWU9bGFiZWxfY29sdW1uX25hbWUsCiAgICAgICAgKiphdXRvbWxfc2V0dGluZ3MsCiAgICApCgogICAgIyBSdW4gZXhwZXJpbWVudCBvbiBBenVyZU1MOgogICAgY29udGV4dC5sb2dnZXIuaW5mbygiU3VibWl0dGluZyBhbmQgcnVubmluZyBleHBlcmltZW50IikKICAgIHJlbW90ZV9ydW4gPSBleHBlcmltZW50LnN1Ym1pdChhdXRvbWxfY29uZmlnKQogICAgcmVtb3RlX3J1bi53YWl0X2Zvcl9jb21wbGV0aW9uKHNob3dfb3V0cHV0PXNob3dfb3V0cHV0KQogICAgaWYgc2hvd19vdXRwdXQ6CiAgICAgICAgIyBBenVyZSBsb2cgZW5kaW5nIHJvdzoKICAgICAgICBwcmludChmIlxueycqJyAqIDkyfVxuIikKICAgICMgR2V0IHRvcCBOIHJ1bnMgdG8gbG9nOgogICAgdG9wX3J1bnMgPSBfZ2V0X3RvcF9uX3J1bnMoCiAgICAgICAgcmVtb3RlX3J1bj1yZW1vdGVfcnVuLAogICAgICAgIG49c2F2ZV9uX21vZGVscywKICAgICAgICBwcmltYXJ5X21ldHJpYz1hdXRvbWxfc2V0dGluZ3NbInByaW1hcnlfbWV0cmljIl0sCiAgICApCgogICAgIyBSZWdpc3RlciwgZG93bmxvYWQsIGFuZCBsb2cgbW9kZWxzOgogICAgZm9yIGksIHJ1biBpbiBlbnVtZXJhdGUodG9wX3J1bnMpOgogICAgICAgICMgUmVnaXN0ZXIgbW9kZWw6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygiUmVnaXN0ZXJpbmcgbW9kZWwiKQogICAgICAgIG1vZGVsID0gcnVuLnJlZ2lzdGVyX21vZGVsKAogICAgICAgICAgICBtb2RlbF9uYW1lPXJlZ2lzdGVyX21vZGVsX25hbWUsIG1vZGVsX3BhdGg9Im91dHB1dHMvbW9kZWwucGtsIgogICAgICAgICkKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIlJlZ2lzdGVyZWQgbW9kZWwgd2l0aCBuYW1lICd7bW9kZWwubmFtZX0nLCBpZCAne21vZGVsLmlkfScsIHZlcnNpb24gJ3ttb2RlbC52ZXJzaW9ufSciCiAgICAgICAgKQoKICAgICAgICAjIERvd25sb2FkIG1vZGVsIGxvY2FsbHk6CiAgICAgICAgZG93bmxvYWRfbW9kZWwoCiAgICAgICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICAgICAgbW9kZWxfbmFtZT1yZWdpc3Rlcl9tb2RlbF9uYW1lLAogICAgICAgICAgICBtb2RlbF92ZXJzaW9uPW1vZGVsLnZlcnNpb24sCiAgICAgICAgICAgIHRhcmdldF9kaXI9ZiIuL3ttb2RlbC52ZXJzaW9ufSIsCiAgICAgICAgKQoKICAgICAgICBtZXRyaWNzID0ge2subG93ZXIoKTogdmFsIGZvciBrLCB2YWwgaW4gcnVuLmdldF9tZXRyaWNzKCkuaXRlbXMoKX0KICAgICAgICBkZWwgbWV0cmljc1siY29uZnVzaW9uX21hdHJpeCJdCiAgICAgICAgZGVsIG1ldHJpY3NbImFjY3VyYWN5X3RhYmxlIl0KCiAgICAgICAgIyBDb2xsZWN0IG1vZGVsIGh5cGVyLXBhcmFtZXRlcnM6CiAgICAgICAgbW9kZWxfaHBfZGljdCA9IF9nZXRfbW9kZWxfaHAocnVuKQogICAgICAgIHdpdGggY29udGV4dC5nZXRfY2hpbGRfY29udGV4dCgqKm1vZGVsX2hwX2RpY3QpIGFzIGNoaWxkOgogICAgICAgICAgICBtb2RlbF9rZXkgPSBmIm1vZGVsX3tpICsgMX1fe21vZGVsX2hwX2RpY3RbJ2RhdGFfdHJhbnNfY2xhc3NfbmFtZSddLmxvd2VyKCl9X3ttb2RlbF9ocF9kaWN0Wyd0cmFpbl9jbGFzc19uYW1lJ10ubG93ZXIoKX0iCiAgICAgICAgICAgICMgTG9nIG1vZGVsOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiTG9nZ2luZyB7bW9kZWxfa2V5fSBtb2RlbCB0byBNTFJ1biIpCiAgICAgICAgICAgIGNoaWxkLmxvZ19yZXN1bHRzKG1ldHJpY3MpCiAgICAgICAgICAgIGNoaWxkLmxvZ19tb2RlbCgKICAgICAgICAgICAgICAgICJtb2RlbCIsCiAgICAgICAgICAgICAgICBkYl9rZXk9bW9kZWxfa2V5LAogICAgICAgICAgICAgICAgYXJ0aWZhY3RfcGF0aD1jb250ZXh0LmFydGlmYWN0X3N1YnBhdGgoIm1vZGVscyIpLAogICAgICAgICAgICAgICAgbWV0cmljcz1tZXRyaWNzLAogICAgICAgICAgICAgICAgbW9kZWxfZmlsZT1mInttb2RlbC52ZXJzaW9ufS9tb2RlbC5wa2wiLAogICAgICAgICAgICAgICAgdHJhaW5pbmdfc2V0PXRyYWluaW5nX3NldCwKICAgICAgICAgICAgICAgIGxhYmVsX2NvbHVtbj1sYWJlbF9jb2x1bW5fbmFtZSwKICAgICAgICAgICAgICAgIGZlYXR1cmVfdmVjdG9yPWZlYXR1cmVfdmVjdG9yLAogICAgICAgICAgICAgICAgZnJhbWV3b3JrPSJBenVyZU1MIiwKICAgICAgICAgICAgICAgIGFsZ29yaXRobT1tb2RlbF9ocF9kaWN0LmdldCgidHJhaW5fY2xhc3NfbmFtZSIpLAogICAgICAgICAgICApCiAgICAgICAgICAgIGlmIGkgPT0gMDoKICAgICAgICAgICAgICAgICMgVGhpcyBhbHNvIGxvZ3MgdGhlIG1vZGVsOgogICAgICAgICAgICAgICAgY2hpbGQubWFya19hc19iZXN0KCkKCgpkZWYgdHJhaW4oCiAgICAjIE1sUnVuCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIGRhdGFzZXQ6IERhdGFJdGVtLAogICAgIyBJbml0IGV4cGVyaW1lbnQgYW5kIGNvbXB1dGUKICAgIGV4cGVyaW1lbnRfbmFtZTogc3RyID0gIiIsCiAgICBjcHVfY2x1c3Rlcl9uYW1lOiBzdHIgPSAiIiwKICAgIHZtX3NpemU6IHN0ciA9ICJTVEFOREFSRF9EMl9WMiIsCiAgICBtYXhfbm9kZXM6IGludCA9IDEsCiAgICAjIFJlZ2lzdGVyIGRhdGFzZXQKICAgIGRhdGFzZXRfbmFtZTogc3RyID0gIiIsCiAgICBkYXRhc2V0X2Rlc2NyaXB0aW9uOiBzdHIgPSAiIiwKICAgIGNyZWF0ZV9uZXdfdmVyc2lvbjogYm9vbCA9IEZhbHNlLAogICAgbGFiZWxfY29sdW1uX25hbWU6IHN0ciA9ICIiLAogICAgIyBTdWJtaXQgdHJhaW5pbmcgam9iCiAgICByZWdpc3Rlcl9tb2RlbF9uYW1lOiBzdHIgPSAiIiwKICAgIHNhdmVfbl9tb2RlbHM6IGludCA9IDEsCiAgICBsb2dfYXp1cmU6IGJvb2wgPSBUcnVlLAogICAgYXV0b21sX3NldHRpbmdzOiBzdHIgPSBOb25lLAopIC0+IE5vbmU6CiAgICAiIiIKICAgIFdob2xlIHRyYWluaW5nIGZsb3cgZm9yIEF6dXJlIEF1dG9NTC4gUmVnaXN0ZXJzIGRhdGFzZXQvZmVhdHVyZSB2ZWN0b3IsCiAgICBzdWJtaXRzIHRyYWluaW5nIGpvYiB0byBBenVyZSBBdXRvTUwsIGFuZCBkb3dubG9hZHMgdHJhaW5lZCBtb2RlbAogICAgd2hlbiBjb21wbGV0ZWQuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQuCgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgIE1MUnVuIEZlYXR1cmVWZWN0b3Igb3IgZGF0YXNldCBVUkkgdG8gdXBsb2FkLiBXaWxsIGRyb3AKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpbmRleCBiZWZvcmUgdXBsb2FkaW5nIHdoZW4gaXQgaXMgYSBGZWF0dXJlVmVjdG9yLgoKICAgIDpwYXJhbSBleHBlcmltZW50X25hbWU6ICAgICBOYW1lIG9mIGV4cGVyaW1lbnQgdG8gY3JlYXRlIGluIEF6dXJlIE1MLgogICAgOnBhcmFtIGNwdV9jbHVzdGVyX25hbWU6ICAgIE5hbWUgb2YgQXp1cmUgTUwgY29tcHV0ZSB0YXJnZXQuIENyZWF0ZWQgaWYgZG9lcyBub3QgZXhpc3QuCiAgICA6cGFyYW0gdm1fc2l6ZTogICAgICAgICAgICAgQXp1cmUgbWFjaGluZSB0eXBlIGZvciBjb21wdXRlIHRhcmdldC4KICAgIDpwYXJhbSBtYXhfbm9kZXM6ICAgICAgICAgICBNYXhpbXVtIG51bWJlciBvZiBjb25jdXJyZW50IGNvbXB1dGUgdGFyZ2V0cy4KCiAgICA6cGFyYW0gZGF0YXNldF9uYW1lOiAgICAgICAgTmFtZSBvZiBBenVyZSBkYXRhc2V0IHRvIHJlZ2lzdGVyLgogICAgOnBhcmFtIGRhdGFzZXRfZGVzY3JpcHRpb246IERlc2NyaXB0aW9uIG9mIEF6dXJlIGRhdGFzZXQgdG8gcmVnaXN0ZXIuCgogICAgOnBhcmFtIGNyZWF0ZV9uZXdfdmVyc2lvbjogIFJlZ2lzdGVyIEF6dXJlIGRhdGFzZXQgYXMgbmV3IHZlcnNpb24uIE11c3QgYmUgdXNlZCB3aGVuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kaWZ5aW5nIGRhdGFzZXQgc2NoZW1hLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbl9uYW1lOiAgIFRhcmdldCBjb2x1bW4gaW4gZGF0YXNldC4KCiAgICA6cGFyYW0gcmVnaXN0ZXJfbW9kZWxfbmFtZTogTmFtZSBvZiBtb2RlbCB0byByZWdpc3RlciBpbiBBenVyZS4KICAgIDpwYXJhbSBzYXZlX25fbW9kZWxzOiAgICAgICBIb3cgbWFueSBvZiB0aGUgdG9wIHBlcmZvcm1pbmcgbW9kZWxzIHRvIGxvZy4KICAgIDpwYXJhbSBsb2dfYXp1cmU6ICAgICAgICAgICBEaXNwbGF5aW5nIEF6dXJlIGxvZ3MuCiAgICA6cGFyYW0gYXV0b21sX3NldHRpbmdzOiAgICAgSlNPTiBzdHJpbmcgb2YgYWxsIEF6dXJlIEF1dG9NTCBzZXR0aW5ncy4KICAgICIiIgogICAgaWYgbm90IGF1dG9tbF9zZXR0aW5nczoKICAgICAgICBhdXRvbWxfc2V0dGluZ3MgPSB7CiAgICAgICAgICAgICJ0YXNrIjogImNsYXNzaWZpY2F0aW9uIiwKICAgICAgICAgICAgImRlYnVnX2xvZyI6ICJhdXRvbWxfZXJyb3JzLmxvZyIsCiAgICAgICAgICAgICMgImV4cGVyaW1lbnRfZXhpdF9zY29yZSI6IDAuOSwKICAgICAgICAgICAgImVuYWJsZV9lYXJseV9zdG9wcGluZyI6IEZhbHNlLAogICAgICAgICAgICAiYWxsb3dlZF9tb2RlbHMiOiBbIkxvZ2lzdGljUmVncmVzc2lvbiIsICJTR0QiLCAiU1ZNIl0sCiAgICAgICAgICAgICJpdGVyYXRpb25zIjogMywKICAgICAgICAgICAgIml0ZXJhdGlvbl90aW1lb3V0X21pbnV0ZXMiOiAyLAogICAgICAgICAgICAibWF4X2NvbmN1cnJlbnRfaXRlcmF0aW9ucyI6IDIsCiAgICAgICAgICAgICJtYXhfY29yZXNfcGVyX2l0ZXJhdGlvbiI6IC0xLAogICAgICAgICAgICAibl9jcm9zc192YWxpZGF0aW9ucyI6IDUsCiAgICAgICAgICAgICJwcmltYXJ5X21ldHJpYyI6ICJhY2N1cmFjeSIsCiAgICAgICAgICAgICJmZWF0dXJpemF0aW9uIjogIm9mZiIsCiAgICAgICAgICAgICJtb2RlbF9leHBsYWluYWJpbGl0eSI6IEZhbHNlLAogICAgICAgICAgICAiZW5hYmxlX3ZvdGluZ19lbnNlbWJsZSI6IEZhbHNlLAogICAgICAgICAgICAiZW5hYmxlX3N0YWNrX2Vuc2VtYmxlIjogRmFsc2UsCiAgICAgICAgfQoKICAgICMgSW5pdCBleHBlcmltZW50IGFuZCBjb21wdXRlCiAgICB3b3Jrc3BhY2UsIGV4cGVyaW1lbnQgPSBfaW5pdF9leHBlcmltZW50KAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwgZXhwZXJpbWVudF9uYW1lPWV4cGVyaW1lbnRfbmFtZQogICAgKQoKICAgIGNvbXB1dGVfdGFyZ2V0ID0gaW5pdF9jb21wdXRlKAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICBjcHVfY2x1c3Rlcl9uYW1lPWNwdV9jbHVzdGVyX25hbWUsCiAgICAgICAgdm1fc2l6ZT12bV9zaXplLAogICAgICAgIG1heF9ub2Rlcz1tYXhfbm9kZXMsCiAgICApCgogICAgIyBSZWdpc3RlciBkYXRhc2V0CiAgICByZWdpc3Rlcl9kYXRhc2V0KAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICBkYXRhc2V0X25hbWU9ZGF0YXNldF9uYW1lLAogICAgICAgIGRhdGFzZXRfZGVzY3JpcHRpb249ZGF0YXNldF9kZXNjcmlwdGlvbiwKICAgICAgICBkYXRhPWRhdGFzZXQsCiAgICAgICAgY3JlYXRlX25ld192ZXJzaW9uPWNyZWF0ZV9uZXdfdmVyc2lvbiwKICAgICkKCiAgICAjIFN1Ym1pdCB0cmFpbmluZyBqb2IKICAgIHN1Ym1pdF90cmFpbmluZ19qb2IoCiAgICAgICAgY29udGV4dCwKICAgICAgICBleHBlcmltZW50PWV4cGVyaW1lbnQsCiAgICAgICAgY29tcHV0ZV90YXJnZXQ9Y29tcHV0ZV90YXJnZXQsCiAgICAgICAgcmVnaXN0ZXJfbW9kZWxfbmFtZT1yZWdpc3Rlcl9tb2RlbF9uYW1lLAogICAgICAgIHJlZ2lzdGVyZWRfZGF0YXNldF9uYW1lPWRhdGFzZXRfbmFtZSwKICAgICAgICBsYWJlbF9jb2x1bW5fbmFtZT1sYWJlbF9jb2x1bW5fbmFtZSwKICAgICAgICBhdXRvbWxfc2V0dGluZ3M9YXV0b21sX3NldHRpbmdzLAogICAgICAgIHRyYWluaW5nX3NldD1kYXRhc2V0LAogICAgICAgIHNob3dfb3V0cHV0PWxvZ19henVyZSwKICAgICAgICBzYXZlX25fbW9kZWxzPXNhdmVfbl9tb2RlbHMsCiAgICApCg== requirements: - azureml-core==1.54.0.post1 - azureml-train-automl-client==1.54.0.post1 - plotly~=5.23 - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IG9zCmltcG9ydCBqc29uCmltcG9ydCBsb2dnaW5nCmZyb20gdHlwaW5nIGltcG9ydCBUdXBsZSwgTGlzdAoKZnJvbSBtbHJ1biBpbXBvcnQgTUxDbGllbnRDdHgsIERhdGFJdGVtLCBnZXRfZGF0YWl0ZW0KaW1wb3J0IG1scnVuLmZlYXR1cmVfc3RvcmUgYXMgZl9zdG9yZQppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwpmcm9tIG1scnVuLmRhdGFzdG9yZS50YXJnZXRzIGltcG9ydCBQYXJxdWV0VGFyZ2V0Cgpmcm9tIGF6dXJlbWwuY29yZS5hdXRoZW50aWNhdGlvbiBpbXBvcnQgU2VydmljZVByaW5jaXBhbEF1dGhlbnRpY2F0aW9uCmZyb20gYXp1cmVtbC5jb3JlLndvcmtzcGFjZSBpbXBvcnQgV29ya3NwYWNlCmZyb20gYXp1cmVtbC5jb3JlLmV4cGVyaW1lbnQgaW1wb3J0IEV4cGVyaW1lbnQKZnJvbSBhenVyZW1sLmNvcmUuZGF0YXNldCBpbXBvcnQgRGF0YXNldApmcm9tIGF6dXJlbWwuY29yZS5tb2RlbCBpbXBvcnQgTW9kZWwKZnJvbSBhenVyZW1sLmNvcmUuY29tcHV0ZSBpbXBvcnQgQ29tcHV0ZVRhcmdldCwgQW1sQ29tcHV0ZQpmcm9tIGF6dXJlbWwuY29yZS5jb21wdXRlX3RhcmdldCBpbXBvcnQgQ29tcHV0ZVRhcmdldEV4Y2VwdGlvbgpmcm9tIGF6dXJlbWwuY29yZS5zY3JpcHRfcnVuIGltcG9ydCBTY3JpcHRSdW4KCmZyb20gYXp1cmVtbC50cmFpbi5hdXRvbWwgaW1wb3J0IEF1dG9NTENvbmZpZwpmcm9tIGF6dXJlbWwudHJhaW4uYXV0b21sLnJ1biBpbXBvcnQgQXV0b01MUnVuCgoKZGVmIF9lbnZfb3Jfc2VjcmV0KGNvbnRleHQsIGtleSk6CiAgICBpZiBrZXkgaW4gb3MuZW52aXJvbjoKICAgICAgICByZXR1cm4gb3MuZW52aXJvbltrZXldCiAgICByZXR1cm4gY29udGV4dC5nZXRfc2VjcmV0KGtleSkKCgpkZWYgX2xvYWRfd29ya3NwYWNlKGNvbnRleHQ6IE1MQ2xpZW50Q3R4KSAtPiBXb3Jrc3BhY2U6CiAgICAiIiIKICAgIExvYWRpbmcgQXp1cmVNTCBXb3Jrc3BhY2Ugd2l0aCBBenVyZSBzZWNyZXRzLgoKICAgIDpwYXJhbSBjb250ZXh0OiBNTFJ1biBjb250ZXh0LgogICAgOnJldHVybnM6ICAgICAgIEF6dXJlTUwgV29ya3NwYWNlCiAgICAiIiIKCiAgICBpZiBoYXNhdHRyKGNvbnRleHQsICJfYXp1cmVfd29ya3NwYWNlIik6CiAgICAgICAgcmV0dXJuIGNvbnRleHQuX2F6dXJlX3dvcmtzcGFjZQoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkxvYWRpbmcgQXp1cmVNTCBXb3Jrc3BhY2UiKQogICAgIyBBenVyZSBzZXJ2aWNlIGF1dGhlbnRpY2F0aW9uOgogICAgc2VydmljZV9hdXRoZW50aWNhdGlvbiA9IFNlcnZpY2VQcmluY2lwYWxBdXRoZW50aWNhdGlvbigKICAgICAgICB0ZW5hbnRfaWQ9X2Vudl9vcl9zZWNyZXQoY29udGV4dCwgIkFaVVJFX1RFTkFOVF9JRCIpLAogICAgICAgIHNlcnZpY2VfcHJpbmNpcGFsX2lkPV9lbnZfb3Jfc2VjcmV0KGNvbnRleHQsICJBWlVSRV9TRVJWSUNFX1BSSU5DSVBBTF9JRCIpLAogICAgICAgIHNlcnZpY2VfcHJpbmNpcGFsX3Bhc3N3b3JkPV9lbnZfb3Jfc2VjcmV0KAogICAgICAgICAgICBjb250ZXh0LCAiQVpVUkVfU0VSVklDRV9QUklOQ0lQQUxfUEFTU1dPUkQiCiAgICAgICAgKSwKICAgICkKCiAgICAjIExvYWRpbmcgQXp1cmUgd29ya3NwYWNlOgogICAgd29ya3NwYWNlID0gV29ya3NwYWNlKAogICAgICAgIHN1YnNjcmlwdGlvbl9pZD1fZW52X29yX3NlY3JldChjb250ZXh0LCAiQVpVUkVfU1VCU0NSSVBUSU9OX0lEIiksCiAgICAgICAgcmVzb3VyY2VfZ3JvdXA9X2Vudl9vcl9zZWNyZXQoY29udGV4dCwgIkFaVVJFX1JFU09VUkNFX0dST1VQIiksCiAgICAgICAgd29ya3NwYWNlX25hbWU9X2Vudl9vcl9zZWNyZXQoY29udGV4dCwgIkFaVVJFX1dPUktTUEFDRV9OQU1FIiksCiAgICAgICAgYXV0aD1zZXJ2aWNlX2F1dGhlbnRpY2F0aW9uLAogICAgKQoKICAgIGNvbnRleHQuX2F6dXJlX3dvcmtzcGFjZSA9IHdvcmtzcGFjZQogICAgcmV0dXJuIHdvcmtzcGFjZQoKCmRlZiBfaW5pdF9leHBlcmltZW50KAogICAgY29udGV4dDogTUxDbGllbnRDdHgsIGV4cGVyaW1lbnRfbmFtZTogc3RyCikgLT4gVHVwbGVbV29ya3NwYWNlLCBFeHBlcmltZW50XToKICAgICIiIgogICAgSW5pdGlhbGl6ZSB3b3Jrc3BhY2UgYW5kIGV4cGVyaW1lbnQgaW4gQXp1cmUgTUwuIFVzZXMgU2VydmljZQogICAgUHJpbmNpcGFsIGF1dGhlbnRpY2F0aW9uIHZpYSBlbnZpcm9ubWVudCB2YXJpYWJsZXMuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBleHBlcmltZW50X25hbWU6IE5hbWUgb2YgZXhwZXJpbWVudCB0byBjcmVhdGUgaW4gQXp1cmUgTUwuCiAgICA6cmV0dXJuczogICAgICAgICAgICAgICBBenVyZSBNTCBXb3Jrc3BhY2UgYW5kIEV4cGVyaW1lbnQuCiAgICAiIiIKCiAgICAjIEluaXRpYWxpemUgZXhwZXJpbWVudCB2aWEgU2VydmljZSBQcmluY2lwYWwgQXV0aGVudGljYXRpb246CiAgICAjIGh0dHBzOi8vZG9jcy5taWNyb3NvZnQuY29tL2VuLXVzL2F6dXJlL21hY2hpbmUtbGVhcm5pbmcvaG93LXRvLXNldHVwLWF1dGhlbnRpY2F0aW9uI3VzZS1zZXJ2aWNlLXByaW5jaXBhbC1hdXRoZW50aWNhdGlvbgoKICAgIHdvcmtzcGFjZSA9IF9sb2FkX3dvcmtzcGFjZShjb250ZXh0KQoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJJbml0aWFsaXppbmcgQXp1cmVNTCBleHBlcmltZW50IHtleHBlcmltZW50X25hbWV9IikKICAgICMgQ3JlYXRpbmcgZXhwZXJpbWVudDoKICAgIGV4cGVyaW1lbnQgPSBFeHBlcmltZW50KHdvcmtzcGFjZSwgZXhwZXJpbWVudF9uYW1lKQoKICAgIHJldHVybiB3b3Jrc3BhY2UsIGV4cGVyaW1lbnQKCgpkZWYgaW5pdF9jb21wdXRlKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBjcHVfY2x1c3Rlcl9uYW1lOiBzdHIsCiAgICB2bV9zaXplOiBzdHIgPSAiU1RBTkRBUkRfRDJfVjIiLAogICAgbWF4X25vZGVzOiBpbnQgPSAxLAopIC0+IENvbXB1dGVUYXJnZXQ6CiAgICAiIiIKICAgIEluaXRpYWxpemUgQXp1cmUgTUwgY29tcHV0ZSB0YXJnZXQgdG8gcnVuIGV4cGVyaW1lbnQuIENoZWNrcyBmb3IKICAgIGV4aXN0aW5nIGNvbXB1dGUgdGFyZ2V0IGFuZCBjcmVhdGVzIG5ldyBpZiBkb2VzIG5vdCBleGlzdC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBjcHVfY2x1c3Rlcl9uYW1lOiBOYW1lIG9mIEF6dXJlIE1MIGNvbXB1dGUgdGFyZ2V0LiBDcmVhdGVkIGlmIGRvZXMgbm90IGV4aXN0LgogICAgOnBhcmFtIHZtX3NpemU6ICAgICAgICAgIEF6dXJlIG1hY2hpbmUgdHlwZSBmb3IgY29tcHV0ZSB0YXJnZXQuCiAgICA6cGFyYW0gbWF4X25vZGVzOiAgICAgICAgTWF4aW11bSBudW1iZXIgb2YgY29uY3VycmVudCBjb21wdXRlIHRhcmdldHMuCiAgICA6cmV0dXJuczogICAgICAgICAgICAgICAgQXp1cmUgTUwgQ29tcHV0ZSBUYXJnZXQuCiAgICAiIiIKCiAgICB3b3Jrc3BhY2UgPSBfbG9hZF93b3Jrc3BhY2UoY29udGV4dCkKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJJbml0aWFsaXppbmcgQXp1cmVNTCBjb21wdXRlIHRhcmdldCB7Y3B1X2NsdXN0ZXJfbmFtZX0iKQoKICAgICMgVmVyaWZ5IHRoYXQgY2x1c3RlciBkb2VzIG5vdCBleGlzdCBhbHJlYWR5OgogICAgdHJ5OgogICAgICAgIGNvbXB1dGVfdGFyZ2V0ID0gQ29tcHV0ZVRhcmdldCh3b3Jrc3BhY2U9d29ya3NwYWNlLCBuYW1lPWNwdV9jbHVzdGVyX25hbWUpCiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygiRm91bmQgZXhpc3RpbmcgY2x1c3Rlciwgd2lsbCB1c2UgaXQuIikKICAgIGV4Y2VwdCBDb21wdXRlVGFyZ2V0RXhjZXB0aW9uOgogICAgICAgIGNvbXB1dGVfY29uZmlnID0gQW1sQ29tcHV0ZS5wcm92aXNpb25pbmdfY29uZmlndXJhdGlvbigKICAgICAgICAgICAgdm1fc2l6ZT12bV9zaXplLCBtYXhfbm9kZXM9bWF4X25vZGVzCiAgICAgICAgKQogICAgICAgIGNvbXB1dGVfdGFyZ2V0ID0gQ29tcHV0ZVRhcmdldC5jcmVhdGUoCiAgICAgICAgICAgIHdvcmtzcGFjZSwgY3B1X2NsdXN0ZXJfbmFtZSwgY29tcHV0ZV9jb25maWcKICAgICAgICApCgogICAgY29tcHV0ZV90YXJnZXQud2FpdF9mb3JfY29tcGxldGlvbihzaG93X291dHB1dD1UcnVlKQogICAgcmV0dXJuIGNvbXB1dGVfdGFyZ2V0CgoKZGVmIHJlZ2lzdGVyX2RhdGFzZXQoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIGRhdGFzZXRfbmFtZTogc3RyLAogICAgZGF0YXNldF9kZXNjcmlwdGlvbjogc3RyLAogICAgZGF0YTogRGF0YUl0ZW0sCiAgICBjcmVhdGVfbmV3X3ZlcnNpb246IGJvb2wgPSBGYWxzZSwKKToKICAgICIiIgogICAgUmVnaXN0ZXIgZGF0YXNldCBvYmplY3QgKGNhbiBiZSBhbHNvIGFuIElndWF6aW8gRmVhdHVyZVZlY3RvcikgaW4gQXp1cmUgTUwuCiAgICBVcGxvYWRzIHBhcnF1ZXQgZmlsZSB0byBBenVyZSBibG9iIHN0b3JhZ2UgYW5kIHJlZ2lzdGVycwogICAgdGhhdCBmaWxlIGFzIGEgZGF0YXNldCBpbiBBenVyZSBNTC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIGRhdGFzZXRfbmFtZTogICAgICAgICAgTmFtZSBvZiBBenVyZSBkYXRhc2V0IHRvIHJlZ2lzdGVyLgogICAgOnBhcmFtIGRhdGFzZXRfZGVzY3JpcHRpb246ICAgRGVzY3JpcHRpb24gb2YgQXp1cmUgZGF0YXNldCB0byByZWdpc3Rlci4KICAgIDpwYXJhbSBkYXRhOiAgICAgICAgICAgICAgICAgIE1MUnVuIEZlYXR1cmVWZWN0b3Igb3IgZGF0YXNldCBvYmplY3QgdG8gdXBsb2FkLgogICAgOnBhcmFtIGNyZWF0ZV9uZXdfdmVyc2lvbjogICAgUmVnaXN0ZXIgQXp1cmUgZGF0YXNldCBhcyBuZXcgdmVyc2lvbi4gTXVzdCBiZSB1c2VkIHdoZW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGlmeWluZyBkYXRhc2V0IHNjaGVtYS4KICAgICIiIgoKICAgICMgdGVzdCBmb3IgQXp1cmUgc3RvcmFnZSBjb25uZWN0aW9uIGVudmlyb25tZW50IHZhcmlhYmxlIG9yIHNlY3JldDoKICAgIGFzc2VydCBfZW52X29yX3NlY3JldCgKICAgICAgICBjb250ZXh0LCAiQVpVUkVfU1RPUkFHRV9DT05ORUNUSU9OX1NUUklORyIKICAgICksICJBWlVSRV9TVE9SQUdFX0NPTk5FQ1RJT05fU1RSSU5HIHNlY3JldCBub3Qgc2V0IgoKICAgICMgQ29ubmVjdCB0byBBenVyZU1MIGV4cGVyaW1lbnQgYW5kIGRhdGFzdG9yZToKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkNvbm5lY3RpbmcgdG8gQXp1cmVNTCBleHBlcmltZW50IGRlZmF1bHQgZGF0YXN0b3JlIikKCiAgICB3b3Jrc3BhY2UgPSBfbG9hZF93b3Jrc3BhY2UoY29udGV4dCkKICAgIGRhdGFzdG9yZSA9IHdvcmtzcGFjZS5nZXRfZGVmYXVsdF9kYXRhc3RvcmUoKQoKICAgICMgQXp1cmUgYmxvYiBwYXRoIChkZWZhdWx0IGRhdGFzdG9yZSBmb3Igd29ya3NwYWNlKToKICAgIGJsb2JfcGF0aCA9IGYiYXo6Ly97ZGF0YXN0b3JlLmNvbnRhaW5lcl9uYW1lfS97ZGF0YXNldF9uYW1lfSIKCiAgICBzdG9yZV91cmlfcHJlZml4LCBfID0gbWxydW4uZGF0YXN0b3JlLnBhcnNlX3N0b3JlX3VyaShkYXRhLmFydGlmYWN0X3VybCkKICAgIGZlYXR1cmVfdmVjdG9yX2Nhc2UgPSBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXgKICAgICMgUmV0cmlldmUgZGF0YSBzb3VyY2UgYXMgZGF0YWZyYW1lOgogICAgaWYgZmVhdHVyZV92ZWN0b3JfY2FzZToKICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIlJldHJpZXZpbmcgZmVhdHVyZSB2ZWN0b3IgYW5kIHVwbG9hZGluZyB0byBBenVyZSBibG9iIHN0b3JhZ2U6IHtibG9iX3BhdGh9IgogICAgICAgICkKICAgICAgICBmX3N0b3JlLmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRhdGEubWV0YS51cmksIHRhcmdldD1QYXJxdWV0VGFyZ2V0KHBhdGg9YmxvYl9wYXRoKSkKICAgIGVsc2U6CiAgICAgICAgYmxvYl9wYXRoICs9IGRhdGEuc3VmZml4CiAgICAgICAgIyBEYXRhSXRlbSBjYXNlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgIGYiUmV0cmlldmluZyBmZWF0dXJlIHZlY3RvciBhbmQgdXBsb2FkaW5nIHRvIEF6dXJlIGJsb2Igc3RvcmFnZToge2Jsb2JfcGF0aH0iCiAgICAgICAgKQogICAgICAgIGRhdGFfaW5fYnl0ZXMgPSBkYXRhLmdldCgpCiAgICAgICAgZ2V0X2RhdGFpdGVtKGJsb2JfcGF0aCkucHV0KGRhdGFfaW5fYnl0ZXMpCgogICAgIyBSZWdpc3RlciBkYXRhc2V0IGluIEF6dXJlTUw6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiUmVnaXN0ZXJpbmcgZGF0YXNldCB7ZGF0YXNldF9uYW1lfSBpbiBBenVyZSBNTCIpCiAgICBpZiBkYXRhLnN1ZmZpeCA9PSAiLnBhcnF1ZXQiIG9yIGZlYXR1cmVfdmVjdG9yX2Nhc2U6CiAgICAgICAgZGF0YXNldCA9IERhdGFzZXQuVGFidWxhci5mcm9tX3BhcnF1ZXRfZmlsZXMoCiAgICAgICAgICAgIHBhdGg9KGRhdGFzdG9yZSwgZiJ7ZGF0YXNldF9uYW1lfS5wYXJxdWV0IiksIHZhbGlkYXRlPUZhbHNlCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIk9wZW5TU0wgdmVyc2lvbiBtdXN0IGJlIDEuMS4gT3ZlcnJpZGluZyB0aGUgT3BlblNTTCB2ZXJzaW9uIHRvIDEuMSIKICAgICAgICApCiAgICAgICAgIyBPcGVuU1NMIHZlcnNpb24gbXVzdCBiZSAxLjEKICAgICAgICBvcy5lbnZpcm9uWyJDTFJfT1BFTlNTTF9WRVJTSU9OX09WRVJSSURFIl0gPSAiMS4xIgogICAgICAgIGRhdGFzZXQgPSBEYXRhc2V0LlRhYnVsYXIuZnJvbV9kZWxpbWl0ZWRfZmlsZXMoCiAgICAgICAgICAgIHBhdGg9KGRhdGFzdG9yZSwgZiJ7ZGF0YXNldF9uYW1lfXtkYXRhLnN1ZmZpeH0iKSwgdmFsaWRhdGU9RmFsc2UKICAgICAgICApCgogICAgZGF0YXNldC5yZWdpc3RlcigKICAgICAgICB3b3Jrc3BhY2U9d29ya3NwYWNlLAogICAgICAgIG5hbWU9ZGF0YXNldF9uYW1lLAogICAgICAgIGRlc2NyaXB0aW9uPWRhdGFzZXRfZGVzY3JpcHRpb24sCiAgICAgICAgY3JlYXRlX25ld192ZXJzaW9uPWNyZWF0ZV9uZXdfdmVyc2lvbiwKICAgICkKCiAgICAjIE91dHB1dCByZWdpc3RlcmVkIGRhdGFzZXQgbmFtZSBpbiBBenVyZToKICAgIGNvbnRleHQubG9nX3Jlc3VsdCgiZGF0YXNldF9ibG9iX3BhdGgiLCBibG9iX3BhdGgpCgoKZGVmIGRvd25sb2FkX21vZGVsKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBtb2RlbF9uYW1lOiBzdHIsCiAgICBtb2RlbF92ZXJzaW9uOiBpbnQsCiAgICB0YXJnZXRfZGlyOiBzdHIgPSAiLiIsCikgLT4gTm9uZToKICAgICIiIgogICAgRG93bmxvYWQgdHJhaW5lZCBtb2RlbCBmcm9tIEF6dXJlIE1MIHRvIGxvY2FsIGZpbGVzeXN0ZW0uCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gbW9kZWxfbmFtZTogICAgTmFtZSBvZiB0cmFpbmVkIGFuZCByZWdpc3RlcmVkIG1vZGVsLgogICAgOnBhcmFtIG1vZGVsX3ZlcnNpb246IFZlcnNpb24gb2YgbW9kZWwgdG8gZG93bmxvYWQuCiAgICA6cGFyYW0gdGFyZ2V0X2RpcjogICAgVGFyZ2V0IGRpcmVjdG9yeSB0byBkb3dubG9hZCBtb2RlbC4KICAgICIiIgogICAgIyBMb2FkaW5nIHdvcmtzcGFjZSBpZiBub3QgcHJvdmlkZWQ6CiAgICB3b3Jrc3BhY2UgPSBfbG9hZF93b3Jrc3BhY2UoY29udGV4dCkKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJEb3dubG9hZGluZyBtb2RlbCB7bW9kZWxfbmFtZX06e21vZGVsX3ZlcnNpb259IikKICAgIG1vZGVsID0gTW9kZWwod29ya3NwYWNlLCBtb2RlbF9uYW1lLCB2ZXJzaW9uPW1vZGVsX3ZlcnNpb24pCiAgICBtb2RlbC5kb3dubG9hZCh0YXJnZXRfZGlyPXRhcmdldF9kaXIsIGV4aXN0X29rPVRydWUpCgoKZGVmIHVwbG9hZF9tb2RlbCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWxfbmFtZTogc3RyLAogICAgbW9kZWxfcGF0aDogc3RyLAogICAgbW9kZWxfZGVzY3JpcHRpb246IHN0ciA9IE5vbmUsCiAgICBtb2RlbF90YWdzOiBkaWN0ID0gTm9uZSwKKSAtPiBOb25lOgogICAgIiIiCiAgICBVcGxvYWQgcHJlLXRyYWluZWQgbW9kZWwgZnJvbSBsb2NhbCBmaWxlc3lzdGVtIHRvIEF6dXJlIE1MLgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICBOYW1lIG9mIHRyYWluZWQgYW5kIHJlZ2lzdGVyZWQgbW9kZWwuCiAgICA6cGFyYW0gbW9kZWxfcGF0aDogICAgICAgIFBhdGggdG8gZmlsZSBvbiBsb2NhbCBmaWxlc3lzdGVtLgogICAgOnBhcmFtIG1vZGVsX2Rlc2NyaXB0aW9uOiBEZXNjcmlwdGlvbiBvZiBtb2RlbHMuCiAgICA6cGFyYW0gbW9kZWxfdGFnczogICAgICAgIEtWIHBhaXJzIG9mIG1vZGVsIHRhZ3MuCiAgICAiIiIKICAgICMgTG9hZGluZyB3b3Jrc3BhY2UgaWYgbm90IHByb3ZpZGVkOgogICAgd29ya3NwYWNlID0gX2xvYWRfd29ya3NwYWNlKGNvbnRleHQpCgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIlVwbG9hZCBtb2RlbCB7bW9kZWxfbmFtZX0gZnJvbSB7bW9kZWxfcGF0aH0iKQogICAgTW9kZWwucmVnaXN0ZXIoCiAgICAgICAgd29ya3NwYWNlPXdvcmtzcGFjZSwKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsX3BhdGgsCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGRlc2NyaXB0aW9uPW1vZGVsX2Rlc2NyaXB0aW9uLAogICAgICAgIHRhZ3M9bW9kZWxfdGFncywKICAgICkKCgpkZWYgX2dldF90b3Bfbl9ydW5zKAogICAgcmVtb3RlX3J1bjogQXV0b01MUnVuLCBuOiBpbnQgPSA1LCBwcmltYXJ5X21ldHJpYzogc3RyID0gImFjY3VyYWN5IgopIC0+IExpc3RbU2NyaXB0UnVuXToKICAgICIiIgogICAgR2V0IHRvcCBOIGNvbXBsZXRlIHJ1bnMgZnJvbSBleHBlcmltZW50IHNvcnRlZCBieSBwcmltYXJ5IG1ldHJpYy4KCiAgICA6cGFyYW0gcmVtb3RlX3J1bjogICAgIEF6dXJlIE1MIFJ1bi4KICAgIDpwYXJhbSBuOiAgICAgICAgICAgICAgTnVtYmVyIG9mIHRvcCBydW5zIHRvIHJldHVybi4KICAgIDpwYXJhbSBwcmltYXJ5X21ldHJpYzogTWV0cmljIHRvIHNvcnQgYnkuCgogICAgOnJldHVybnM6ICAgICAgICAgICAgICBMaXN0IG9mIHRvcCBOIHJ1bnMgc29ydGVkIGJ5IHByaW1hcnkgbWV0cmljLgogICAgIiIiCiAgICAjIENvbGxlY3QgYWxsIG1vZGVsczoKICAgIGNvbXBsZXRlX3J1bnMgPSBbCiAgICAgICAgcnVuCiAgICAgICAgZm9yIHJ1biBpbiByZW1vdGVfcnVuLmdldF9jaGlsZHJlbihzdGF0dXM9IkNvbXBsZXRlZCIpCiAgICAgICAgaWYgbm90IGFueShzIGluIHJ1bi5pZCBmb3IgcyBpbiBbInNldHVwIiwgIndvcmtlciJdKQogICAgXQoKICAgICMgQ2hlY2tpbmcgdGhhdCB0aGUgcmVxdWlyZWQgbnVtYmVyIG9mIHJ1bnMgYXJlIGRvbmU6CiAgICBpZiBsZW4oY29tcGxldGVfcnVucykgPCBuOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoZiJFeHBlY3RlZCB7bn0gcnVucyBidXQgb25seSByZWNlaXZlZCB7bGVuKGNvbXBsZXRlX3J1bnMpfSIpCgogICAgIyBTb3J0aW5nIGJ5IHRoZSBwcmltYXJ5IG1ldHJpYzoKICAgIHNvcnRlZF9ydW5zID0gc29ydGVkKAogICAgICAgIGNvbXBsZXRlX3J1bnMsIGtleT1sYW1iZGEgcnVuOiBydW4uZ2V0X21ldHJpY3MoKVtwcmltYXJ5X21ldHJpY10sIHJldmVyc2U9VHJ1ZQogICAgKQogICAgcmV0dXJuIHNvcnRlZF9ydW5zWzpuXQoKCmRlZiBfZ2V0X21vZGVsX2hwKAogICAgcnVuOiBTY3JpcHRSdW4sCikgLT4gZGljdDoKICAgICIiIgogICAgR2V0IGh5cGVyLXBhcmFtZXRlcnMgb2YgdHJhaW5lZCBBenVyZU1MIG1vZGVsLgogICAgQ29tYmluZSB0aGUgaHlwZXItcGFyYW1ldGVycyBvZiB0aGUgZGF0YSB0cmFuc2Zvcm1hdGlvbiBhbmQgdHJhaW5pbmcgdG8gYSBkaWN0aW9uYXJ5LgogICAgVGhlIHByZWZpeCBvZiB0aGUgZGljdGlvbmFyeSBrZXlzIGNvcnJlc3BvbmRzIHRvICdkYXRhIHRyYW5zZm9ybWF0aW9uJyBhbmQgJ3RyYWluaW5nJy4KCiAgICA6cGFyYW0gcnVuOiBSdW4gb2JqZWN0IG9mIEF6dXJlTUwgdHJhaW5lZCBtb2RlbC4KCiAgICA6cmV0dXJuczogICAgQSBkaWN0aW9uYXJ5IGFzIGRlc2NyaWJlZCBpbiB0aGUgZG9jc3RyaW5nLgogICAgIiIiCgogICAgc3BlY19maWVsZCA9ICJwaXBlbGluZV9zcGVjIgogICAgaWYgc3BlY19maWVsZCBub3QgaW4gcnVuLnByb3BlcnRpZXM6CiAgICAgICAgcmV0dXJuIHt9CiAgICBzcGVjX3N0cmluZyA9IHJ1bi5wcm9wZXJ0aWVzW3NwZWNfZmllbGRdCiAgICBzcGVjX2RpY3QgPSBqc29uLmxvYWRzKHNwZWNfc3RyaW5nKQoKICAgIGlmICJvYmplY3RzIiBub3QgaW4gc3BlY19kaWN0OgogICAgICAgICMgTm8gaHlwZXItcGFyYW1zCiAgICAgICAgcmV0dXJuIHt9CiAgICBocF9kaWN0cyA9IHNwZWNfZGljdFsib2JqZWN0cyJdCiAgICAjIGFmdGVyIHRyYWluaW5nIHRoZXJlIGFyZSB0d28gaHlwZXItcGFyYW1ldGVycyBkaWN0cyBpbnNpZGUgdGhlIHJ1biBvYmplY3Q6CiAgICBhc3NlcnQgKAogICAgICAgIGxlbihocF9kaWN0cykgPT0gMgogICAgKSwgImFmdGVyIHRyYWluaW5nIHRoZXJlIGFyZSB0d28gaHlwZXItcGFyYW1ldGVycyBkaWN0cyBpbnNpZGUgdGhlIHJ1biBvYmplY3QiCiAgICByZXN1bHRfZGljdCA9IHt9CiAgICBkaWN0X2tleXMgPSBbCiAgICAgICAgWyJkYXRhX3RyYW5zX2NsYXNzX25hbWUiLCAiZGF0YV90cmFuc19tb2R1bGUiLCAiZGF0YV90cmFuc19zcGVjX2NsYXNzIl0sCiAgICAgICAgWwogICAgICAgICAgICAidHJhaW5fY2xhc3NfbmFtZSIsCiAgICAgICAgICAgICJ0cmFpbl9tb2R1bGUiLAogICAgICAgICAgICAidHJhaW5fcGFyYW1fa3dhcmdzX0MiLAogICAgICAgICAgICAidHJhaW5fcGFyYW1fa3dhcmdzX2NsYXNzX3dlaWdodCIsCiAgICAgICAgICAgICJ0cmFpbl9zcGVjX2NsYXNzIiwKICAgICAgICBdLAogICAgXQoKICAgICMgY3JlYXRpbmcgaHlwZXItcGFyYW1zIGRpY3Qgd2l0aCBrZXkgcHJlZml4ZXMgZm9yIGVhY2ggcGFydDoKICAgIGt3YXJnc19wcmVmaXggPSAicGFyYW1fa3dhcmdzIgogICAgZm9yIGQsIG5hbWUsIGtleXMgaW4gemlwKGhwX2RpY3RzLCBbImRhdGFfdHJhbnMiLCAidHJhaW4iXSwgZGljdF9rZXlzKToKICAgICAgICBmb3Iga2V5IGluIGtleXM6CgogICAgICAgICAgICBpZiBrd2FyZ3NfcHJlZml4IGluIGtleToKICAgICAgICAgICAgICAgIHJlc3VsdF9kaWN0W2tleV0gPSBkW2t3YXJnc19wcmVmaXhdWwogICAgICAgICAgICAgICAgICAgIGtleS5yZXBsYWNlKGYie25hbWV9X3trd2FyZ3NfcHJlZml4fV8iLCAiIikKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIHJlc3VsdF9kaWN0W2tleV0gPSBkW2tleS5yZXBsYWNlKGYie25hbWV9XyIsICIiKV0KICAgICAgICAgICAgaWYgbm90IHJlc3VsdF9kaWN0W2tleV06CiAgICAgICAgICAgICAgICByZXN1bHRfZGljdFtrZXldID0gIiIKCiAgICByZXR1cm4gcmVzdWx0X2RpY3QKCgpkZWYgc3VibWl0X3RyYWluaW5nX2pvYigKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZXhwZXJpbWVudDogRXhwZXJpbWVudCwKICAgIGNvbXB1dGVfdGFyZ2V0OiBDb21wdXRlVGFyZ2V0LAogICAgcmVnaXN0ZXJfbW9kZWxfbmFtZTogc3RyLAogICAgcmVnaXN0ZXJlZF9kYXRhc2V0X25hbWU6IHN0ciwKICAgIGF1dG9tbF9zZXR0aW5nczogZGljdCwKICAgIHRyYWluaW5nX3NldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5fbmFtZTogc3RyID0gJycsCiAgICBzYXZlX25fbW9kZWxzOiBpbnQgPSAzLAogICAgc2hvd19vdXRwdXQ6IGJvb2wgPSBUcnVlLAopIC0+IE5vbmU6CiAgICAiIiIKICAgIFN1Ym1pdCB0cmFpbmluZyBqb2IgdG8gQXp1cmUgQXV0b01MIGFuZCBkb3dubG9hZCB0cmFpbmVkIG1vZGVsCiAgICB3aGVuIGNvbXBsZXRlZC4gVXNlcyBwcmV2aW91c2x5IHJlZ2lzdGVyZWQgZGF0YXNldCBmb3IgdHJhaW5pbmcuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIGV4cGVyaW1lbnQ6ICAgICAgICAgICAgICBBenVyZSBleHBlcmltZW50LgogICAgOnBhcmFtIGNvbXB1dGVfdGFyZ2V0OiAgICAgICAgICBBenVyZSBjb21wdXRlIHRhcmdldC4KICAgIDpwYXJhbSByZWdpc3Rlcl9tb2RlbF9uYW1lOiAgICAgTmFtZSBvZiBtb2RlbCB0byByZWdpc3RlciBpbiBBenVyZS4KICAgIDpwYXJhbSByZWdpc3RlcmVkX2RhdGFzZXRfbmFtZTogTmFtZSBvZiBkYXRhc2V0IHJlZ2lzdGVyZWQgaW4gQXp1cmUgTUwuCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uX25hbWU6ICAgICAgIE5hbWUgb2YgdGFyZ2V0IGNvbHVtbiBpbiBkYXRhc2V0LgogICAgOnBhcmFtIGF1dG9tbF9zZXR0aW5nczogICAgICAgICBKU09OIHN0cmluZyBvZiBhbGwgQXp1cmUgQXV0b01MIHNldHRpbmdzLgogICAgOnBhcmFtIHRyYWluaW5nX3NldDogICAgICAgICAgICBUcmFpbmluZyBzZXQgdG8gbG9nIHdpdGggbW9kZWwuIEZvciBtb2RlbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb25pdG9yaW5nIGludGVncmF0aW9uLgogICAgOnBhcmFtIHNob3dfb3V0cHV0OiAgICAgICAgICAgICBEaXNwbGF5aW5nIEF6dXJlIGxvZ3MuCiAgICA6cGFyYW0gc2F2ZV9uX21vZGVsczogICAgICAgICAgIEhvdyBtYW55IG9mIHRoZSB0b3AgcGVyZm9ybWluZyBtb2RlbHMgdG8gbG9nLgogICAgIiIiCiAgICAjIExvYWRpbmcgd29ya3NwYWNlIGlmIG5vdCBwcm92aWRlZDoKICAgIHdvcmtzcGFjZSA9IF9sb2FkX3dvcmtzcGFjZShjb250ZXh0KQoKICAgICMgU2V0dXAgZXhwZXJpbWVudDoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIlNldHRpbmcgdXAgZXhwZXJpbWVudCBwYXJhbWV0ZXJzIikKICAgIGRhdGFzZXQgPSBEYXRhc2V0LmdldF9ieV9uYW1lKHdvcmtzcGFjZSwgbmFtZT1yZWdpc3RlcmVkX2RhdGFzZXRfbmFtZSkKCiAgICAjIEdldCB0cmFpbmluZyBzZXQgdG8gbG9nIHdpdGggbW9kZWw6CiAgICBmZWF0dXJlX3ZlY3RvciA9IE5vbmUKICAgIHN0b3JlX3VyaV9wcmVmaXgsIF8gPSBtbHJ1bi5kYXRhc3RvcmUucGFyc2Vfc3RvcmVfdXJpKHRyYWluaW5nX3NldC5hcnRpZmFjdF91cmwpCiAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgZmVhdHVyZV92ZWN0b3IgPSB0cmFpbmluZ19zZXQubWV0YS51cmkKICAgICAgICBsYWJlbF9jb2x1bW5fbmFtZSA9IGxhYmVsX2NvbHVtbl9uYW1lIG9yIHRyYWluaW5nX3NldC5tZXRhLnN0YXR1cy5sYWJlbF9jb2x1bW4KICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYnbGFiZWwgY29sdW1uIG5hbWU6IHtsYWJlbF9jb2x1bW5fbmFtZX0nKQogICAgICAgIHRyYWluaW5nX3NldCA9IGZfc3RvcmUuZ2V0X29mZmxpbmVfZmVhdHVyZXMoZmVhdHVyZV92ZWN0b3IpLnRvX2RhdGFmcmFtZSgpCiAgICBlbHNlOgogICAgICAgIHRyYWluaW5nX3NldCA9IHRyYWluaW5nX3NldC5hc19kZigpCgogICAgYXV0b21sX2NvbmZpZyA9IEF1dG9NTENvbmZpZygKICAgICAgICBjb21wdXRlX3RhcmdldD1jb21wdXRlX3RhcmdldCwKICAgICAgICB0cmFpbmluZ19kYXRhPWRhdGFzZXQsCiAgICAgICAgdmVyYm9zaXR5PWxvZ2dpbmcuSU5GTywKICAgICAgICBsYWJlbF9jb2x1bW5fbmFtZT1sYWJlbF9jb2x1bW5fbmFtZSwKICAgICAgICAqKmF1dG9tbF9zZXR0aW5ncywKICAgICkKCiAgICAjIFJ1biBleHBlcmltZW50IG9uIEF6dXJlTUw6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJTdWJtaXR0aW5nIGFuZCBydW5uaW5nIGV4cGVyaW1lbnQiKQogICAgcmVtb3RlX3J1biA9IGV4cGVyaW1lbnQuc3VibWl0KGF1dG9tbF9jb25maWcpCiAgICByZW1vdGVfcnVuLndhaXRfZm9yX2NvbXBsZXRpb24oc2hvd19vdXRwdXQ9c2hvd19vdXRwdXQpCiAgICBpZiBzaG93X291dHB1dDoKICAgICAgICAjIEF6dXJlIGxvZyBlbmRpbmcgcm93OgogICAgICAgIHByaW50KGYiXG57JyonICogOTJ9XG4iKQogICAgIyBHZXQgdG9wIE4gcnVucyB0byBsb2c6CiAgICB0b3BfcnVucyA9IF9nZXRfdG9wX25fcnVucygKICAgICAgICByZW1vdGVfcnVuPXJlbW90ZV9ydW4sCiAgICAgICAgbj1zYXZlX25fbW9kZWxzLAogICAgICAgIHByaW1hcnlfbWV0cmljPWF1dG9tbF9zZXR0aW5nc1sicHJpbWFyeV9tZXRyaWMiXSwKICAgICkKCiAgICAjIFJlZ2lzdGVyLCBkb3dubG9hZCwgYW5kIGxvZyBtb2RlbHM6CiAgICBmb3IgaSwgcnVuIGluIGVudW1lcmF0ZSh0b3BfcnVucyk6CiAgICAgICAgIyBSZWdpc3RlciBtb2RlbDoKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJSZWdpc3RlcmluZyBtb2RlbCIpCiAgICAgICAgbW9kZWwgPSBydW4ucmVnaXN0ZXJfbW9kZWwoCiAgICAgICAgICAgIG1vZGVsX25hbWU9cmVnaXN0ZXJfbW9kZWxfbmFtZSwgbW9kZWxfcGF0aD0ib3V0cHV0cy9tb2RlbC5wa2wiCiAgICAgICAgKQogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgIGYiUmVnaXN0ZXJlZCBtb2RlbCB3aXRoIG5hbWUgJ3ttb2RlbC5uYW1lfScsIGlkICd7bW9kZWwuaWR9JywgdmVyc2lvbiAne21vZGVsLnZlcnNpb259JyIKICAgICAgICApCgogICAgICAgICMgRG93bmxvYWQgbW9kZWwgbG9jYWxseToKICAgICAgICBkb3dubG9hZF9tb2RlbCgKICAgICAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgICAgICBtb2RlbF9uYW1lPXJlZ2lzdGVyX21vZGVsX25hbWUsCiAgICAgICAgICAgIG1vZGVsX3ZlcnNpb249bW9kZWwudmVyc2lvbiwKICAgICAgICAgICAgdGFyZ2V0X2Rpcj1mIi4ve21vZGVsLnZlcnNpb259IiwKICAgICAgICApCgogICAgICAgIG1ldHJpY3MgPSB7ay5sb3dlcigpOiB2YWwgZm9yIGssIHZhbCBpbiBydW4uZ2V0X21ldHJpY3MoKS5pdGVtcygpfQogICAgICAgIGRlbCBtZXRyaWNzWyJjb25mdXNpb25fbWF0cml4Il0KICAgICAgICBkZWwgbWV0cmljc1siYWNjdXJhY3lfdGFibGUiXQoKICAgICAgICAjIENvbGxlY3QgbW9kZWwgaHlwZXItcGFyYW1ldGVyczoKICAgICAgICBtb2RlbF9ocF9kaWN0ID0gX2dldF9tb2RlbF9ocChydW4pCiAgICAgICAgd2l0aCBjb250ZXh0LmdldF9jaGlsZF9jb250ZXh0KCoqbW9kZWxfaHBfZGljdCkgYXMgY2hpbGQ6CiAgICAgICAgICAgIG1vZGVsX2tleSA9IGYibW9kZWxfe2kgKyAxfV97bW9kZWxfaHBfZGljdFsnZGF0YV90cmFuc19jbGFzc19uYW1lJ10ubG93ZXIoKX1fe21vZGVsX2hwX2RpY3RbJ3RyYWluX2NsYXNzX25hbWUnXS5sb3dlcigpfSIKICAgICAgICAgICAgIyBMb2cgbW9kZWw6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICBmIkxvZ2dpbmcge21vZGVsX2tleX0gbW9kZWwgdG8gTUxSdW4iCiAgICAgICAgICAgICkKICAgICAgICAgICAgY2hpbGQubG9nX3Jlc3VsdHMobWV0cmljcykKICAgICAgICAgICAgY2hpbGQubG9nX21vZGVsKAogICAgICAgICAgICAgICAgIm1vZGVsIiwKICAgICAgICAgICAgICAgIGRiX2tleT1tb2RlbF9rZXksCiAgICAgICAgICAgICAgICBhcnRpZmFjdF9wYXRoPWNvbnRleHQuYXJ0aWZhY3Rfc3VicGF0aCgibW9kZWxzIiksCiAgICAgICAgICAgICAgICBtZXRyaWNzPW1ldHJpY3MsCiAgICAgICAgICAgICAgICBtb2RlbF9maWxlPWYie21vZGVsLnZlcnNpb259L21vZGVsLnBrbCIsCiAgICAgICAgICAgICAgICB0cmFpbmluZ19zZXQ9dHJhaW5pbmdfc2V0LAogICAgICAgICAgICAgICAgbGFiZWxfY29sdW1uPWxhYmVsX2NvbHVtbl9uYW1lLAogICAgICAgICAgICAgICAgZmVhdHVyZV92ZWN0b3I9ZmVhdHVyZV92ZWN0b3IsCiAgICAgICAgICAgICAgICBmcmFtZXdvcms9IkF6dXJlTUwiLAogICAgICAgICAgICAgICAgYWxnb3JpdGhtPW1vZGVsX2hwX2RpY3QuZ2V0KCJ0cmFpbl9jbGFzc19uYW1lIiksCiAgICAgICAgICAgICkKICAgICAgICAgICAgaWYgaSA9PSAwOgogICAgICAgICAgICAgICAgIyBUaGlzIGFsc28gbG9ncyB0aGUgbW9kZWw6CiAgICAgICAgICAgICAgICBjaGlsZC5tYXJrX2FzX2Jlc3QoKQoKCmRlZiB0cmFpbigKICAgICMgTWxSdW4KICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICAjIEluaXQgZXhwZXJpbWVudCBhbmQgY29tcHV0ZQogICAgZXhwZXJpbWVudF9uYW1lOiBzdHIgPSAiIiwKICAgIGNwdV9jbHVzdGVyX25hbWU6IHN0ciA9ICIiLAogICAgdm1fc2l6ZTogc3RyID0gIlNUQU5EQVJEX0QyX1YyIiwKICAgIG1heF9ub2RlczogaW50ID0gMSwKICAgICMgUmVnaXN0ZXIgZGF0YXNldAogICAgZGF0YXNldF9uYW1lOiBzdHIgPSAiIiwKICAgIGRhdGFzZXRfZGVzY3JpcHRpb246IHN0ciA9ICIiLAogICAgY3JlYXRlX25ld192ZXJzaW9uOiBib29sID0gRmFsc2UsCiAgICBsYWJlbF9jb2x1bW5fbmFtZTogc3RyID0gIiIsCiAgICAjIFN1Ym1pdCB0cmFpbmluZyBqb2IKICAgIHJlZ2lzdGVyX21vZGVsX25hbWU6IHN0ciA9ICIiLAogICAgc2F2ZV9uX21vZGVsczogaW50ID0gMSwKICAgIGxvZ19henVyZTogYm9vbCA9IFRydWUsCiAgICBhdXRvbWxfc2V0dGluZ3M6IHN0ciA9IE5vbmUsCikgLT4gTm9uZToKICAgICIiIgogICAgV2hvbGUgdHJhaW5pbmcgZmxvdyBmb3IgQXp1cmUgQXV0b01MLiBSZWdpc3RlcnMgZGF0YXNldC9mZWF0dXJlIHZlY3RvciwKICAgIHN1Ym1pdHMgdHJhaW5pbmcgam9iIHRvIEF6dXJlIEF1dG9NTCwgYW5kIGRvd25sb2FkcyB0cmFpbmVkIG1vZGVsCiAgICB3aGVuIGNvbXBsZXRlZC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgTUxSdW4gRmVhdHVyZVZlY3RvciBvciBkYXRhc2V0IFVSSSB0byB1cGxvYWQuIFdpbGwgZHJvcAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGluZGV4IGJlZm9yZSB1cGxvYWRpbmcgd2hlbiBpdCBpcyBhIEZlYXR1cmVWZWN0b3IuCgogICAgOnBhcmFtIGV4cGVyaW1lbnRfbmFtZTogICAgIE5hbWUgb2YgZXhwZXJpbWVudCB0byBjcmVhdGUgaW4gQXp1cmUgTUwuCiAgICA6cGFyYW0gY3B1X2NsdXN0ZXJfbmFtZTogICAgTmFtZSBvZiBBenVyZSBNTCBjb21wdXRlIHRhcmdldC4gQ3JlYXRlZCBpZiBkb2VzIG5vdCBleGlzdC4KICAgIDpwYXJhbSB2bV9zaXplOiAgICAgICAgICAgICBBenVyZSBtYWNoaW5lIHR5cGUgZm9yIGNvbXB1dGUgdGFyZ2V0LgogICAgOnBhcmFtIG1heF9ub2RlczogICAgICAgICAgIE1heGltdW0gbnVtYmVyIG9mIGNvbmN1cnJlbnQgY29tcHV0ZSB0YXJnZXRzLgoKICAgIDpwYXJhbSBkYXRhc2V0X25hbWU6ICAgICAgICBOYW1lIG9mIEF6dXJlIGRhdGFzZXQgdG8gcmVnaXN0ZXIuCiAgICA6cGFyYW0gZGF0YXNldF9kZXNjcmlwdGlvbjogRGVzY3JpcHRpb24gb2YgQXp1cmUgZGF0YXNldCB0byByZWdpc3Rlci4KCiAgICA6cGFyYW0gY3JlYXRlX25ld192ZXJzaW9uOiAgUmVnaXN0ZXIgQXp1cmUgZGF0YXNldCBhcyBuZXcgdmVyc2lvbi4gTXVzdCBiZSB1c2VkIHdoZW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RpZnlpbmcgZGF0YXNldCBzY2hlbWEuCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uX25hbWU6ICAgVGFyZ2V0IGNvbHVtbiBpbiBkYXRhc2V0LgoKICAgIDpwYXJhbSByZWdpc3Rlcl9tb2RlbF9uYW1lOiBOYW1lIG9mIG1vZGVsIHRvIHJlZ2lzdGVyIGluIEF6dXJlLgogICAgOnBhcmFtIHNhdmVfbl9tb2RlbHM6ICAgICAgIEhvdyBtYW55IG9mIHRoZSB0b3AgcGVyZm9ybWluZyBtb2RlbHMgdG8gbG9nLgogICAgOnBhcmFtIGxvZ19henVyZTogICAgICAgICAgIERpc3BsYXlpbmcgQXp1cmUgbG9ncy4KICAgIDpwYXJhbSBhdXRvbWxfc2V0dGluZ3M6ICAgICBKU09OIHN0cmluZyBvZiBhbGwgQXp1cmUgQXV0b01MIHNldHRpbmdzLgogICAgIiIiCiAgICBpZiBub3QgYXV0b21sX3NldHRpbmdzOgogICAgICAgIGF1dG9tbF9zZXR0aW5ncyA9IHsKICAgICAgICAgICAgInRhc2siOiAiY2xhc3NpZmljYXRpb24iLAogICAgICAgICAgICAiZGVidWdfbG9nIjogImF1dG9tbF9lcnJvcnMubG9nIiwKICAgICAgICAgICAgIyAiZXhwZXJpbWVudF9leGl0X3Njb3JlIjogMC45LAogICAgICAgICAgICAiZW5hYmxlX2Vhcmx5X3N0b3BwaW5nIjogRmFsc2UsCiAgICAgICAgICAgICJhbGxvd2VkX21vZGVscyI6IFsiTG9naXN0aWNSZWdyZXNzaW9uIiwgIlNHRCIsICJTVk0iXSwKICAgICAgICAgICAgIml0ZXJhdGlvbnMiOiAzLAogICAgICAgICAgICAiaXRlcmF0aW9uX3RpbWVvdXRfbWludXRlcyI6IDIsCiAgICAgICAgICAgICJtYXhfY29uY3VycmVudF9pdGVyYXRpb25zIjogMiwKICAgICAgICAgICAgIm1heF9jb3Jlc19wZXJfaXRlcmF0aW9uIjogLTEsCiAgICAgICAgICAgICJuX2Nyb3NzX3ZhbGlkYXRpb25zIjogNSwKICAgICAgICAgICAgInByaW1hcnlfbWV0cmljIjogImFjY3VyYWN5IiwKICAgICAgICAgICAgImZlYXR1cml6YXRpb24iOiAib2ZmIiwKICAgICAgICAgICAgIm1vZGVsX2V4cGxhaW5hYmlsaXR5IjogRmFsc2UsCiAgICAgICAgICAgICJlbmFibGVfdm90aW5nX2Vuc2VtYmxlIjogRmFsc2UsCiAgICAgICAgICAgICJlbmFibGVfc3RhY2tfZW5zZW1ibGUiOiBGYWxzZSwKICAgICAgICB9CgogICAgIyBJbml0IGV4cGVyaW1lbnQgYW5kIGNvbXB1dGUKICAgIHdvcmtzcGFjZSwgZXhwZXJpbWVudCA9IF9pbml0X2V4cGVyaW1lbnQoCiAgICAgICAgY29udGV4dD1jb250ZXh0LCBleHBlcmltZW50X25hbWU9ZXhwZXJpbWVudF9uYW1lCiAgICApCgogICAgY29tcHV0ZV90YXJnZXQgPSBpbml0X2NvbXB1dGUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGNwdV9jbHVzdGVyX25hbWU9Y3B1X2NsdXN0ZXJfbmFtZSwKICAgICAgICB2bV9zaXplPXZtX3NpemUsCiAgICAgICAgbWF4X25vZGVzPW1heF9ub2RlcywKICAgICkKCiAgICAjIFJlZ2lzdGVyIGRhdGFzZXQKICAgIHJlZ2lzdGVyX2RhdGFzZXQoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXRfbmFtZT1kYXRhc2V0X25hbWUsCiAgICAgICAgZGF0YXNldF9kZXNjcmlwdGlvbj1kYXRhc2V0X2Rlc2NyaXB0aW9uLAogICAgICAgIGRhdGE9ZGF0YXNldCwKICAgICAgICBjcmVhdGVfbmV3X3ZlcnNpb249Y3JlYXRlX25ld192ZXJzaW9uLAogICAgKQoKICAgICMgU3VibWl0IHRyYWluaW5nIGpvYgogICAgc3VibWl0X3RyYWluaW5nX2pvYigKICAgICAgICBjb250ZXh0LAogICAgICAgIGV4cGVyaW1lbnQ9ZXhwZXJpbWVudCwKICAgICAgICBjb21wdXRlX3RhcmdldD1jb21wdXRlX3RhcmdldCwKICAgICAgICByZWdpc3Rlcl9tb2RlbF9uYW1lPXJlZ2lzdGVyX21vZGVsX25hbWUsCiAgICAgICAgcmVnaXN0ZXJlZF9kYXRhc2V0X25hbWU9ZGF0YXNldF9uYW1lLAogICAgICAgIGxhYmVsX2NvbHVtbl9uYW1lPWxhYmVsX2NvbHVtbl9uYW1lLAogICAgICAgIGF1dG9tbF9zZXR0aW5ncz1hdXRvbWxfc2V0dGluZ3MsCiAgICAgICAgdHJhaW5pbmdfc2V0PWRhdGFzZXQsCiAgICAgICAgc2hvd19vdXRwdXQ9bG9nX2F6dXJlLAogICAgICAgIHNhdmVfbl9tb2RlbHM9c2F2ZV9uX21vZGVscywKICAgICkK + code_origin: '' commands: - apt-get update && apt-get install -y --no-install-recommends git - apt install -y liblttng-ust0 + auto_build: true base_image: python:3.9-bullseye - origin_filename: '' - default_handler: train allow_empty_resources: true - disable_auto_mount: false - image: '' + filename: azureml_utils.py entry_points: init_compute: - doc: 'Initialize Azure ML compute target to run experiment. Checks for - - existing compute target and creates new if does not exist.' - name: init_compute - lineno: 102 - has_kwargs: false + outputs: + - doc: Azure ML Compute Target. + type: ComputeTarget parameters: - name: context type: MLClientCtx @@ -42,20 +45,14 @@ spec: type: int doc: Maximum number of concurrent compute targets. default: 1 - outputs: - - doc: Azure ML Compute Target. - type: ComputeTarget - has_varargs: false - register_dataset: - doc: 'Register dataset object (can be also an Iguazio FeatureVector) in Azure - ML. - - Uploads parquet file to Azure blob storage and registers + name: init_compute + doc: 'Initialize Azure ML compute target to run experiment. Checks for - that file as a dataset in Azure ML.' - name: register_dataset - lineno: 138 + existing compute target and creates new if does not exist.' has_kwargs: false + has_varargs: false + lineno: 99 + register_dataset: parameters: - name: context type: MLClientCtx @@ -74,12 +71,19 @@ spec: doc: Register Azure dataset as new version. Must be used when modifying dataset schema. default: false + name: register_dataset + doc: 'Register dataset object (can be also an Iguazio FeatureVector) in Azure + ML. + + Uploads parquet file to Azure blob storage and registers + + that file as a dataset in Azure ML.' + has_kwargs: false has_varargs: false + lineno: 135 download_model: - doc: Download trained model from Azure ML to local filesystem. - name: download_model - lineno: 217 - has_kwargs: false + outputs: + - type: None parameters: - name: context type: MLClientCtx @@ -94,14 +98,14 @@ spec: type: str doc: Target directory to download model. default: . - outputs: - - type: None + name: download_model + doc: Download trained model from Azure ML to local filesystem. + has_kwargs: false has_varargs: false + lineno: 216 upload_model: - doc: Upload pre-trained model from local filesystem to Azure ML. - name: upload_model - lineno: 238 - has_kwargs: false + outputs: + - type: None parameters: - name: context type: MLClientCtx @@ -120,16 +124,14 @@ spec: type: dict doc: KV pairs of model tags. default: null - outputs: - - type: None + name: upload_model + doc: Upload pre-trained model from local filesystem to Azure ML. + has_kwargs: false has_varargs: false + lineno: 237 submit_training_job: - doc: 'Submit training job to Azure AutoML and download trained model - - when completed. Uses previously registered dataset for training.' - name: submit_training_job - lineno: 352 - has_kwargs: false + outputs: + - type: None parameters: - name: context type: MLClientCtx @@ -164,18 +166,16 @@ spec: type: bool doc: Displaying Azure logs. default: true - outputs: - - type: None - has_varargs: false - train: - doc: 'Whole training flow for Azure AutoML. Registers dataset/feature vector, - - submits training job to Azure AutoML, and downloads trained model + name: submit_training_job + doc: 'Submit training job to Azure AutoML and download trained model - when completed.' - name: train - lineno: 469 + when completed. Uses previously registered dataset for training.' has_kwargs: false + has_varargs: false + lineno: 350 + train: + outputs: + - type: None parameters: - name: context type: MLClientCtx @@ -233,15 +233,16 @@ spec: type: str doc: JSON string of all Azure AutoML settings. default: null - outputs: - - type: None + name: train + doc: 'Whole training flow for Azure AutoML. Registers dataset/feature vector, + + submits training job to Azure AutoML, and downloads trained model + + when completed.' + has_kwargs: false has_varargs: false + lineno: 465 + command: '' description: Azure AutoML integration in MLRun, including utils functions for training models on Azure AutoML platfrom. -kind: job -metadata: - categories: - - model-serving - - utils - tag: '' - name: azureml-utils + default_handler: train diff --git a/functions/src/azureml_utils/test_azureml_utils.py b/functions/src/azureml_utils/test_azureml_utils.py index d6ef80d12..752fc3fee 100644 --- a/functions/src/azureml_utils/test_azureml_utils.py +++ b/functions/src/azureml_utils/test_azureml_utils.py @@ -13,11 +13,11 @@ # limitations under the License. # import os -import tempfile import shutil -import pytest +import tempfile import mlrun +import pytest from mlrun import import_function EXPERIMENT_NAME = "azure-automl-test" @@ -117,7 +117,9 @@ def test_train(): local=True, ) # Get trained models: - num_saved_models = len(azureml_run.status.iterations) - 1 # The first one in the list is the 'columns' + num_saved_models = ( + len(azureml_run.status.iterations) - 1 + ) # The first one in the list is the 'columns' test_pass = num_saved_models == save_n_models except Exception as exception: @@ -125,4 +127,4 @@ def test_train(): _cleanup_environment(artifact_path) - assert test_pass, f'Created {len(model_paths)} models instead of {save_n_models}' + assert test_pass, f"Created {len(model_paths)} models instead of {save_n_models}" diff --git a/functions/src/batch_inference/batch_inference.py b/functions/src/batch_inference/batch_inference.py index 844fdf392..3070c6f72 100644 --- a/functions/src/batch_inference/batch_inference.py +++ b/functions/src/batch_inference/batch_inference.py @@ -15,14 +15,15 @@ import hashlib import json from datetime import datetime -from typing import Any, Dict, List, Tuple, Union -import semver +from typing import Any, Union import mlrun +import semver + if semver.compare(mlrun.__version__, "1.5.0") >= 0: raise mlrun.errors.MLRunNotFoundError( - f"When using `mlrun` version >=1.5.0, please use " - f"batch inference `v2` function ('hub://batch_inference_v2')." + "When using `mlrun` version >=1.5.0, please use " + "batch inference `v2` function ('hub://batch_inference_v2')." ) import mlrun.datastore @@ -45,10 +46,10 @@ def _read_dataset_as_dataframe( dataset: DatasetType, - feature_columns: Union[str, List[str]] = None, - label_columns: Union[str, List[str]] = None, - drop_columns: Union[str, List[str], int, List[int]] = None, -) -> Tuple[pd.DataFrame, List[str]]: + feature_columns: str | list[str] = None, + label_columns: str | list[str] = None, + drop_columns: str | list[str] | int | list[int] = None, +) -> tuple[pd.DataFrame, list[str]]: """ Parse the given dataset into a DataFrame and drop the columns accordingly. In addition, the label columns will be parsed and validated as well. @@ -120,7 +121,7 @@ def _read_dataset_as_dataframe( def _prepare_result_set( - x: pd.DataFrame, label_columns: List[str], y_pred: np.ndarray + x: pd.DataFrame, label_columns: list[str], y_pred: np.ndarray ) -> pd.DataFrame: """ Set default label column names and validate given names to prepare the result set - a concatenation of the inputs @@ -204,7 +205,7 @@ def _get_drift_result( tvd: float, hellinger: float, threshold: float, -) -> Tuple[bool, float]: +) -> tuple[bool, float]: """ Calculate the drift result by the following equation: (tvd + hellinger) / 2 @@ -228,7 +229,7 @@ def _perform_drift_analysis( drift_threshold: float, possible_drift_threshold: float, inf_capping: float, -) -> Tuple[Artifact, Artifact, dict]: +) -> tuple[Artifact, Artifact, dict]: """ Perform drift analysis, producing the drift table artifact for logging post prediction. @@ -318,9 +319,9 @@ def infer( context: mlrun.MLClientCtx, model: str, dataset: DatasetType, - drop_columns: Union[str, List[str], int, List[int]] = None, - label_columns: Union[str, List[str]] = None, - feature_columns: Union[str, List[str]] = None, + drop_columns: str | list[str] | int | list[int] = None, + label_columns: str | list[str] = None, + feature_columns: str | list[str] = None, log_result_set: bool = True, result_set_name: str = "prediction", batch_id: str = None, @@ -330,7 +331,7 @@ def infer( possible_drift_threshold: float = 0.5, inf_capping: float = 10.0, artifacts_tag: str = "", - **predict_kwargs: Dict[str, Any], + **predict_kwargs: dict[str, Any], ): """ Perform a prediction on a given dataset with the given model. Can perform drift analysis between the sample set @@ -368,7 +369,7 @@ def infer( :param artifacts_tag: Tag to use for all the artifacts resulted from the function. """ # Loading the model: - context.logger.info(f"Loading model...") + context.logger.info("Loading model...") model_handler = AutoMLRun.load_model(model_path=model, context=context) if label_columns is None: label_columns = [ @@ -381,7 +382,7 @@ def infer( ] # Get dataset by object, URL or by FeatureVector: - context.logger.info(f"Loading data...") + context.logger.info("Loading data...") x, label_columns = _read_dataset_as_dataframe( dataset=dataset, feature_columns=feature_columns, @@ -390,7 +391,7 @@ def infer( ) # Predict: - context.logger.info(f"Calculating prediction...") + context.logger.info("Calculating prediction...") y_pred = model_handler.model.predict(x, **predict_kwargs) # Prepare the result set: @@ -399,7 +400,7 @@ def infer( # Check for logging the result set: if log_result_set: # Log the result set: - context.logger.info(f"Logging result set (x | prediction)...") + context.logger.info("Logging result set (x | prediction)...") context.log_dataset( key=result_set_name, df=result_set, diff --git a/functions/src/batch_inference/function.yaml b/functions/src/batch_inference/function.yaml index 74b672d4a..0c0ada9cb 100644 --- a/functions/src/batch_inference/function.yaml +++ b/functions/src/batch_inference/function.yaml @@ -1,22 +1,23 @@ -kind: job -verbose: false metadata: - name: batch-inference tag: '' + name: batch-inference categories: - model-serving +verbose: false +kind: job spec: image: mlrun/ml-models + disable_auto_mount: false + build: + origin_filename: '' + with_mlrun: false + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IGhhc2hsaWIKaW1wb3J0IGpzb24KZnJvbSBkYXRldGltZSBpbXBvcnQgZGF0ZXRpbWUKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgc2VtdmVyCgppZiBzZW12ZXIuY29tcGFyZShtbHJ1bi5fX3ZlcnNpb25fXywgIjEuNS4wIikgPj0gMDoKICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bk5vdEZvdW5kRXJyb3IoCiAgICAgICAgIldoZW4gdXNpbmcgYG1scnVuYCB2ZXJzaW9uID49MS41LjAsIHBsZWFzZSB1c2UgIgogICAgICAgICJiYXRjaCBpbmZlcmVuY2UgYHYyYCBmdW5jdGlvbiAoJ2h1YjovL2JhdGNoX2luZmVyZW5jZV92MicpLiIKICAgICkKCmltcG9ydCBtbHJ1bi5kYXRhc3RvcmUKaW1wb3J0IG1scnVuLnV0aWxzCmltcG9ydCBudW1weSBhcyBucAppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5hcnRpZmFjdHMgaW1wb3J0IEFydGlmYWN0CmZyb20gbWxydW4uZGF0YV90eXBlcy5pbmZlciBpbXBvcnQgSW5mZXJPcHRpb25zLCBnZXRfZGZfc3RhdHMKZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLmF1dG9fbWxydW4gaW1wb3J0IEF1dG9NTFJ1bgpmcm9tIG1scnVuLm1vZGVsX21vbml0b3JpbmcuZmVhdHVyZXNfZHJpZnRfdGFibGUgaW1wb3J0IEZlYXR1cmVzRHJpZnRUYWJsZVBsb3QKZnJvbSBtbHJ1bi5tb2RlbF9tb25pdG9yaW5nLm1vZGVsX21vbml0b3JpbmdfYmF0Y2ggaW1wb3J0ICgKICAgIFZpcnR1YWxEcmlmdCwKICAgIGNhbGN1bGF0ZV9pbnB1dHNfc3RhdGlzdGljcywKKQoKIyBBIHVuaW9uIG9mIGFsbCBzdXBwb3J0ZWQgZGF0YXNldCB0eXBlczoKRGF0YXNldFR5cGUgPSBVbmlvblttbHJ1bi5EYXRhSXRlbSwgbGlzdCwgZGljdCwgcGQuRGF0YUZyYW1lLCBwZC5TZXJpZXMsIG5wLm5kYXJyYXldCgoKZGVmIF9yZWFkX2RhdGFzZXRfYXNfZGF0YWZyYW1lKAogICAgZGF0YXNldDogRGF0YXNldFR5cGUsCiAgICBmZWF0dXJlX2NvbHVtbnM6IHN0ciB8IGxpc3Rbc3RyXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBzdHIgfCBsaXN0W3N0cl0gPSBOb25lLAogICAgZHJvcF9jb2x1bW5zOiBzdHIgfCBsaXN0W3N0cl0gfCBpbnQgfCBsaXN0W2ludF0gPSBOb25lLAopIC0+IHR1cGxlW3BkLkRhdGFGcmFtZSwgbGlzdFtzdHJdXToKICAgICIiIgogICAgUGFyc2UgdGhlIGdpdmVuIGRhdGFzZXQgaW50byBhIERhdGFGcmFtZSBhbmQgZHJvcCB0aGUgY29sdW1ucyBhY2NvcmRpbmdseS4gSW4gYWRkaXRpb24sIHRoZSBsYWJlbCBjb2x1bW5zIHdpbGwgYmUKICAgIHBhcnNlZCBhbmQgdmFsaWRhdGVkIGFzIHdlbGwuCgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgQSBkYXRhc2V0IHRoYXQgd2lsbCBiZSBjb252ZXJ0ZWQgaW50byBhIERhdGFGcmFtZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgIENhbiBiZSBlaXRoZXIgYSBsaXN0IG9mIGxpc3RzLCBkaWN0LCBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGZlYXR1cmVfY29sdW1uczogTGlzdCBvZiBmZWF0dXJlIGNvbHVtbnMgdGhhdCB3aWxsIGJlIHVzZWQgdG8gYnVpbGQgdGhlIGRhdGFmcmFtZSB3aGVuIGRhdGFzZXQgaXMgZnJvbQogICAgICAgICAgICAgICAgICAgICAgICAgICAgdHlwZSBsaXN0IG9yIG51bXB5IGFycmF5LgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2xhc3NpZmljYXRpb24gdGFza3MuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICBgYHN0cmBgIC8gYGBpbnRgYCBvciBhIGxpc3Qgb2YgYGBzdHJgYCAvIGBgaW50YGAgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbiBuYW1lcyAvIGluZGljZXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRvIGRyb3AuCgogICAgOnJldHVybnM6IEEgdHVwbGUgb2Y6CiAgICAgICAgICAgICAgWzBdID0gVGhlIHBhcnNlZCBkYXRhc2V0IGFzIGEgRGF0YUZyYW1lCiAgICAgICAgICAgICAgWzFdID0gTGFiZWwgY29sdW1ucy4KCiAgICByYWlzZXMgTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcjogSWYgdGhlIGBkcm9wX2NvbHVtbnNgIGFyZSBub3QgbWF0Y2hpbmcgdGhlIGRhdGFzZXQgb3IgdW5zdXBwb3J0ZWQgZGF0YXNldCB0eXBlLgogICAgIiIiCiAgICAjIFR1cm4gdGhlIGBkcm9wIGxhYmVsc2AgaW50byBhIGxpc3QgaWYgZ2l2ZW46CiAgICBpZiBkcm9wX2NvbHVtbnMgaXMgbm90IE5vbmU6CiAgICAgICAgaWYgbm90IGlzaW5zdGFuY2UoZHJvcF9jb2x1bW5zLCBsaXN0KToKICAgICAgICAgICAgZHJvcF9jb2x1bW5zID0gW2Ryb3BfY29sdW1uc10KCiAgICAjIENoZWNrIGlmIHRoZSBkYXRhc2V0IGlzIGluIGZhY3QgYSBGZWF0dXJlIFZlY3RvcjoKICAgIGlmIGlzaW5zdGFuY2UoZGF0YXNldCwgZnMuRmVhdHVyZVZlY3Rvcik6CiAgICAgICAgIyBUcnkgdG8gZ2V0IHRoZSBsYWJlbCBjb2x1bW5zIGlmIG5vdCBwcm92aWRlZDoKICAgICAgICBpZiBsYWJlbF9jb2x1bW5zIGlzIE5vbmU6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBkYXRhc2V0LnN0YXR1cy5sYWJlbF9jb2x1bW4KICAgICAgICAjIEdldCB0aGUgZmVhdHVyZXMgYW5kIHBhcnNlIHRvIERhdGFGcmFtZToKICAgICAgICBkYXRhc2V0ID0gZnMuZ2V0X29mZmxpbmVfZmVhdHVyZXMoCiAgICAgICAgICAgIGRhdGFzZXQudXJpLCBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zCiAgICAgICAgKS50b19kYXRhZnJhbWUoKQoKICAgIGVsaWYgaXNpbnN0YW5jZShkYXRhc2V0LCAobGlzdCwgbnAubmRhcnJheSkpOgogICAgICAgIGlmIG5vdCBmZWF0dXJlX2NvbHVtbnM6CiAgICAgICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICAgICAgIkZlYXR1cmUgY29sdW1ucyBsaXN0IG11c3QgYmUgcHJvdmlkZWQgd2hlbiBkYXRhc2V0IGlucHV0IGFzIGZyb20gdHlwZSBsaXN0IG9yIG51bXB5IGFycmF5IgogICAgICAgICAgICApCiAgICAgICAgIyBQYXJzZSB0aGUgbGlzdCAvIG51bXB5IGFycmF5IGludG8gYSBEYXRhRnJhbWU6CiAgICAgICAgZGF0YXNldCA9IHBkLkRhdGFGcmFtZShkYXRhc2V0LCBjb2x1bW5zPWZlYXR1cmVfY29sdW1ucykKICAgICAgICAjIFZhbGlkYXRlIHRoZSBgZHJvcF9jb2x1bW5zYCBpcyBnaXZlbiBhcyBpbnRlZ2VyczoKICAgICAgICBpZiBkcm9wX2NvbHVtbnMgYW5kIG5vdCBhbGwoaXNpbnN0YW5jZShjb2wsIGludCkgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgICAgICJgZHJvcF9jb2x1bW5zYCBtdXN0IGJlIGFuIGludGVnZXIgLyBsaXN0IG9mIGludGVnZXJzIGlmIHByb3ZpZGVkIGFzIGEgbGlzdC4iCiAgICAgICAgICAgICkKICAgIGVsaWYgaXNpbnN0YW5jZShkYXRhc2V0LCBtbHJ1bi5EYXRhSXRlbSk6CiAgICAgICAgIyBUdXJuIHRoZSBEYXRhSVRlbSB0byBEYXRhRnJhbWU6CiAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuYXNfZGYoKQogICAgZWxzZToKICAgICAgICAjIFBhcnNlIHRoZSBvYmplY3QgKHNob3VsZCBiZSBhIHBkLkRhdGFGcmFtZSAvIHBkLlNlcmllcywgZGljdGlvbmFyeSkgaW50byBhIERhdGFGcmFtZToKICAgICAgICB0cnk6CiAgICAgICAgICAgIGRhdGFzZXQgPSBwZC5EYXRhRnJhbWUoZGF0YXNldCkKICAgICAgICBleGNlcHQgVmFsdWVFcnJvciBhcyBlOgogICAgICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgICAgIGYiQ291bGQgbm90IHBhcnNlIHRoZSBnaXZlbiBkYXRhc2V0IG9mIHR5cGUge3R5cGUoZGF0YXNldCl9IGludG8gYSBwYW5kYXMgRGF0YUZyYW1lLiAiCiAgICAgICAgICAgICAgICBmIlJlY2VpdmVkIHRoZSBmb2xsb3dpbmcgZXJyb3I6IHtlfSIKICAgICAgICAgICAgKQogICAgIyBEcm9wIGNvbHVtbnMgaWYgbmVlZGVkOgogICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgIGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSwgaW5wbGFjZT1UcnVlKQoKICAgICMgVHVybiB0aGUgYGxhYmVsX2NvbHVtbnNgIGludG8gYSBsaXN0IGJ5IGRlZmF1bHQ6CiAgICBpZiBsYWJlbF9jb2x1bW5zIGlzIE5vbmU6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtdCiAgICBlbGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgKHN0ciwgaW50KSk6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtsYWJlbF9jb2x1bW5zXQogICAgcmV0dXJuIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMKCgpkZWYgX3ByZXBhcmVfcmVzdWx0X3NldCgKICAgIHg6IHBkLkRhdGFGcmFtZSwgbGFiZWxfY29sdW1uczogbGlzdFtzdHJdLCB5X3ByZWQ6IG5wLm5kYXJyYXkKKSAtPiBwZC5EYXRhRnJhbWU6CiAgICAiIiIKICAgIFNldCBkZWZhdWx0IGxhYmVsIGNvbHVtbiBuYW1lcyBhbmQgdmFsaWRhdGUgZ2l2ZW4gbmFtZXMgdG8gcHJlcGFyZSB0aGUgcmVzdWx0IHNldCAtIGEgY29uY2F0ZW5hdGlvbiBvZiB0aGUgaW5wdXRzCiAgICAoeCkgYW5kIHRoZSBtb2RlbCBwcmVkaWN0aW9ucyAoeV9wcmVkKS4KCiAgICA6cGFyYW0geDogICAgICAgICAgICAgVGhlIGlucHV0cy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiBBIGxpc3Qgb2Ygc3RyaW5ncyByZXByZXNlbnRpbmcgdGhlIHRhcmdldCBjb2x1bW4gbmFtZXMgdG8gYWRkIHRvIHRoZSBwcmVkaWN0aW9ucy4gRGVmYXVsdCBuYW1lCiAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSB1c2VkIGluIGNhc2UgdGhlIGxpc3QgaXMgZW1wdHkgKHByZWRpY3RlZF9sYWJlbF97aX0pLgogICAgOnBhcmFtIHlfcHJlZDogICAgICAgIFRoZSBtb2RlbCBwcmVkaWN0aW9ucyBvbiB0aGUgaW5wdXRzLgoKICAgIDpyZXR1cm5zOiBUaGUgcmVzdWx0IHNldC4KCiAgICByYWlzZXMgTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcjogSWYgdGhlIGxhYmVscyBjb2x1bW5zIGFtb3VudCBkbyBub3QgbWF0Y2ggdGhlIG91dHB1dHMgb3IgaWYgb25lIG9mIHRoZSBsYWJlbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb2x1bW4gYWxyZWFkeSBleGlzdHMgaW4gdGhlIGRhdGFzZXQuCiAgICAiIiIKICAgICMgUHJlcGFyZSBkZWZhdWx0IHRhcmdldCBjb2x1bW5zIG5hbWVzIGlmIG5vdCBwcm92aWRlZDoKICAgIHByZWRpY3Rpb25fY29sdW1uc19hbW91bnQgPSAxIGlmIGxlbih5X3ByZWQuc2hhcGUpID09IDEgZWxzZSB5X3ByZWQuc2hhcGVbMV0KICAgIGlmIGxlbihsYWJlbF9jb2x1bW5zKSA9PSAwOgogICAgICAgICMgQWRkIGRlZmF1bHQgbGFiZWwgY29sdW1uIG5hbWVzOgogICAgICAgIGlmIHByZWRpY3Rpb25fY29sdW1uc19hbW91bnQgPT0gMToKICAgICAgICAgICAgbGFiZWxfY29sdW1ucyA9IFsicHJlZGljdGVkX2xhYmVsIl0KICAgICAgICBlbHNlOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWwogICAgICAgICAgICAgICAgZiJwcmVkaWN0ZWRfbGFiZWxfe2l9IiBmb3IgaSBpbiByYW5nZShwcmVkaWN0aW9uX2NvbHVtbnNfYW1vdW50KQogICAgICAgICAgICBdCgogICAgIyBWYWxpZGF0ZSB0aGUgbGFiZWwgY29sdW1uczoKICAgIGlmIHByZWRpY3Rpb25fY29sdW1uc19hbW91bnQgIT0gbGVuKGxhYmVsX2NvbHVtbnMpOgogICAgICAgICMgTm8gZXF1YWxpdHkgYmV0d2VlbiBwcm92aWRlZCBsYWJlbCBjb2x1bW4gbmFtZXMgYW5kIG91dHB1dHMgYW1vdW50OgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIlRoZSBudW1iZXIgb2YgcHJlZGljdGVkIGxhYmVsczoge3ByZWRpY3Rpb25fY29sdW1uc19hbW91bnR9ICIKICAgICAgICAgICAgZiJpcyBub3QgZXF1YWwgdG8gdGhlIGdpdmVuIGxhYmVsIGNvbHVtbnM6IHtsZW4obGFiZWxfY29sdW1ucyl9IgogICAgICAgICkKICAgIGNvbW1vbl9sYWJlbHMgPSBzZXQobGFiZWxfY29sdW1ucykgJiBzZXQoeC5jb2x1bW5zLnRvbGlzdCgpKQogICAgaWYgY29tbW9uX2xhYmVsczoKICAgICAgICAjIExhYmVsIGNvbHVtbiBleGlzdCBpbiB0aGUgb3JpZ2luYWwgaW5wdXRzOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIlRoZSBsYWJlbHM6IHtjb21tb25fbGFiZWxzfSBhcmUgYWxyZWFkeSBleGlzdGVkIGluIHRoZSBnaXZlbiBkYXRhc2V0LiIKICAgICAgICApCgogICAgcmV0dXJuIHBkLmNvbmNhdCgKICAgICAgICBbeCwgcGQuRGF0YUZyYW1lKHlfcHJlZCwgY29sdW1ucz1sYWJlbF9jb2x1bW5zLCBpbmRleD14LmluZGV4KV0sIGF4aXM9MQogICAgKQoKCmRlZiBfZ2V0X3NhbXBsZV9zZXRfc3RhdGlzdGljcygKICAgIHNhbXBsZV9zZXQ6IERhdGFzZXRUeXBlID0gTm9uZSwgbW9kZWxfYXJ0aWZhY3RfZmVhdHVyZV9zdGF0czogZGljdCA9IE5vbmUKKSAtPiBkaWN0OgogICAgIiIiCiAgICBHZXQgdGhlIHNhbXBsZSBzZXQgc3RhdGlzdGljcyBlaXRoZXIgZnJvbSB0aGUgZ2l2ZW4gc2FtcGxlIHNldCBvciB0aGUgc3RhdGlzdGljcyBsb2dnZWQgd2l0aCB0aGUgbW9kZWwgd2hpbGUKICAgIGZhdm9yaW5nIHRoZSBnaXZlbiBzYW1wbGUgc2V0LgoKICAgIDpwYXJhbSBzYW1wbGVfc2V0OiAgICAgICAgICAgICAgICAgICBBIHNhbXBsZSBkYXRhc2V0IHRvIGdpdmUgdG8gY29tcGFyZSB0aGUgaW5wdXRzIGluIHRoZSBkcmlmdCBhbmFseXNpcy4KICAgIDpwYXJhbSBtb2RlbF9hcnRpZmFjdF9mZWF0dXJlX3N0YXRzOiBUaGUgYGZlYXR1cmVfc3RhdHNgIGF0dHJpYnV0ZSBpbiB0aGUgc3BlYyBvZiB0aGUgbW9kZWwgYXJ0aWZhY3QsIHdoZXJlIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9yaWdpbmFsIHNhbXBsZSBzZXQgc3RhdGlzdGljcyBvZiB0aGUgbW9kZWwgd2FzIHVzZWQuCgogICAgOnJldHVybnM6IFRoZSBzYW1wbGUgc2V0IHN0YXRpc3RpY3MuCgogICAgcmFpc2VzIE1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3I6IElmIG5vIHNhbXBsZSBzZXQgb3Igc3RhdGlzdGljcyB3ZXJlIGdpdmVuLgogICAgIiIiCiAgICAjIENoZWNrIGlmIGEgc2FtcGxlIHNldCB3YXMgcHJvdmlkZWQ6CiAgICBpZiBzYW1wbGVfc2V0IGlzIE5vbmU6CiAgICAgICAgIyBDaGVjayBpZiB0aGUgbW9kZWwgd2FzIGxvZ2dlZCB3aXRoIGEgc2FtcGxlIHNldDoKICAgICAgICBpZiBtb2RlbF9hcnRpZmFjdF9mZWF0dXJlX3N0YXRzIGlzIE5vbmU6CiAgICAgICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICAgICAgIkNhbm5vdCBwZXJmb3JtIGRyaWZ0IGFuYWx5c2lzIGFzIHRoZXJlIGlzIG5vIHNhbXBsZSBzZXQgdG8gY29tcGFyZSB0by4gVGhlIG1vZGVsIGFydGlmYWN0IHdhcyBub3QgIgogICAgICAgICAgICAgICAgImxvZ2dlZCB3aXRoIGEgc2FtcGxlIHNldCBhbmQgYHNhbXBsZV9zZXRgIHdhcyBub3QgcHJvdmlkZWQgdG8gdGhlIGZ1bmN0aW9uLiIKICAgICAgICAgICAgKQogICAgICAgICMgUmV0dXJuIHRoZSBzdGF0aXN0aWNzIGxvZ2dlZCB3aXRoIHRoZSBtb2RlbDoKICAgICAgICByZXR1cm4gbW9kZWxfYXJ0aWZhY3RfZmVhdHVyZV9zdGF0cwoKICAgICMgVHVybiB0aGUgRGF0YUl0ZW0gdG8gRGF0YUZyYW1lOgogICAgaWYgaXNpbnN0YW5jZShzYW1wbGVfc2V0LCBtbHJ1bi5EYXRhSXRlbSk6CiAgICAgICAgc2FtcGxlX3NldCwgXyA9IF9yZWFkX2RhdGFzZXRfYXNfZGF0YWZyYW1lKGRhdGFzZXQ9c2FtcGxlX3NldCkKCiAgICAjIFJldHVybiB0aGUgc2FtcGxlIHNldCBzdGF0aXN0aWNzOgogICAgcmV0dXJuIGdldF9kZl9zdGF0cyhkZj1zYW1wbGVfc2V0LCBvcHRpb25zPUluZmVyT3B0aW9ucy5IaXN0b2dyYW0pCgoKZGVmIF9nZXRfZHJpZnRfcmVzdWx0KAogICAgdHZkOiBmbG9hdCwKICAgIGhlbGxpbmdlcjogZmxvYXQsCiAgICB0aHJlc2hvbGQ6IGZsb2F0LAopIC0+IHR1cGxlW2Jvb2wsIGZsb2F0XToKICAgICIiIgogICAgQ2FsY3VsYXRlIHRoZSBkcmlmdCByZXN1bHQgYnkgdGhlIGZvbGxvd2luZyBlcXVhdGlvbjogKHR2ZCArIGhlbGxpbmdlcikgLyAyCgogICAgOnBhcmFtIHR2ZDogICAgICAgVGhlIGZlYXR1cmUncyBUVkQgdmFsdWUuCiAgICA6cGFyYW0gaGVsbGluZ2VyOiBUaGUgZmVhdHVyZSdzIEhlbGxpbmdlciB2YWx1ZS4KICAgIDpwYXJhbSB0aHJlc2hvbGQ6IFRoZSB0aHJlc2hvbGQgZnJvbSB3aGljaCB0aGUgdmFsdWUgaXMgY29uc2lkZXJlZCBhIGRyaWZ0LgoKICAgIDpyZXR1cm5zOiBBIHR1cGxlIG9mOgogICAgICAgICAgICAgIFswXSA9IEJvb2xlYW4gdmFsdWUgYXMgdGhlIGRyaWZ0IHN0YXR1cy4KICAgICAgICAgICAgICBbMV0gPSBUaGUgcmVzdWx0LgogICAgIiIiCiAgICByZXN1bHQgPSAodHZkICsgaGVsbGluZ2VyKSAvIDIKICAgIGlmIHJlc3VsdCA+PSB0aHJlc2hvbGQ6CiAgICAgICAgcmV0dXJuIFRydWUsIHJlc3VsdAogICAgcmV0dXJuIEZhbHNlLCByZXN1bHQKCgpkZWYgX3BlcmZvcm1fZHJpZnRfYW5hbHlzaXMoCiAgICBzYW1wbGVfc2V0X3N0YXRpc3RpY3M6IGRpY3QsCiAgICBpbnB1dHM6IHBkLkRhdGFGcmFtZSwKICAgIGRyaWZ0X3RocmVzaG9sZDogZmxvYXQsCiAgICBwb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQ6IGZsb2F0LAogICAgaW5mX2NhcHBpbmc6IGZsb2F0LAopIC0+IHR1cGxlW0FydGlmYWN0LCBBcnRpZmFjdCwgZGljdF06CiAgICAiIiIKICAgIFBlcmZvcm0gZHJpZnQgYW5hbHlzaXMsIHByb2R1Y2luZyB0aGUgZHJpZnQgdGFibGUgYXJ0aWZhY3QgZm9yIGxvZ2dpbmcgcG9zdCBwcmVkaWN0aW9uLgoKICAgIDpwYXJhbSBzYW1wbGVfc2V0X3N0YXRpc3RpY3M6ICAgIFRoZSBzdGF0aXN0aWNzIG9mIHRoZSBzYW1wbGUgc2V0IGxvZ2dlZCBhbG9uZyBhIG1vZGVsLgogICAgOnBhcmFtIGlucHV0czogICAgICAgICAgICAgICAgICAgSW5wdXQgZGF0YXNldCB0byBwZXJmb3JtIHRoZSBkcmlmdCBjYWxjdWxhdGlvbiBvbi4KICAgIDpwYXJhbSBkcmlmdF90aHJlc2hvbGQ6ICAgICAgICAgIFRoZSB0aHJlc2hvbGQgb2Ygd2hpY2ggdG8gbWFyayBkcmlmdHMuCiAgICA6cGFyYW0gcG9zc2libGVfZHJpZnRfdGhyZXNob2xkOiBUaGUgdGhyZXNob2xkIG9mIHdoaWNoIHRvIG1hcmsgcG9zc2libGUgZHJpZnRzLgogICAgOnBhcmFtIGluZl9jYXBwaW5nOiAgICAgICAgICAgICAgVGhlIHZhbHVlIHRvIHNldCBmb3Igd2hlbiBpdCByZWFjaGVkIGluZmluaXR5LgoKICAgIDpyZXR1cm5zOiBBIHR1cGxlIG9mCiAgICAgICAgICAgICAgWzBdID0gQW4gTUxSdW4gYXJ0aWZhY3QgaG9sZGluZyB0aGUgSFRNTCBjb2RlIG9mIHRoZSBkcmlmdCB0YWJsZSBwbG90LgogICAgICAgICAgICAgIFsxXSA9IEFuIE1MUnVuIGFydGlmYWN0IGhvbGRpbmcgdGhlIG1ldHJpYyBwZXIgZmVhdHVyZSBkaWN0aW9uYXJ5LgogICAgICAgICAgICAgIFsyXSA9IFJlc3VsdHMgdG8gbG9nIHRoZSBmaW5hbCBhbmFseXNpcyBvdXRjb21lLgogICAgIiIiCiAgICAjIENhbGN1bGF0ZSB0aGUgaW5wdXQncyBzdGF0aXN0aWNzOgogICAgaW5wdXRzX3N0YXRpc3RpY3MgPSBjYWxjdWxhdGVfaW5wdXRzX3N0YXRpc3RpY3MoCiAgICAgICAgc2FtcGxlX3NldF9zdGF0aXN0aWNzPXNhbXBsZV9zZXRfc3RhdGlzdGljcywKICAgICAgICBpbnB1dHM9aW5wdXRzLAogICAgKQoKICAgICMgQ2FsY3VsYXRlIGRyaWZ0OgogICAgdmlydHVhbF9kcmlmdCA9IFZpcnR1YWxEcmlmdChpbmZfY2FwcGluZz1pbmZfY2FwcGluZykKICAgIG1ldHJpY3MgPSB2aXJ0dWFsX2RyaWZ0LmNvbXB1dGVfZHJpZnRfZnJvbV9oaXN0b2dyYW1zKAogICAgICAgIGZlYXR1cmVfc3RhdHM9c2FtcGxlX3NldF9zdGF0aXN0aWNzLAogICAgICAgIGN1cnJlbnRfc3RhdHM9aW5wdXRzX3N0YXRpc3RpY3MsCiAgICApCiAgICBkcmlmdF9yZXN1bHRzID0gdmlydHVhbF9kcmlmdC5jaGVja19mb3JfZHJpZnRfcGVyX2ZlYXR1cmUoCiAgICAgICAgbWV0cmljc19yZXN1bHRzX2RpY3Rpb25hcnk9bWV0cmljcywKICAgICAgICBwb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQ9cG9zc2libGVfZHJpZnRfdGhyZXNob2xkLAogICAgICAgIGRyaWZ0X2RldGVjdGVkX3RocmVzaG9sZD1kcmlmdF90aHJlc2hvbGQsCiAgICApCgogICAgIyBWYWxpZGF0ZSBhbGwgZmVhdHVyZSBjb2x1bW5zIG5hbWVkIHRoZSBzYW1lIGJldHdlZW4gdGhlIGlucHV0cyBhbmQgc2FtcGxlIHNldHM6CiAgICBzYW1wbGVfZmVhdHVyZXMgPSBzZXQoCiAgICAgICAgWwogICAgICAgICAgICBmZWF0dXJlX25hbWUKICAgICAgICAgICAgZm9yIGZlYXR1cmVfbmFtZSwgZmVhdHVyZV9zdGF0aXN0aWNzIGluIHNhbXBsZV9zZXRfc3RhdGlzdGljcy5pdGVtcygpCiAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoZmVhdHVyZV9zdGF0aXN0aWNzLCBkaWN0KQogICAgICAgIF0KICAgICkKICAgIGlucHV0X2ZlYXR1cmVzID0gc2V0KGlucHV0cy5jb2x1bW5zKQogICAgaWYgbGVuKHNhbXBsZV9mZWF0dXJlcyAmIGlucHV0X2ZlYXR1cmVzKSAhPSBsZW4oaW5wdXRfZmVhdHVyZXMpOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIk5vdCBhbGwgZmVhdHVyZSBuYW1lcyB3ZXJlIG1hdGNoaW5nIGJldHdlZW4gdGhlIGlucHV0cyBhbmQgdGhlIHNhbXBsZSBzZXQgcHJvdmlkZWQ6ICIKICAgICAgICAgICAgZiJ7aW5wdXRfZmVhdHVyZXMgLSBzYW1wbGVfZmVhdHVyZXMgfCBzYW1wbGVfZmVhdHVyZXMgLSBpbnB1dF9mZWF0dXJlc30iCiAgICAgICAgKQoKICAgICMgUGxvdDoKICAgIGh0bWxfcGxvdCA9IEZlYXR1cmVzRHJpZnRUYWJsZVBsb3QoKS5wcm9kdWNlKAogICAgICAgIGZlYXR1cmVzPWxpc3QoaW5wdXRfZmVhdHVyZXMpLAogICAgICAgIHNhbXBsZV9zZXRfc3RhdGlzdGljcz1zYW1wbGVfc2V0X3N0YXRpc3RpY3MsCiAgICAgICAgaW5wdXRzX3N0YXRpc3RpY3M9aW5wdXRzX3N0YXRpc3RpY3MsCiAgICAgICAgbWV0cmljcz1tZXRyaWNzLAogICAgICAgIGRyaWZ0X3Jlc3VsdHM9ZHJpZnRfcmVzdWx0cywKICAgICkKCiAgICAjIFByZXBhcmUgbWV0cmljcyBwZXIgZmVhdHVyZSBkaWN0aW9uYXJ5OgogICAgbWV0cmljc19wZXJfZmVhdHVyZSA9IHsKICAgICAgICBmZWF0dXJlOiBfZ2V0X2RyaWZ0X3Jlc3VsdCgKICAgICAgICAgICAgdHZkPW1ldHJpY19kaWN0aW9uYXJ5WyJ0dmQiXSwKICAgICAgICAgICAgaGVsbGluZ2VyPW1ldHJpY19kaWN0aW9uYXJ5WyJoZWxsaW5nZXIiXSwKICAgICAgICAgICAgdGhyZXNob2xkPWRyaWZ0X3RocmVzaG9sZCwKICAgICAgICApWzFdCiAgICAgICAgZm9yIGZlYXR1cmUsIG1ldHJpY19kaWN0aW9uYXJ5IGluIG1ldHJpY3MuaXRlbXMoKQogICAgICAgIGlmIGlzaW5zdGFuY2UobWV0cmljX2RpY3Rpb25hcnksIGRpY3QpCiAgICB9CgogICAgIyBDYWxjdWxhdGUgdGhlIGZpbmFsIGFuYWx5c2lzIHJlc3VsdDoKICAgIGRyaWZ0X3N0YXR1cywgZHJpZnRfbWV0cmljID0gX2dldF9kcmlmdF9yZXN1bHQoCiAgICAgICAgdHZkPW1ldHJpY3NbInR2ZF9tZWFuIl0sCiAgICAgICAgaGVsbGluZ2VyPW1ldHJpY3NbImhlbGxpbmdlcl9tZWFuIl0sCiAgICAgICAgdGhyZXNob2xkPWRyaWZ0X3RocmVzaG9sZCwKICAgICkKCiAgICByZXR1cm4gKAogICAgICAgIEFydGlmYWN0KGJvZHk9aHRtbF9wbG90LCBmb3JtYXQ9Imh0bWwiLCBrZXk9ImRyaWZ0X3RhYmxlX3Bsb3QiKSwKICAgICAgICBBcnRpZmFjdCgKICAgICAgICAgICAgYm9keT1qc29uLmR1bXBzKG1ldHJpY3NfcGVyX2ZlYXR1cmUpLAogICAgICAgICAgICBmb3JtYXQ9Impzb24iLAogICAgICAgICAgICBrZXk9ImZlYXR1cmVzX2RyaWZ0X3Jlc3VsdHMiLAogICAgICAgICksCiAgICAgICAgeyJkcmlmdF9zdGF0dXMiOiBkcmlmdF9zdGF0dXMsICJkcmlmdF9tZXRyaWMiOiBkcmlmdF9tZXRyaWN9LAogICAgKQoKCmRlZiBpbmZlcigKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgbW9kZWw6IHN0ciwKICAgIGRhdGFzZXQ6IERhdGFzZXRUeXBlLAogICAgZHJvcF9jb2x1bW5zOiBzdHIgfCBsaXN0W3N0cl0gfCBpbnQgfCBsaXN0W2ludF0gPSBOb25lLAogICAgbGFiZWxfY29sdW1uczogc3RyIHwgbGlzdFtzdHJdID0gTm9uZSwKICAgIGZlYXR1cmVfY29sdW1uczogc3RyIHwgbGlzdFtzdHJdID0gTm9uZSwKICAgIGxvZ19yZXN1bHRfc2V0OiBib29sID0gVHJ1ZSwKICAgIHJlc3VsdF9zZXRfbmFtZTogc3RyID0gInByZWRpY3Rpb24iLAogICAgYmF0Y2hfaWQ6IHN0ciA9IE5vbmUsCiAgICBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzOiBib29sID0gTm9uZSwKICAgIHNhbXBsZV9zZXQ6IERhdGFzZXRUeXBlID0gTm9uZSwKICAgIGRyaWZ0X3RocmVzaG9sZDogZmxvYXQgPSAwLjcsCiAgICBwb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQ6IGZsb2F0ID0gMC41LAogICAgaW5mX2NhcHBpbmc6IGZsb2F0ID0gMTAuMCwKICAgIGFydGlmYWN0c190YWc6IHN0ciA9ICIiLAogICAgKipwcmVkaWN0X2t3YXJnczogZGljdFtzdHIsIEFueV0sCik6CiAgICAiIiIKICAgIFBlcmZvcm0gYSBwcmVkaWN0aW9uIG9uIGEgZ2l2ZW4gZGF0YXNldCB3aXRoIHRoZSBnaXZlbiBtb2RlbC4gQ2FuIHBlcmZvcm0gZHJpZnQgYW5hbHlzaXMgYmV0d2VlbiB0aGUgc2FtcGxlIHNldAogICAgc3RhdGlzdGljcyBzdG9yZWQgaW4gdGhlIG1vZGVsIHRvIHRoZSBjdXJyZW50IGlucHV0IGRhdGEuIFRoZSBkcmlmdCBydWxlIGlzIHRoZSB2YWx1ZSBwZXItZmVhdHVyZSBtZWFuIG9mIHRoZSBUVkQKICAgIGFuZCBIZWxsaW5nZXIgc2NvcmVzIGFjY29yZGluZyB0byB0aGUgdGhyZXNob2xkcyBjb25maWd1cmVzIGhlcmUuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgIFRoZSBtb2RlbCBTdG9yZSBwYXRoLgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gaW5mZXIgdGhyb3VnaCB0aGUgbW9kZWwuIENhbiBiZSBwYXNzZWQgaW4gYGlucHV0c2AgYXMgZWl0aGVyIGEKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERhdGFzZXQgYXJ0aWZhY3QgLyBGZWF0dXJlIHZlY3RvciBVUkkuIE9yLCBpbiBgcGFyYW1ldGVyc2AgYXMgYSBsaXN0LCBkaWN0aW9uYXJ5IG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBudW1weSBhcnJheS4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgIEEgc3RyaW5nIC8gaW50ZWdlciBvciBhIGxpc3Qgb2Ygc3RyaW5ncyAvIGludGVnZXJzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC8gaW5kaWNlcyB0byBkcm9wLiBXaGVuIHRoZSBkYXRhc2V0IGlzIGEgbGlzdCBvciBhIG51bXB5IGFycmF5IHRoaXMgcGFyYW1ldGVyIG11c3QKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJlIHJlcHJlc2VudGVkIGJ5IGludGVnZXJzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0IGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gVGhlIGxhYmVsIGNvbHVtbiBjYW4gYmUgYWNjZXNzZWQgZnJvbSB0aGUgbW9kZWwgb2JqZWN0LCBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhlIGZlYXR1cmUgdmVjdG9yIHByb3ZpZGVkIGlmIGF2YWlsYWJsZS4KICAgIDpwYXJhbSBmZWF0dXJlX2NvbHVtbnM6ICAgICAgICAgIExpc3Qgb2YgZmVhdHVyZSBjb2x1bW5zIHRoYXQgd2lsbCBiZSB1c2VkIHRvIGJ1aWxkIHRoZSBkYXRhZnJhbWUgd2hlbiBkYXRhc2V0IGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmcm9tIHR5cGUgbGlzdCBvciBudW1weSBhcnJheS4KICAgIDpwYXJhbSBsb2dfcmVzdWx0X3NldDogICAgICAgICAgIFdoZXRoZXIgdG8gbG9nIHRoZSByZXN1bHQgc2V0IC0gYSBEYXRhRnJhbWUgb2YgdGhlIGdpdmVuIGlucHV0cyBjb25jYXRlbmF0ZWQgd2l0aAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhlIHByZWRpY3Rpb25zLiBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgIDpwYXJhbSByZXN1bHRfc2V0X25hbWU6ICAgICAgICAgIFRoZSBkYiBrZXkgdG8gc2V0IG5hbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0IGFuZCB0aGUgZmlsZW5hbWUuIERlZmF1bHRlZCB0bwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ3ByZWRpY3Rpb24nLgogICAgOnBhcmFtIGJhdGNoX2lkOiAgICAgICAgICAgICAgICAgVGhlIElEIG9mIHRoZSBnaXZlbiBiYXRjaCAoaW5mZXJlbmNlIGRhdGFzZXQpLiBJZiBgTm9uZWAsIGl0IHdpbGwgYmUgZ2VuZXJhdGVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV2lsbCBiZSBsb2dnZWQgYXMgYSByZXN1bHQgb2YgdGhlIHJ1bi4KICAgIDpwYXJhbSBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzOiAgIFdoZXRoZXIgdG8gcGVyZm9ybSBkcmlmdCBhbmFseXNpcyBiZXR3ZWVuIHRoZSBzYW1wbGUgc2V0IG9mIHRoZSBtb2RlbCBvYmplY3QgdG8gdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkYXRhc2V0IGdpdmVuLiBCeSBkZWZhdWx0LCBOb25lLCB3aGljaCBtZWFucyBpdCB3aWxsIHBlcmZvcm0gZHJpZnQgYW5hbHlzaXMgaWYgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlbCBoYXMgYSBzYW1wbGUgc2V0IHN0YXRpc3RpY3MuIFBlcmZvcm0gZHJpZnQgYW5hbHlzaXMgd2lsbCBwcm9kdWNlIGEgZGF0YSBkcmlmdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGFibGUgYXJ0aWZhY3QuCiAgICA6cGFyYW0gc2FtcGxlX3NldDogICAgICAgICAgICAgICBBIHNhbXBsZSBkYXRhc2V0IHRvIGdpdmUgdG8gY29tcGFyZSB0aGUgaW5wdXRzIGluIHRoZSBkcmlmdCBhbmFseXNpcy4gVGhlIGRlZmF1bHQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGNob3NlbiBzYW1wbGUgc2V0IHdpbGwgYWx3YXlzIGJlIHRoZSBvbmUgd2hvIGlzIHNldCBpbiB0aGUgbW9kZWwgYXJ0aWZhY3QgaXRzZWxmLgogICAgOnBhcmFtIGRyaWZ0X3RocmVzaG9sZDogICAgICAgICAgVGhlIHRocmVzaG9sZCBvZiB3aGljaCB0byBtYXJrIGRyaWZ0cy4gRGVmYXVsdGVkIHRvIDAuNy4KICAgIDpwYXJhbSBwb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQ6IFRoZSB0aHJlc2hvbGQgb2Ygd2hpY2ggdG8gbWFyayBwb3NzaWJsZSBkcmlmdHMuIERlZmF1bHRlZCB0byAwLjUuCiAgICA6cGFyYW0gaW5mX2NhcHBpbmc6ICAgICAgICAgICAgICBUaGUgdmFsdWUgdG8gc2V0IGZvciB3aGVuIGl0IHJlYWNoZWQgaW5maW5pdHkuIERlZmF1bHRlZCB0byAxMC4wLgogICAgOnBhcmFtIGFydGlmYWN0c190YWc6ICAgICAgICAgICAgVGFnIHRvIHVzZSBmb3IgYWxsIHRoZSBhcnRpZmFjdHMgcmVzdWx0ZWQgZnJvbSB0aGUgZnVuY3Rpb24uCiAgICAiIiIKICAgICMgTG9hZGluZyB0aGUgbW9kZWw6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJMb2FkaW5nIG1vZGVsLi4uIikKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbChtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQpCiAgICBpZiBsYWJlbF9jb2x1bW5zIGlzIE5vbmU6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFsKICAgICAgICAgICAgb3V0cHV0Lm5hbWUgZm9yIG91dHB1dCBpbiBtb2RlbF9oYW5kbGVyLl9tb2RlbF9hcnRpZmFjdC5zcGVjLm91dHB1dHMKICAgICAgICBdCgogICAgaWYgZmVhdHVyZV9jb2x1bW5zIGlzIE5vbmU6CiAgICAgICAgZmVhdHVyZV9jb2x1bW5zID0gWwogICAgICAgICAgICBpbnB1dC5uYW1lIGZvciBpbnB1dCBpbiBtb2RlbF9oYW5kbGVyLl9tb2RlbF9hcnRpZmFjdC5zcGVjLmlucHV0cwogICAgICAgIF0KCiAgICAjIEdldCBkYXRhc2V0IGJ5IG9iamVjdCwgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJMb2FkaW5nIGRhdGEuLi4iKQogICAgeCwgbGFiZWxfY29sdW1ucyA9IF9yZWFkX2RhdGFzZXRfYXNfZGF0YWZyYW1lKAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBmZWF0dXJlX2NvbHVtbnM9ZmVhdHVyZV9jb2x1bW5zLAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgUHJlZGljdDoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkNhbGN1bGF0aW5nIHByZWRpY3Rpb24uLi4iKQogICAgeV9wcmVkID0gbW9kZWxfaGFuZGxlci5tb2RlbC5wcmVkaWN0KHgsICoqcHJlZGljdF9rd2FyZ3MpCgogICAgIyBQcmVwYXJlIHRoZSByZXN1bHQgc2V0OgogICAgcmVzdWx0X3NldCA9IF9wcmVwYXJlX3Jlc3VsdF9zZXQoeD14LCBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsIHlfcHJlZD15X3ByZWQpCgogICAgIyBDaGVjayBmb3IgbG9nZ2luZyB0aGUgcmVzdWx0IHNldDoKICAgIGlmIGxvZ19yZXN1bHRfc2V0OgogICAgICAgICMgTG9nIHRoZSByZXN1bHQgc2V0OgogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkxvZ2dpbmcgcmVzdWx0IHNldCAoeCB8IHByZWRpY3Rpb24pLi4uIikKICAgICAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgICAgICBrZXk9cmVzdWx0X3NldF9uYW1lLAogICAgICAgICAgICBkZj1yZXN1bHRfc2V0LAogICAgICAgICAgICBkYl9rZXk9cmVzdWx0X3NldF9uYW1lLAogICAgICAgICAgICB0YWc9YXJ0aWZhY3RzX3RhZywKICAgICAgICApCiAgICAgICAgIyBMb2cgdGhlIGJhdGNoIElEOgogICAgICAgIGlmIGJhdGNoX2lkIGlzIE5vbmU6CiAgICAgICAgICAgIGJhdGNoX2lkID0gaGFzaGxpYi5zaGEyMjQoc3RyKGRhdGV0aW1lLm5vdygpKS5lbmNvZGUoKSkuaGV4ZGlnZXN0KCkKICAgICAgICBjb250ZXh0LmxvZ19yZXN1bHQoCiAgICAgICAgICAgIGtleT0iYmF0Y2hfaWQiLAogICAgICAgICAgICB2YWx1ZT1iYXRjaF9pZCwKICAgICAgICApCgogICAgIyBDaGVjayBmb3IgcGVyZm9ybWluZyBkcmlmdCBhbmFseXNpczoKICAgIGlmICgKICAgICAgICBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzIGlzIE5vbmUKICAgICAgICBhbmQgbW9kZWxfaGFuZGxlci5fbW9kZWxfYXJ0aWZhY3Quc3BlYy5mZWF0dXJlX3N0YXRzIGlzIG5vdCBOb25lCiAgICApOgogICAgICAgIHBlcmZvcm1fZHJpZnRfYW5hbHlzaXMgPSBUcnVlCiAgICBpZiBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIlBlcmZvcm1pbmcgZHJpZnQgYW5hbHlzaXMuLi4iKQogICAgICAgICMgR2V0IHRoZSBzYW1wbGUgc2V0IHN0YXRpc3RpY3MgKGVpdGhlciBmcm9tIHRoZSBzYW1wbGUgc2V0IG9yIGZyb20gdGhlIHN0YXRpc3RpY3MgbG9nZ2VkIHdpdGggdGhlIG1vZGVsKToKICAgICAgICBzYW1wbGVfc2V0X3N0YXRpc3RpY3MgPSBfZ2V0X3NhbXBsZV9zZXRfc3RhdGlzdGljcygKICAgICAgICAgICAgc2FtcGxlX3NldD1zYW1wbGVfc2V0LAogICAgICAgICAgICBtb2RlbF9hcnRpZmFjdF9mZWF0dXJlX3N0YXRzPW1vZGVsX2hhbmRsZXIuX21vZGVsX2FydGlmYWN0LnNwZWMuZmVhdHVyZV9zdGF0cywKICAgICAgICApCiAgICAgICAgIyBQcm9kdWNlIHRoZSBhcnRpZmFjdDoKICAgICAgICAoCiAgICAgICAgICAgIGRyaWZ0X3RhYmxlX3Bsb3QsCiAgICAgICAgICAgIG1ldHJpY19wZXJfZmVhdHVyZV9kaWN0LAogICAgICAgICAgICBhbmFseXNpc19yZXN1bHRzLAogICAgICAgICkgPSBfcGVyZm9ybV9kcmlmdF9hbmFseXNpcygKICAgICAgICAgICAgc2FtcGxlX3NldF9zdGF0aXN0aWNzPXNhbXBsZV9zZXRfc3RhdGlzdGljcywKICAgICAgICAgICAgaW5wdXRzPXJlc3VsdF9zZXQsCiAgICAgICAgICAgIGRyaWZ0X3RocmVzaG9sZD1kcmlmdF90aHJlc2hvbGQsCiAgICAgICAgICAgIHBvc3NpYmxlX2RyaWZ0X3RocmVzaG9sZD1wb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQsCiAgICAgICAgICAgIGluZl9jYXBwaW5nPWluZl9jYXBwaW5nLAogICAgICAgICkKICAgICAgICAjIExvZyB0aGUgYXJ0aWZhY3QgYW5kIHJlc3VsdHM6CiAgICAgICAgY29udGV4dC5sb2dfYXJ0aWZhY3QoZHJpZnRfdGFibGVfcGxvdCwgdGFnPWFydGlmYWN0c190YWcpCiAgICAgICAgY29udGV4dC5sb2dfYXJ0aWZhY3QobWV0cmljX3Blcl9mZWF0dXJlX2RpY3QsIHRhZz1hcnRpZmFjdHNfdGFnKQogICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdHMocmVzdWx0cz1hbmFseXNpc19yZXN1bHRzKQo= + code_origin: '' + auto_build: false + allow_empty_resources: true + filename: batch_inference.py entry_points: infer: - name: infer - doc: 'Perform a prediction on a given dataset with the given model. Can perform - drift analysis between the sample set - - statistics stored in the model to the current input data. The drift rule is - the value per-feature mean of the TVD - - and Hellinger scores according to the thresholds configures here.' parameters: - name: context type: MLClientCtx @@ -30,19 +31,16 @@ spec: either a Dataset artifact / Feature vector URI. Or, in `parameters` as a list, dictionary or numpy array. - name: drop_columns - type: Union[str, List[str], int, List[int]] doc: A string / integer or a list of strings / integers that represent the column names / indices to drop. When the dataset is a list or a numpy array this parameter must be represented by integers. default: null - name: label_columns - type: Union[str, List[str]] doc: The target label(s) of the column(s) in the dataset for Regression or Classification tasks. The label column can be accessed from the model object, or the feature vector provided if available. default: null - name: feature_columns - type: Union[str, List[str]] doc: List of feature columns that will be used to build the dataframe when dataset is from type list or numpy array. default: null @@ -90,18 +88,18 @@ spec: type: str doc: Tag to use for all the artifacts resulted from the function. default: '' - lineno: 317 + name: infer + doc: 'Perform a prediction on a given dataset with the given model. Can perform + drift analysis between the sample set + + statistics stored in the model to the current input data. The drift rule is + the value per-feature mean of the TVD + + and Hellinger scores according to the thresholds configures here.' has_kwargs: true has_varargs: false - allow_empty_resources: true - default_handler: infer + lineno: 318 command: '' - build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IGhhc2hsaWIKaW1wb3J0IGpzb24KZnJvbSBkYXRldGltZSBpbXBvcnQgZGF0ZXRpbWUKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgRGljdCwgTGlzdCwgVHVwbGUsIFVuaW9uCmltcG9ydCBzZW12ZXIKCmltcG9ydCBtbHJ1bgppZiBzZW12ZXIuY29tcGFyZShtbHJ1bi5fX3ZlcnNpb25fXywgIjEuNS4wIikgPj0gMDoKICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bk5vdEZvdW5kRXJyb3IoCiAgICAgICAgZiJXaGVuIHVzaW5nIGBtbHJ1bmAgdmVyc2lvbiA+PTEuNS4wLCBwbGVhc2UgdXNlICIKICAgICAgICBmImJhdGNoIGluZmVyZW5jZSBgdjJgIGZ1bmN0aW9uICgnaHViOi8vYmF0Y2hfaW5mZXJlbmNlX3YyJykuIgogICAgKQoKaW1wb3J0IG1scnVuLmRhdGFzdG9yZQppbXBvcnQgbWxydW4udXRpbHMKaW1wb3J0IG51bXB5IGFzIG5wCmltcG9ydCBwYW5kYXMgYXMgcGQKZnJvbSBtbHJ1biBpbXBvcnQgZmVhdHVyZV9zdG9yZSBhcyBmcwpmcm9tIG1scnVuLmFydGlmYWN0cyBpbXBvcnQgQXJ0aWZhY3QKZnJvbSBtbHJ1bi5kYXRhX3R5cGVzLmluZmVyIGltcG9ydCBJbmZlck9wdGlvbnMsIGdldF9kZl9zdGF0cwpmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4ubW9kZWxfbW9uaXRvcmluZy5mZWF0dXJlc19kcmlmdF90YWJsZSBpbXBvcnQgRmVhdHVyZXNEcmlmdFRhYmxlUGxvdApmcm9tIG1scnVuLm1vZGVsX21vbml0b3JpbmcubW9kZWxfbW9uaXRvcmluZ19iYXRjaCBpbXBvcnQgKAogICAgVmlydHVhbERyaWZ0LAogICAgY2FsY3VsYXRlX2lucHV0c19zdGF0aXN0aWNzLAopCgojIEEgdW5pb24gb2YgYWxsIHN1cHBvcnRlZCBkYXRhc2V0IHR5cGVzOgpEYXRhc2V0VHlwZSA9IFVuaW9uW21scnVuLkRhdGFJdGVtLCBsaXN0LCBkaWN0LCBwZC5EYXRhRnJhbWUsIHBkLlNlcmllcywgbnAubmRhcnJheV0KCgpkZWYgX3JlYWRfZGF0YXNldF9hc19kYXRhZnJhbWUoCiAgICBkYXRhc2V0OiBEYXRhc2V0VHlwZSwKICAgIGZlYXR1cmVfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IFVuaW9uW3N0ciwgTGlzdFtzdHJdXSA9IE5vbmUsCiAgICBkcm9wX2NvbHVtbnM6IFVuaW9uW3N0ciwgTGlzdFtzdHJdLCBpbnQsIExpc3RbaW50XV0gPSBOb25lLAopIC0+IFR1cGxlW3BkLkRhdGFGcmFtZSwgTGlzdFtzdHJdXToKICAgICIiIgogICAgUGFyc2UgdGhlIGdpdmVuIGRhdGFzZXQgaW50byBhIERhdGFGcmFtZSBhbmQgZHJvcCB0aGUgY29sdW1ucyBhY2NvcmRpbmdseS4gSW4gYWRkaXRpb24sIHRoZSBsYWJlbCBjb2x1bW5zIHdpbGwgYmUKICAgIHBhcnNlZCBhbmQgdmFsaWRhdGVkIGFzIHdlbGwuCgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgQSBkYXRhc2V0IHRoYXQgd2lsbCBiZSBjb252ZXJ0ZWQgaW50byBhIERhdGFGcmFtZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgIENhbiBiZSBlaXRoZXIgYSBsaXN0IG9mIGxpc3RzLCBkaWN0LCBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGZlYXR1cmVfY29sdW1uczogTGlzdCBvZiBmZWF0dXJlIGNvbHVtbnMgdGhhdCB3aWxsIGJlIHVzZWQgdG8gYnVpbGQgdGhlIGRhdGFmcmFtZSB3aGVuIGRhdGFzZXQgaXMgZnJvbQogICAgICAgICAgICAgICAgICAgICAgICAgICAgdHlwZSBsaXN0IG9yIG51bXB5IGFycmF5LgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2xhc3NpZmljYXRpb24gdGFza3MuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICBgYHN0cmBgIC8gYGBpbnRgYCBvciBhIGxpc3Qgb2YgYGBzdHJgYCAvIGBgaW50YGAgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbiBuYW1lcyAvIGluZGljZXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRvIGRyb3AuCgogICAgOnJldHVybnM6IEEgdHVwbGUgb2Y6CiAgICAgICAgICAgICAgWzBdID0gVGhlIHBhcnNlZCBkYXRhc2V0IGFzIGEgRGF0YUZyYW1lCiAgICAgICAgICAgICAgWzFdID0gTGFiZWwgY29sdW1ucy4KCiAgICByYWlzZXMgTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcjogSWYgdGhlIGBkcm9wX2NvbHVtbnNgIGFyZSBub3QgbWF0Y2hpbmcgdGhlIGRhdGFzZXQgb3IgdW5zdXBwb3J0ZWQgZGF0YXNldCB0eXBlLgogICAgIiIiCiAgICAjIFR1cm4gdGhlIGBkcm9wIGxhYmVsc2AgaW50byBhIGxpc3QgaWYgZ2l2ZW46CiAgICBpZiBkcm9wX2NvbHVtbnMgaXMgbm90IE5vbmU6CiAgICAgICAgaWYgbm90IGlzaW5zdGFuY2UoZHJvcF9jb2x1bW5zLCBsaXN0KToKICAgICAgICAgICAgZHJvcF9jb2x1bW5zID0gW2Ryb3BfY29sdW1uc10KCiAgICAjIENoZWNrIGlmIHRoZSBkYXRhc2V0IGlzIGluIGZhY3QgYSBGZWF0dXJlIFZlY3RvcjoKICAgIGlmIGlzaW5zdGFuY2UoZGF0YXNldCwgZnMuRmVhdHVyZVZlY3Rvcik6CiAgICAgICAgIyBUcnkgdG8gZ2V0IHRoZSBsYWJlbCBjb2x1bW5zIGlmIG5vdCBwcm92aWRlZDoKICAgICAgICBpZiBsYWJlbF9jb2x1bW5zIGlzIE5vbmU6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBkYXRhc2V0LnN0YXR1cy5sYWJlbF9jb2x1bW4KICAgICAgICAjIEdldCB0aGUgZmVhdHVyZXMgYW5kIHBhcnNlIHRvIERhdGFGcmFtZToKICAgICAgICBkYXRhc2V0ID0gZnMuZ2V0X29mZmxpbmVfZmVhdHVyZXMoCiAgICAgICAgICAgIGRhdGFzZXQudXJpLCBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zCiAgICAgICAgKS50b19kYXRhZnJhbWUoKQoKICAgIGVsaWYgaXNpbnN0YW5jZShkYXRhc2V0LCAobGlzdCwgbnAubmRhcnJheSkpOgogICAgICAgIGlmIG5vdCBmZWF0dXJlX2NvbHVtbnM6CiAgICAgICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICAgICAgIkZlYXR1cmUgY29sdW1ucyBsaXN0IG11c3QgYmUgcHJvdmlkZWQgd2hlbiBkYXRhc2V0IGlucHV0IGFzIGZyb20gdHlwZSBsaXN0IG9yIG51bXB5IGFycmF5IgogICAgICAgICAgICApCiAgICAgICAgIyBQYXJzZSB0aGUgbGlzdCAvIG51bXB5IGFycmF5IGludG8gYSBEYXRhRnJhbWU6CiAgICAgICAgZGF0YXNldCA9IHBkLkRhdGFGcmFtZShkYXRhc2V0LCBjb2x1bW5zPWZlYXR1cmVfY29sdW1ucykKICAgICAgICAjIFZhbGlkYXRlIHRoZSBgZHJvcF9jb2x1bW5zYCBpcyBnaXZlbiBhcyBpbnRlZ2VyczoKICAgICAgICBpZiBkcm9wX2NvbHVtbnMgYW5kIG5vdCBhbGwoaXNpbnN0YW5jZShjb2wsIGludCkgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgICAgICJgZHJvcF9jb2x1bW5zYCBtdXN0IGJlIGFuIGludGVnZXIgLyBsaXN0IG9mIGludGVnZXJzIGlmIHByb3ZpZGVkIGFzIGEgbGlzdC4iCiAgICAgICAgICAgICkKICAgIGVsaWYgaXNpbnN0YW5jZShkYXRhc2V0LCBtbHJ1bi5EYXRhSXRlbSk6CiAgICAgICAgIyBUdXJuIHRoZSBEYXRhSVRlbSB0byBEYXRhRnJhbWU6CiAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuYXNfZGYoKQogICAgZWxzZToKICAgICAgICAjIFBhcnNlIHRoZSBvYmplY3QgKHNob3VsZCBiZSBhIHBkLkRhdGFGcmFtZSAvIHBkLlNlcmllcywgZGljdGlvbmFyeSkgaW50byBhIERhdGFGcmFtZToKICAgICAgICB0cnk6CiAgICAgICAgICAgIGRhdGFzZXQgPSBwZC5EYXRhRnJhbWUoZGF0YXNldCkKICAgICAgICBleGNlcHQgVmFsdWVFcnJvciBhcyBlOgogICAgICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgICAgIGYiQ291bGQgbm90IHBhcnNlIHRoZSBnaXZlbiBkYXRhc2V0IG9mIHR5cGUge3R5cGUoZGF0YXNldCl9IGludG8gYSBwYW5kYXMgRGF0YUZyYW1lLiAiCiAgICAgICAgICAgICAgICBmIlJlY2VpdmVkIHRoZSBmb2xsb3dpbmcgZXJyb3I6IHtlfSIKICAgICAgICAgICAgKQogICAgIyBEcm9wIGNvbHVtbnMgaWYgbmVlZGVkOgogICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgIGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSwgaW5wbGFjZT1UcnVlKQoKICAgICMgVHVybiB0aGUgYGxhYmVsX2NvbHVtbnNgIGludG8gYSBsaXN0IGJ5IGRlZmF1bHQ6CiAgICBpZiBsYWJlbF9jb2x1bW5zIGlzIE5vbmU6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtdCiAgICBlbGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgKHN0ciwgaW50KSk6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtsYWJlbF9jb2x1bW5zXQogICAgcmV0dXJuIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMKCgpkZWYgX3ByZXBhcmVfcmVzdWx0X3NldCgKICAgIHg6IHBkLkRhdGFGcmFtZSwgbGFiZWxfY29sdW1uczogTGlzdFtzdHJdLCB5X3ByZWQ6IG5wLm5kYXJyYXkKKSAtPiBwZC5EYXRhRnJhbWU6CiAgICAiIiIKICAgIFNldCBkZWZhdWx0IGxhYmVsIGNvbHVtbiBuYW1lcyBhbmQgdmFsaWRhdGUgZ2l2ZW4gbmFtZXMgdG8gcHJlcGFyZSB0aGUgcmVzdWx0IHNldCAtIGEgY29uY2F0ZW5hdGlvbiBvZiB0aGUgaW5wdXRzCiAgICAoeCkgYW5kIHRoZSBtb2RlbCBwcmVkaWN0aW9ucyAoeV9wcmVkKS4KCiAgICA6cGFyYW0geDogICAgICAgICAgICAgVGhlIGlucHV0cy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiBBIGxpc3Qgb2Ygc3RyaW5ncyByZXByZXNlbnRpbmcgdGhlIHRhcmdldCBjb2x1bW4gbmFtZXMgdG8gYWRkIHRvIHRoZSBwcmVkaWN0aW9ucy4gRGVmYXVsdCBuYW1lCiAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSB1c2VkIGluIGNhc2UgdGhlIGxpc3QgaXMgZW1wdHkgKHByZWRpY3RlZF9sYWJlbF97aX0pLgogICAgOnBhcmFtIHlfcHJlZDogICAgICAgIFRoZSBtb2RlbCBwcmVkaWN0aW9ucyBvbiB0aGUgaW5wdXRzLgoKICAgIDpyZXR1cm5zOiBUaGUgcmVzdWx0IHNldC4KCiAgICByYWlzZXMgTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcjogSWYgdGhlIGxhYmVscyBjb2x1bW5zIGFtb3VudCBkbyBub3QgbWF0Y2ggdGhlIG91dHB1dHMgb3IgaWYgb25lIG9mIHRoZSBsYWJlbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb2x1bW4gYWxyZWFkeSBleGlzdHMgaW4gdGhlIGRhdGFzZXQuCiAgICAiIiIKICAgICMgUHJlcGFyZSBkZWZhdWx0IHRhcmdldCBjb2x1bW5zIG5hbWVzIGlmIG5vdCBwcm92aWRlZDoKICAgIHByZWRpY3Rpb25fY29sdW1uc19hbW91bnQgPSAxIGlmIGxlbih5X3ByZWQuc2hhcGUpID09IDEgZWxzZSB5X3ByZWQuc2hhcGVbMV0KICAgIGlmIGxlbihsYWJlbF9jb2x1bW5zKSA9PSAwOgogICAgICAgICMgQWRkIGRlZmF1bHQgbGFiZWwgY29sdW1uIG5hbWVzOgogICAgICAgIGlmIHByZWRpY3Rpb25fY29sdW1uc19hbW91bnQgPT0gMToKICAgICAgICAgICAgbGFiZWxfY29sdW1ucyA9IFsicHJlZGljdGVkX2xhYmVsIl0KICAgICAgICBlbHNlOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWwogICAgICAgICAgICAgICAgZiJwcmVkaWN0ZWRfbGFiZWxfe2l9IiBmb3IgaSBpbiByYW5nZShwcmVkaWN0aW9uX2NvbHVtbnNfYW1vdW50KQogICAgICAgICAgICBdCgogICAgIyBWYWxpZGF0ZSB0aGUgbGFiZWwgY29sdW1uczoKICAgIGlmIHByZWRpY3Rpb25fY29sdW1uc19hbW91bnQgIT0gbGVuKGxhYmVsX2NvbHVtbnMpOgogICAgICAgICMgTm8gZXF1YWxpdHkgYmV0d2VlbiBwcm92aWRlZCBsYWJlbCBjb2x1bW4gbmFtZXMgYW5kIG91dHB1dHMgYW1vdW50OgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIlRoZSBudW1iZXIgb2YgcHJlZGljdGVkIGxhYmVsczoge3ByZWRpY3Rpb25fY29sdW1uc19hbW91bnR9ICIKICAgICAgICAgICAgZiJpcyBub3QgZXF1YWwgdG8gdGhlIGdpdmVuIGxhYmVsIGNvbHVtbnM6IHtsZW4obGFiZWxfY29sdW1ucyl9IgogICAgICAgICkKICAgIGNvbW1vbl9sYWJlbHMgPSBzZXQobGFiZWxfY29sdW1ucykgJiBzZXQoeC5jb2x1bW5zLnRvbGlzdCgpKQogICAgaWYgY29tbW9uX2xhYmVsczoKICAgICAgICAjIExhYmVsIGNvbHVtbiBleGlzdCBpbiB0aGUgb3JpZ2luYWwgaW5wdXRzOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIlRoZSBsYWJlbHM6IHtjb21tb25fbGFiZWxzfSBhcmUgYWxyZWFkeSBleGlzdGVkIGluIHRoZSBnaXZlbiBkYXRhc2V0LiIKICAgICAgICApCgogICAgcmV0dXJuIHBkLmNvbmNhdCgKICAgICAgICBbeCwgcGQuRGF0YUZyYW1lKHlfcHJlZCwgY29sdW1ucz1sYWJlbF9jb2x1bW5zLCBpbmRleD14LmluZGV4KV0sIGF4aXM9MQogICAgKQoKCmRlZiBfZ2V0X3NhbXBsZV9zZXRfc3RhdGlzdGljcygKICAgIHNhbXBsZV9zZXQ6IERhdGFzZXRUeXBlID0gTm9uZSwgbW9kZWxfYXJ0aWZhY3RfZmVhdHVyZV9zdGF0czogZGljdCA9IE5vbmUKKSAtPiBkaWN0OgogICAgIiIiCiAgICBHZXQgdGhlIHNhbXBsZSBzZXQgc3RhdGlzdGljcyBlaXRoZXIgZnJvbSB0aGUgZ2l2ZW4gc2FtcGxlIHNldCBvciB0aGUgc3RhdGlzdGljcyBsb2dnZWQgd2l0aCB0aGUgbW9kZWwgd2hpbGUKICAgIGZhdm9yaW5nIHRoZSBnaXZlbiBzYW1wbGUgc2V0LgoKICAgIDpwYXJhbSBzYW1wbGVfc2V0OiAgICAgICAgICAgICAgICAgICBBIHNhbXBsZSBkYXRhc2V0IHRvIGdpdmUgdG8gY29tcGFyZSB0aGUgaW5wdXRzIGluIHRoZSBkcmlmdCBhbmFseXNpcy4KICAgIDpwYXJhbSBtb2RlbF9hcnRpZmFjdF9mZWF0dXJlX3N0YXRzOiBUaGUgYGZlYXR1cmVfc3RhdHNgIGF0dHJpYnV0ZSBpbiB0aGUgc3BlYyBvZiB0aGUgbW9kZWwgYXJ0aWZhY3QsIHdoZXJlIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9yaWdpbmFsIHNhbXBsZSBzZXQgc3RhdGlzdGljcyBvZiB0aGUgbW9kZWwgd2FzIHVzZWQuCgogICAgOnJldHVybnM6IFRoZSBzYW1wbGUgc2V0IHN0YXRpc3RpY3MuCgogICAgcmFpc2VzIE1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3I6IElmIG5vIHNhbXBsZSBzZXQgb3Igc3RhdGlzdGljcyB3ZXJlIGdpdmVuLgogICAgIiIiCiAgICAjIENoZWNrIGlmIGEgc2FtcGxlIHNldCB3YXMgcHJvdmlkZWQ6CiAgICBpZiBzYW1wbGVfc2V0IGlzIE5vbmU6CiAgICAgICAgIyBDaGVjayBpZiB0aGUgbW9kZWwgd2FzIGxvZ2dlZCB3aXRoIGEgc2FtcGxlIHNldDoKICAgICAgICBpZiBtb2RlbF9hcnRpZmFjdF9mZWF0dXJlX3N0YXRzIGlzIE5vbmU6CiAgICAgICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICAgICAgIkNhbm5vdCBwZXJmb3JtIGRyaWZ0IGFuYWx5c2lzIGFzIHRoZXJlIGlzIG5vIHNhbXBsZSBzZXQgdG8gY29tcGFyZSB0by4gVGhlIG1vZGVsIGFydGlmYWN0IHdhcyBub3QgIgogICAgICAgICAgICAgICAgImxvZ2dlZCB3aXRoIGEgc2FtcGxlIHNldCBhbmQgYHNhbXBsZV9zZXRgIHdhcyBub3QgcHJvdmlkZWQgdG8gdGhlIGZ1bmN0aW9uLiIKICAgICAgICAgICAgKQogICAgICAgICMgUmV0dXJuIHRoZSBzdGF0aXN0aWNzIGxvZ2dlZCB3aXRoIHRoZSBtb2RlbDoKICAgICAgICByZXR1cm4gbW9kZWxfYXJ0aWZhY3RfZmVhdHVyZV9zdGF0cwoKICAgICMgVHVybiB0aGUgRGF0YUl0ZW0gdG8gRGF0YUZyYW1lOgogICAgaWYgaXNpbnN0YW5jZShzYW1wbGVfc2V0LCBtbHJ1bi5EYXRhSXRlbSk6CiAgICAgICAgc2FtcGxlX3NldCwgXyA9IF9yZWFkX2RhdGFzZXRfYXNfZGF0YWZyYW1lKGRhdGFzZXQ9c2FtcGxlX3NldCkKCiAgICAjIFJldHVybiB0aGUgc2FtcGxlIHNldCBzdGF0aXN0aWNzOgogICAgcmV0dXJuIGdldF9kZl9zdGF0cyhkZj1zYW1wbGVfc2V0LCBvcHRpb25zPUluZmVyT3B0aW9ucy5IaXN0b2dyYW0pCgoKZGVmIF9nZXRfZHJpZnRfcmVzdWx0KAogICAgdHZkOiBmbG9hdCwKICAgIGhlbGxpbmdlcjogZmxvYXQsCiAgICB0aHJlc2hvbGQ6IGZsb2F0LAopIC0+IFR1cGxlW2Jvb2wsIGZsb2F0XToKICAgICIiIgogICAgQ2FsY3VsYXRlIHRoZSBkcmlmdCByZXN1bHQgYnkgdGhlIGZvbGxvd2luZyBlcXVhdGlvbjogKHR2ZCArIGhlbGxpbmdlcikgLyAyCgogICAgOnBhcmFtIHR2ZDogICAgICAgVGhlIGZlYXR1cmUncyBUVkQgdmFsdWUuCiAgICA6cGFyYW0gaGVsbGluZ2VyOiBUaGUgZmVhdHVyZSdzIEhlbGxpbmdlciB2YWx1ZS4KICAgIDpwYXJhbSB0aHJlc2hvbGQ6IFRoZSB0aHJlc2hvbGQgZnJvbSB3aGljaCB0aGUgdmFsdWUgaXMgY29uc2lkZXJlZCBhIGRyaWZ0LgoKICAgIDpyZXR1cm5zOiBBIHR1cGxlIG9mOgogICAgICAgICAgICAgIFswXSA9IEJvb2xlYW4gdmFsdWUgYXMgdGhlIGRyaWZ0IHN0YXR1cy4KICAgICAgICAgICAgICBbMV0gPSBUaGUgcmVzdWx0LgogICAgIiIiCiAgICByZXN1bHQgPSAodHZkICsgaGVsbGluZ2VyKSAvIDIKICAgIGlmIHJlc3VsdCA+PSB0aHJlc2hvbGQ6CiAgICAgICAgcmV0dXJuIFRydWUsIHJlc3VsdAogICAgcmV0dXJuIEZhbHNlLCByZXN1bHQKCgpkZWYgX3BlcmZvcm1fZHJpZnRfYW5hbHlzaXMoCiAgICBzYW1wbGVfc2V0X3N0YXRpc3RpY3M6IGRpY3QsCiAgICBpbnB1dHM6IHBkLkRhdGFGcmFtZSwKICAgIGRyaWZ0X3RocmVzaG9sZDogZmxvYXQsCiAgICBwb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQ6IGZsb2F0LAogICAgaW5mX2NhcHBpbmc6IGZsb2F0LAopIC0+IFR1cGxlW0FydGlmYWN0LCBBcnRpZmFjdCwgZGljdF06CiAgICAiIiIKICAgIFBlcmZvcm0gZHJpZnQgYW5hbHlzaXMsIHByb2R1Y2luZyB0aGUgZHJpZnQgdGFibGUgYXJ0aWZhY3QgZm9yIGxvZ2dpbmcgcG9zdCBwcmVkaWN0aW9uLgoKICAgIDpwYXJhbSBzYW1wbGVfc2V0X3N0YXRpc3RpY3M6ICAgIFRoZSBzdGF0aXN0aWNzIG9mIHRoZSBzYW1wbGUgc2V0IGxvZ2dlZCBhbG9uZyBhIG1vZGVsLgogICAgOnBhcmFtIGlucHV0czogICAgICAgICAgICAgICAgICAgSW5wdXQgZGF0YXNldCB0byBwZXJmb3JtIHRoZSBkcmlmdCBjYWxjdWxhdGlvbiBvbi4KICAgIDpwYXJhbSBkcmlmdF90aHJlc2hvbGQ6ICAgICAgICAgIFRoZSB0aHJlc2hvbGQgb2Ygd2hpY2ggdG8gbWFyayBkcmlmdHMuCiAgICA6cGFyYW0gcG9zc2libGVfZHJpZnRfdGhyZXNob2xkOiBUaGUgdGhyZXNob2xkIG9mIHdoaWNoIHRvIG1hcmsgcG9zc2libGUgZHJpZnRzLgogICAgOnBhcmFtIGluZl9jYXBwaW5nOiAgICAgICAgICAgICAgVGhlIHZhbHVlIHRvIHNldCBmb3Igd2hlbiBpdCByZWFjaGVkIGluZmluaXR5LgoKICAgIDpyZXR1cm5zOiBBIHR1cGxlIG9mCiAgICAgICAgICAgICAgWzBdID0gQW4gTUxSdW4gYXJ0aWZhY3QgaG9sZGluZyB0aGUgSFRNTCBjb2RlIG9mIHRoZSBkcmlmdCB0YWJsZSBwbG90LgogICAgICAgICAgICAgIFsxXSA9IEFuIE1MUnVuIGFydGlmYWN0IGhvbGRpbmcgdGhlIG1ldHJpYyBwZXIgZmVhdHVyZSBkaWN0aW9uYXJ5LgogICAgICAgICAgICAgIFsyXSA9IFJlc3VsdHMgdG8gbG9nIHRoZSBmaW5hbCBhbmFseXNpcyBvdXRjb21lLgogICAgIiIiCiAgICAjIENhbGN1bGF0ZSB0aGUgaW5wdXQncyBzdGF0aXN0aWNzOgogICAgaW5wdXRzX3N0YXRpc3RpY3MgPSBjYWxjdWxhdGVfaW5wdXRzX3N0YXRpc3RpY3MoCiAgICAgICAgc2FtcGxlX3NldF9zdGF0aXN0aWNzPXNhbXBsZV9zZXRfc3RhdGlzdGljcywKICAgICAgICBpbnB1dHM9aW5wdXRzLAogICAgKQoKICAgICMgQ2FsY3VsYXRlIGRyaWZ0OgogICAgdmlydHVhbF9kcmlmdCA9IFZpcnR1YWxEcmlmdChpbmZfY2FwcGluZz1pbmZfY2FwcGluZykKICAgIG1ldHJpY3MgPSB2aXJ0dWFsX2RyaWZ0LmNvbXB1dGVfZHJpZnRfZnJvbV9oaXN0b2dyYW1zKAogICAgICAgIGZlYXR1cmVfc3RhdHM9c2FtcGxlX3NldF9zdGF0aXN0aWNzLAogICAgICAgIGN1cnJlbnRfc3RhdHM9aW5wdXRzX3N0YXRpc3RpY3MsCiAgICApCiAgICBkcmlmdF9yZXN1bHRzID0gdmlydHVhbF9kcmlmdC5jaGVja19mb3JfZHJpZnRfcGVyX2ZlYXR1cmUoCiAgICAgICAgbWV0cmljc19yZXN1bHRzX2RpY3Rpb25hcnk9bWV0cmljcywKICAgICAgICBwb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQ9cG9zc2libGVfZHJpZnRfdGhyZXNob2xkLAogICAgICAgIGRyaWZ0X2RldGVjdGVkX3RocmVzaG9sZD1kcmlmdF90aHJlc2hvbGQsCiAgICApCgogICAgIyBWYWxpZGF0ZSBhbGwgZmVhdHVyZSBjb2x1bW5zIG5hbWVkIHRoZSBzYW1lIGJldHdlZW4gdGhlIGlucHV0cyBhbmQgc2FtcGxlIHNldHM6CiAgICBzYW1wbGVfZmVhdHVyZXMgPSBzZXQoCiAgICAgICAgWwogICAgICAgICAgICBmZWF0dXJlX25hbWUKICAgICAgICAgICAgZm9yIGZlYXR1cmVfbmFtZSwgZmVhdHVyZV9zdGF0aXN0aWNzIGluIHNhbXBsZV9zZXRfc3RhdGlzdGljcy5pdGVtcygpCiAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoZmVhdHVyZV9zdGF0aXN0aWNzLCBkaWN0KQogICAgICAgIF0KICAgICkKICAgIGlucHV0X2ZlYXR1cmVzID0gc2V0KGlucHV0cy5jb2x1bW5zKQogICAgaWYgbGVuKHNhbXBsZV9mZWF0dXJlcyAmIGlucHV0X2ZlYXR1cmVzKSAhPSBsZW4oaW5wdXRfZmVhdHVyZXMpOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIk5vdCBhbGwgZmVhdHVyZSBuYW1lcyB3ZXJlIG1hdGNoaW5nIGJldHdlZW4gdGhlIGlucHV0cyBhbmQgdGhlIHNhbXBsZSBzZXQgcHJvdmlkZWQ6ICIKICAgICAgICAgICAgZiJ7aW5wdXRfZmVhdHVyZXMgLSBzYW1wbGVfZmVhdHVyZXMgfCBzYW1wbGVfZmVhdHVyZXMgLSBpbnB1dF9mZWF0dXJlc30iCiAgICAgICAgKQoKICAgICMgUGxvdDoKICAgIGh0bWxfcGxvdCA9IEZlYXR1cmVzRHJpZnRUYWJsZVBsb3QoKS5wcm9kdWNlKAogICAgICAgIGZlYXR1cmVzPWxpc3QoaW5wdXRfZmVhdHVyZXMpLAogICAgICAgIHNhbXBsZV9zZXRfc3RhdGlzdGljcz1zYW1wbGVfc2V0X3N0YXRpc3RpY3MsCiAgICAgICAgaW5wdXRzX3N0YXRpc3RpY3M9aW5wdXRzX3N0YXRpc3RpY3MsCiAgICAgICAgbWV0cmljcz1tZXRyaWNzLAogICAgICAgIGRyaWZ0X3Jlc3VsdHM9ZHJpZnRfcmVzdWx0cywKICAgICkKCiAgICAjIFByZXBhcmUgbWV0cmljcyBwZXIgZmVhdHVyZSBkaWN0aW9uYXJ5OgogICAgbWV0cmljc19wZXJfZmVhdHVyZSA9IHsKICAgICAgICBmZWF0dXJlOiBfZ2V0X2RyaWZ0X3Jlc3VsdCgKICAgICAgICAgICAgdHZkPW1ldHJpY19kaWN0aW9uYXJ5WyJ0dmQiXSwKICAgICAgICAgICAgaGVsbGluZ2VyPW1ldHJpY19kaWN0aW9uYXJ5WyJoZWxsaW5nZXIiXSwKICAgICAgICAgICAgdGhyZXNob2xkPWRyaWZ0X3RocmVzaG9sZCwKICAgICAgICApWzFdCiAgICAgICAgZm9yIGZlYXR1cmUsIG1ldHJpY19kaWN0aW9uYXJ5IGluIG1ldHJpY3MuaXRlbXMoKQogICAgICAgIGlmIGlzaW5zdGFuY2UobWV0cmljX2RpY3Rpb25hcnksIGRpY3QpCiAgICB9CgogICAgIyBDYWxjdWxhdGUgdGhlIGZpbmFsIGFuYWx5c2lzIHJlc3VsdDoKICAgIGRyaWZ0X3N0YXR1cywgZHJpZnRfbWV0cmljID0gX2dldF9kcmlmdF9yZXN1bHQoCiAgICAgICAgdHZkPW1ldHJpY3NbInR2ZF9tZWFuIl0sCiAgICAgICAgaGVsbGluZ2VyPW1ldHJpY3NbImhlbGxpbmdlcl9tZWFuIl0sCiAgICAgICAgdGhyZXNob2xkPWRyaWZ0X3RocmVzaG9sZCwKICAgICkKCiAgICByZXR1cm4gKAogICAgICAgIEFydGlmYWN0KGJvZHk9aHRtbF9wbG90LCBmb3JtYXQ9Imh0bWwiLCBrZXk9ImRyaWZ0X3RhYmxlX3Bsb3QiKSwKICAgICAgICBBcnRpZmFjdCgKICAgICAgICAgICAgYm9keT1qc29uLmR1bXBzKG1ldHJpY3NfcGVyX2ZlYXR1cmUpLAogICAgICAgICAgICBmb3JtYXQ9Impzb24iLAogICAgICAgICAgICBrZXk9ImZlYXR1cmVzX2RyaWZ0X3Jlc3VsdHMiLAogICAgICAgICksCiAgICAgICAgeyJkcmlmdF9zdGF0dXMiOiBkcmlmdF9zdGF0dXMsICJkcmlmdF9tZXRyaWMiOiBkcmlmdF9tZXRyaWN9LAogICAgKQoKCmRlZiBpbmZlcigKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgbW9kZWw6IHN0ciwKICAgIGRhdGFzZXQ6IERhdGFzZXRUeXBlLAogICAgZHJvcF9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXSwgaW50LCBMaXN0W2ludF1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IFVuaW9uW3N0ciwgTGlzdFtzdHJdXSA9IE5vbmUsCiAgICBmZWF0dXJlX2NvbHVtbnM6IFVuaW9uW3N0ciwgTGlzdFtzdHJdXSA9IE5vbmUsCiAgICBsb2dfcmVzdWx0X3NldDogYm9vbCA9IFRydWUsCiAgICByZXN1bHRfc2V0X25hbWU6IHN0ciA9ICJwcmVkaWN0aW9uIiwKICAgIGJhdGNoX2lkOiBzdHIgPSBOb25lLAogICAgcGVyZm9ybV9kcmlmdF9hbmFseXNpczogYm9vbCA9IE5vbmUsCiAgICBzYW1wbGVfc2V0OiBEYXRhc2V0VHlwZSA9IE5vbmUsCiAgICBkcmlmdF90aHJlc2hvbGQ6IGZsb2F0ID0gMC43LAogICAgcG9zc2libGVfZHJpZnRfdGhyZXNob2xkOiBmbG9hdCA9IDAuNSwKICAgIGluZl9jYXBwaW5nOiBmbG9hdCA9IDEwLjAsCiAgICBhcnRpZmFjdHNfdGFnOiBzdHIgPSAiIiwKICAgICoqcHJlZGljdF9rd2FyZ3M6IERpY3Rbc3RyLCBBbnldLAopOgogICAgIiIiCiAgICBQZXJmb3JtIGEgcHJlZGljdGlvbiBvbiBhIGdpdmVuIGRhdGFzZXQgd2l0aCB0aGUgZ2l2ZW4gbW9kZWwuIENhbiBwZXJmb3JtIGRyaWZ0IGFuYWx5c2lzIGJldHdlZW4gdGhlIHNhbXBsZSBzZXQKICAgIHN0YXRpc3RpY3Mgc3RvcmVkIGluIHRoZSBtb2RlbCB0byB0aGUgY3VycmVudCBpbnB1dCBkYXRhLiBUaGUgZHJpZnQgcnVsZSBpcyB0aGUgdmFsdWUgcGVyLWZlYXR1cmUgbWVhbiBvZiB0aGUgVFZECiAgICBhbmQgSGVsbGluZ2VyIHNjb3JlcyBhY2NvcmRpbmcgdG8gdGhlIHRocmVzaG9sZHMgY29uZmlndXJlcyBoZXJlLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gbW9kZWw6ICAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwgU3RvcmUgcGF0aC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIGluZmVyIHRocm91Z2ggdGhlIG1vZGVsLiBDYW4gYmUgcGFzc2VkIGluIGBpbnB1dHNgIGFzIGVpdGhlciBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEYXRhc2V0IGFydGlmYWN0IC8gRmVhdHVyZSB2ZWN0b3IgVVJJLiBPciwgaW4gYHBhcmFtZXRlcnNgIGFzIGEgbGlzdCwgZGljdGlvbmFyeSBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbnVtcHkgYXJyYXkuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICAgICAgICAgICBBIHN0cmluZyAvIGludGVnZXIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgLyBpbnRlZ2VycyB0aGF0IHJlcHJlc2VudCB0aGUgY29sdW1uIG5hbWVzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAvIGluZGljZXMgdG8gZHJvcC4gV2hlbiB0aGUgZGF0YXNldCBpcyBhIGxpc3Qgb3IgYSBudW1weSBhcnJheSB0aGlzIHBhcmFtZXRlciBtdXN0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSByZXByZXNlbnRlZCBieSBpbnRlZ2Vycy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldCBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2xhc3NpZmljYXRpb24gdGFza3MuIFRoZSBsYWJlbCBjb2x1bW4gY2FuIGJlIGFjY2Vzc2VkIGZyb20gdGhlIG1vZGVsIG9iamVjdCwgb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZSBmZWF0dXJlIHZlY3RvciBwcm92aWRlZCBpZiBhdmFpbGFibGUuCiAgICA6cGFyYW0gZmVhdHVyZV9jb2x1bW5zOiAgICAgICAgICBMaXN0IG9mIGZlYXR1cmUgY29sdW1ucyB0aGF0IHdpbGwgYmUgdXNlZCB0byBidWlsZCB0aGUgZGF0YWZyYW1lIHdoZW4gZGF0YXNldCBpcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZnJvbSB0eXBlIGxpc3Qgb3IgbnVtcHkgYXJyYXkuCiAgICA6cGFyYW0gbG9nX3Jlc3VsdF9zZXQ6ICAgICAgICAgICBXaGV0aGVyIHRvIGxvZyB0aGUgcmVzdWx0IHNldCAtIGEgRGF0YUZyYW1lIG9mIHRoZSBnaXZlbiBpbnB1dHMgY29uY2F0ZW5hdGVkIHdpdGgKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZSBwcmVkaWN0aW9ucy4gRGVmYXVsdGVkIHRvIFRydWUuCiAgICA6cGFyYW0gcmVzdWx0X3NldF9uYW1lOiAgICAgICAgICBUaGUgZGIga2V5IHRvIHNldCBuYW1lIG9mIHRoZSBwcmVkaWN0aW9uIHJlc3VsdCBhbmQgdGhlIGZpbGVuYW1lLiBEZWZhdWx0ZWQgdG8KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdwcmVkaWN0aW9uJy4KICAgIDpwYXJhbSBiYXRjaF9pZDogICAgICAgICAgICAgICAgIFRoZSBJRCBvZiB0aGUgZ2l2ZW4gYmF0Y2ggKGluZmVyZW5jZSBkYXRhc2V0KS4gSWYgYE5vbmVgLCBpdCB3aWxsIGJlIGdlbmVyYXRlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFdpbGwgYmUgbG9nZ2VkIGFzIGEgcmVzdWx0IG9mIHRoZSBydW4uCiAgICA6cGFyYW0gcGVyZm9ybV9kcmlmdF9hbmFseXNpczogICBXaGV0aGVyIHRvIHBlcmZvcm0gZHJpZnQgYW5hbHlzaXMgYmV0d2VlbiB0aGUgc2FtcGxlIHNldCBvZiB0aGUgbW9kZWwgb2JqZWN0IHRvIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGF0YXNldCBnaXZlbi4gQnkgZGVmYXVsdCwgTm9uZSwgd2hpY2ggbWVhbnMgaXQgd2lsbCBwZXJmb3JtIGRyaWZ0IGFuYWx5c2lzIGlmIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kZWwgaGFzIGEgc2FtcGxlIHNldCBzdGF0aXN0aWNzLiBQZXJmb3JtIGRyaWZ0IGFuYWx5c2lzIHdpbGwgcHJvZHVjZSBhIGRhdGEgZHJpZnQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRhYmxlIGFydGlmYWN0LgogICAgOnBhcmFtIHNhbXBsZV9zZXQ6ICAgICAgICAgICAgICAgQSBzYW1wbGUgZGF0YXNldCB0byBnaXZlIHRvIGNvbXBhcmUgdGhlIGlucHV0cyBpbiB0aGUgZHJpZnQgYW5hbHlzaXMuIFRoZSBkZWZhdWx0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjaG9zZW4gc2FtcGxlIHNldCB3aWxsIGFsd2F5cyBiZSB0aGUgb25lIHdobyBpcyBzZXQgaW4gdGhlIG1vZGVsIGFydGlmYWN0IGl0c2VsZi4KICAgIDpwYXJhbSBkcmlmdF90aHJlc2hvbGQ6ICAgICAgICAgIFRoZSB0aHJlc2hvbGQgb2Ygd2hpY2ggdG8gbWFyayBkcmlmdHMuIERlZmF1bHRlZCB0byAwLjcuCiAgICA6cGFyYW0gcG9zc2libGVfZHJpZnRfdGhyZXNob2xkOiBUaGUgdGhyZXNob2xkIG9mIHdoaWNoIHRvIG1hcmsgcG9zc2libGUgZHJpZnRzLiBEZWZhdWx0ZWQgdG8gMC41LgogICAgOnBhcmFtIGluZl9jYXBwaW5nOiAgICAgICAgICAgICAgVGhlIHZhbHVlIHRvIHNldCBmb3Igd2hlbiBpdCByZWFjaGVkIGluZmluaXR5LiBEZWZhdWx0ZWQgdG8gMTAuMC4KICAgIDpwYXJhbSBhcnRpZmFjdHNfdGFnOiAgICAgICAgICAgIFRhZyB0byB1c2UgZm9yIGFsbCB0aGUgYXJ0aWZhY3RzIHJlc3VsdGVkIGZyb20gdGhlIGZ1bmN0aW9uLgogICAgIiIiCiAgICAjIExvYWRpbmcgdGhlIG1vZGVsOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIkxvYWRpbmcgbW9kZWwuLi4iKQogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKG1vZGVsX3BhdGg9bW9kZWwsIGNvbnRleHQ9Y29udGV4dCkKICAgIGlmIGxhYmVsX2NvbHVtbnMgaXMgTm9uZToKICAgICAgICBsYWJlbF9jb2x1bW5zID0gWwogICAgICAgICAgICBvdXRwdXQubmFtZSBmb3Igb3V0cHV0IGluIG1vZGVsX2hhbmRsZXIuX21vZGVsX2FydGlmYWN0LnNwZWMub3V0cHV0cwogICAgICAgIF0KCiAgICBpZiBmZWF0dXJlX2NvbHVtbnMgaXMgTm9uZToKICAgICAgICBmZWF0dXJlX2NvbHVtbnMgPSBbCiAgICAgICAgICAgIGlucHV0Lm5hbWUgZm9yIGlucHV0IGluIG1vZGVsX2hhbmRsZXIuX21vZGVsX2FydGlmYWN0LnNwZWMuaW5wdXRzCiAgICAgICAgXQoKICAgICMgR2V0IGRhdGFzZXQgYnkgb2JqZWN0LCBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJMb2FkaW5nIGRhdGEuLi4iKQogICAgeCwgbGFiZWxfY29sdW1ucyA9IF9yZWFkX2RhdGFzZXRfYXNfZGF0YWZyYW1lKAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBmZWF0dXJlX2NvbHVtbnM9ZmVhdHVyZV9jb2x1bW5zLAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgUHJlZGljdDoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJDYWxjdWxhdGluZyBwcmVkaWN0aW9uLi4uIikKICAgIHlfcHJlZCA9IG1vZGVsX2hhbmRsZXIubW9kZWwucHJlZGljdCh4LCAqKnByZWRpY3Rfa3dhcmdzKQoKICAgICMgUHJlcGFyZSB0aGUgcmVzdWx0IHNldDoKICAgIHJlc3VsdF9zZXQgPSBfcHJlcGFyZV9yZXN1bHRfc2V0KHg9eCwgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLCB5X3ByZWQ9eV9wcmVkKQoKICAgICMgQ2hlY2sgZm9yIGxvZ2dpbmcgdGhlIHJlc3VsdCBzZXQ6CiAgICBpZiBsb2dfcmVzdWx0X3NldDoKICAgICAgICAjIExvZyB0aGUgcmVzdWx0IHNldDoKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiTG9nZ2luZyByZXN1bHQgc2V0ICh4IHwgcHJlZGljdGlvbikuLi4iKQogICAgICAgIGNvbnRleHQubG9nX2RhdGFzZXQoCiAgICAgICAgICAgIGtleT1yZXN1bHRfc2V0X25hbWUsCiAgICAgICAgICAgIGRmPXJlc3VsdF9zZXQsCiAgICAgICAgICAgIGRiX2tleT1yZXN1bHRfc2V0X25hbWUsCiAgICAgICAgICAgIHRhZz1hcnRpZmFjdHNfdGFnLAogICAgICAgICkKICAgICAgICAjIExvZyB0aGUgYmF0Y2ggSUQ6CiAgICAgICAgaWYgYmF0Y2hfaWQgaXMgTm9uZToKICAgICAgICAgICAgYmF0Y2hfaWQgPSBoYXNobGliLnNoYTIyNChzdHIoZGF0ZXRpbWUubm93KCkpLmVuY29kZSgpKS5oZXhkaWdlc3QoKQogICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdCgKICAgICAgICAgICAga2V5PSJiYXRjaF9pZCIsCiAgICAgICAgICAgIHZhbHVlPWJhdGNoX2lkLAogICAgICAgICkKCiAgICAjIENoZWNrIGZvciBwZXJmb3JtaW5nIGRyaWZ0IGFuYWx5c2lzOgogICAgaWYgKAogICAgICAgIHBlcmZvcm1fZHJpZnRfYW5hbHlzaXMgaXMgTm9uZQogICAgICAgIGFuZCBtb2RlbF9oYW5kbGVyLl9tb2RlbF9hcnRpZmFjdC5zcGVjLmZlYXR1cmVfc3RhdHMgaXMgbm90IE5vbmUKICAgICk6CiAgICAgICAgcGVyZm9ybV9kcmlmdF9hbmFseXNpcyA9IFRydWUKICAgIGlmIHBlcmZvcm1fZHJpZnRfYW5hbHlzaXM6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygiUGVyZm9ybWluZyBkcmlmdCBhbmFseXNpcy4uLiIpCiAgICAgICAgIyBHZXQgdGhlIHNhbXBsZSBzZXQgc3RhdGlzdGljcyAoZWl0aGVyIGZyb20gdGhlIHNhbXBsZSBzZXQgb3IgZnJvbSB0aGUgc3RhdGlzdGljcyBsb2dnZWQgd2l0aCB0aGUgbW9kZWwpOgogICAgICAgIHNhbXBsZV9zZXRfc3RhdGlzdGljcyA9IF9nZXRfc2FtcGxlX3NldF9zdGF0aXN0aWNzKAogICAgICAgICAgICBzYW1wbGVfc2V0PXNhbXBsZV9zZXQsCiAgICAgICAgICAgIG1vZGVsX2FydGlmYWN0X2ZlYXR1cmVfc3RhdHM9bW9kZWxfaGFuZGxlci5fbW9kZWxfYXJ0aWZhY3Quc3BlYy5mZWF0dXJlX3N0YXRzLAogICAgICAgICkKICAgICAgICAjIFByb2R1Y2UgdGhlIGFydGlmYWN0OgogICAgICAgICgKICAgICAgICAgICAgZHJpZnRfdGFibGVfcGxvdCwKICAgICAgICAgICAgbWV0cmljX3Blcl9mZWF0dXJlX2RpY3QsCiAgICAgICAgICAgIGFuYWx5c2lzX3Jlc3VsdHMsCiAgICAgICAgKSA9IF9wZXJmb3JtX2RyaWZ0X2FuYWx5c2lzKAogICAgICAgICAgICBzYW1wbGVfc2V0X3N0YXRpc3RpY3M9c2FtcGxlX3NldF9zdGF0aXN0aWNzLAogICAgICAgICAgICBpbnB1dHM9cmVzdWx0X3NldCwKICAgICAgICAgICAgZHJpZnRfdGhyZXNob2xkPWRyaWZ0X3RocmVzaG9sZCwKICAgICAgICAgICAgcG9zc2libGVfZHJpZnRfdGhyZXNob2xkPXBvc3NpYmxlX2RyaWZ0X3RocmVzaG9sZCwKICAgICAgICAgICAgaW5mX2NhcHBpbmc9aW5mX2NhcHBpbmcsCiAgICAgICAgKQogICAgICAgICMgTG9nIHRoZSBhcnRpZmFjdCBhbmQgcmVzdWx0czoKICAgICAgICBjb250ZXh0LmxvZ19hcnRpZmFjdChkcmlmdF90YWJsZV9wbG90LCB0YWc9YXJ0aWZhY3RzX3RhZykKICAgICAgICBjb250ZXh0LmxvZ19hcnRpZmFjdChtZXRyaWNfcGVyX2ZlYXR1cmVfZGljdCwgdGFnPWFydGlmYWN0c190YWcpCiAgICAgICAgY29udGV4dC5sb2dfcmVzdWx0cyhyZXN1bHRzPWFuYWx5c2lzX3Jlc3VsdHMpCg== - origin_filename: '' - auto_build: false - code_origin: '' - with_mlrun: false - disable_auto_mount: false description: Batch inference (also knows as prediction) for the common ML frameworks (SciKit-Learn, XGBoost and LightGBM) while performing data drift analysis. + default_handler: infer diff --git a/functions/src/batch_inference/test_batch_inference.py b/functions/src/batch_inference/test_batch_inference.py index d18d27a9b..e37a7d000 100644 --- a/functions/src/batch_inference/test_batch_inference.py +++ b/functions/src/batch_inference/test_batch_inference.py @@ -86,7 +86,6 @@ def train(training_set: pd.DataFrame): reason="Project's environment variables are not set", ) def test_batch_predict(): - project = mlrun.get_or_create_project( "batch-infer-v9-test", context="./", user_project=True ) @@ -132,7 +131,7 @@ def test_batch_predict(): # Check the features drift results json: drift_results_file = batch_predict_run.artifact("features_drift_results").local() - with open(drift_results_file, "r") as json_file: + with open(drift_results_file) as json_file: drift_results = json.load(json_file) assert len(drift_results) == n_features + 1 diff --git a/functions/src/batch_inference_v2/batch_inference_v2.py b/functions/src/batch_inference_v2/batch_inference_v2.py index c12b04972..3c4ade07b 100644 --- a/functions/src/batch_inference_v2/batch_inference_v2.py +++ b/functions/src/batch_inference_v2/batch_inference_v2.py @@ -13,15 +13,16 @@ # limitations under the License. from inspect import signature -from typing import Any, Dict, List, Union, Optional +from typing import Any + import mlrun try: import mlrun.model_monitoring.api except ModuleNotFoundError: raise mlrun.errors.MLRunNotFoundError( - f"Please update your `mlrun` version to >=1.5.0 or use an " - f"older version of the batch inference function." + "Please update your `mlrun` version to >=1.5.0 or use an " + "older version of the batch inference function." ) import numpy as np @@ -29,7 +30,9 @@ from mlrun.frameworks.auto_mlrun import AutoMLRun -def _prepare_result_set(x: pd.DataFrame, label_columns: List[str], y_pred: np.ndarray) -> pd.DataFrame: +def _prepare_result_set( + x: pd.DataFrame, label_columns: list[str], y_pred: np.ndarray +) -> pd.DataFrame: """ Set default label column names and validate given names to prepare the result set - a concatenation of the inputs (x) and the model predictions (y_pred). @@ -74,63 +77,75 @@ def _prepare_result_set(x: pd.DataFrame, label_columns: List[str], y_pred: np.nd ) -def _get_sample_set_statistics_parameters(context: mlrun.MLClientCtx, - model_endpoint_sample_set: Union[ - mlrun.DataItem, list, dict, pd.DataFrame, pd.Series, np.ndarray], - model_artifact_feature_stats: dict, - feature_columns: Optional[List], - drop_columns: Optional[List], - label_columns: Optional[List]) -> Dict[str, Any]: - statics_input_full_dict = dict(sample_set=model_endpoint_sample_set, - model_artifact_feature_stats=model_artifact_feature_stats, - sample_set_columns=feature_columns, - sample_set_drop_columns=drop_columns, - sample_set_label_columns=label_columns) +def _get_sample_set_statistics_parameters( + context: mlrun.MLClientCtx, + model_endpoint_sample_set: mlrun.DataItem + | list + | dict + | pd.DataFrame + | pd.Series + | np.ndarray, + model_artifact_feature_stats: dict, + feature_columns: list | None, + drop_columns: list | None, + label_columns: list | None, +) -> dict[str, Any]: + statics_input_full_dict = dict( + sample_set=model_endpoint_sample_set, + model_artifact_feature_stats=model_artifact_feature_stats, + sample_set_columns=feature_columns, + sample_set_drop_columns=drop_columns, + sample_set_label_columns=label_columns, + ) get_sample_statics_function = mlrun.model_monitoring.api.get_sample_set_statistics statics_function_input_dict = signature(get_sample_statics_function).parameters # As a result of changes to input parameters in the mlrun-get_sample_set_statistics function, # we will now send only the parameters it expects. - statistics_input_filtered = {key: statics_input_full_dict[key] for key in statics_function_input_dict} + statistics_input_filtered = { + key: statics_input_full_dict[key] for key in statics_function_input_dict + } if len(statistics_input_filtered) != len(statics_function_input_dict): - context.logger.warning(f"get_sample_set_statistics is in an older version; " - "some parameters will not be sent to the function." - f" Expected input: {list(statics_function_input_dict.keys())}," - f" actual input: {list(statistics_input_filtered.keys())}") + context.logger.warning( + f"get_sample_set_statistics is in an older version; " + "some parameters will not be sent to the function." + f" Expected input: {list(statics_function_input_dict.keys())}," + f" actual input: {list(statistics_input_filtered.keys())}" + ) return statistics_input_filtered def infer( - context: mlrun.MLClientCtx, - dataset: Union[mlrun.DataItem, list, dict, pd.DataFrame, pd.Series, np.ndarray], - model_path: Union[str, mlrun.DataItem], - drop_columns: Union[str, List[str], int, List[int]] = None, - label_columns: Union[str, List[str]] = None, - feature_columns: Union[str, List[str]] = None, - log_result_set: bool = True, - result_set_name: str = "prediction", - batch_id: str = None, - artifacts_tag: str = "", - # Drift analysis parameters - perform_drift_analysis: bool = None, - endpoint_id: str = "", - # The following model endpoint parameters are relevant only if: - # perform drift analysis is not disabled - # a new model endpoint record is going to be generated - model_endpoint_name: str = "batch-infer", - model_endpoint_sample_set: Union[ - mlrun.DataItem, list, dict, pd.DataFrame, pd.Series, np.ndarray - ] = None, - - # the following parameters are deprecated and will be removed once the versioning mechanism is implemented - # TODO: Remove the following parameters once FHUB-13 is resolved - trigger_monitoring_job: Optional[bool] = None, - batch_image_job: Optional[str] = None, - model_endpoint_drift_threshold: Optional[float] = None, - model_endpoint_possible_drift_threshold: Optional[float] = None, - - # prediction kwargs to pass to the model predict function - **predict_kwargs: Dict[str, Any], - + context: mlrun.MLClientCtx, + dataset: mlrun.DataItem | list | dict | pd.DataFrame | pd.Series | np.ndarray, + model_path: str | mlrun.DataItem, + drop_columns: str | list[str] | int | list[int] = None, + label_columns: str | list[str] = None, + feature_columns: str | list[str] = None, + log_result_set: bool = True, + result_set_name: str = "prediction", + batch_id: str = None, + artifacts_tag: str = "", + # Drift analysis parameters + perform_drift_analysis: bool = None, + endpoint_id: str = "", + # The following model endpoint parameters are relevant only if: + # perform drift analysis is not disabled + # a new model endpoint record is going to be generated + model_endpoint_name: str = "batch-infer", + model_endpoint_sample_set: mlrun.DataItem + | list + | dict + | pd.DataFrame + | pd.Series + | np.ndarray = None, + # the following parameters are deprecated and will be removed once the versioning mechanism is implemented + # TODO: Remove the following parameters once FHUB-13 is resolved + trigger_monitoring_job: bool | None = None, + batch_image_job: str | None = None, + model_endpoint_drift_threshold: float | None = None, + model_endpoint_possible_drift_threshold: float | None = None, + # prediction kwargs to pass to the model predict function + **predict_kwargs: dict[str, Any], ): """ Perform a prediction on the provided dataset using the specified model. @@ -192,26 +207,33 @@ def infer( raises MLRunInvalidArgumentError: if both `model_path` and `endpoint_id` are not provided """ - if trigger_monitoring_job: - context.logger.warning("The `trigger_monitoring_job` parameter is deprecated and will be removed once the versioning mechanism is implemented. " - "if you are using mlrun<1.7.0, please import the previous version of this function, for example " - "'hub://batch_inference_v2:2.5.0'.") + context.logger.warning( + "The `trigger_monitoring_job` parameter is deprecated and will be removed once the versioning mechanism is implemented. " + "if you are using mlrun<1.7.0, please import the previous version of this function, for example " + "'hub://batch_inference_v2:2.5.0'." + ) if batch_image_job: - context.logger.warning("The `batch_image_job` parameter is deprecated and will be removed once the versioning mechanism is implemented. " - "if you are using mlrun<1.7.0, please import the previous version of this function, for example " - "'hub://batch_inference_v2:2.5.0'.") + context.logger.warning( + "The `batch_image_job` parameter is deprecated and will be removed once the versioning mechanism is implemented. " + "if you are using mlrun<1.7.0, please import the previous version of this function, for example " + "'hub://batch_inference_v2:2.5.0'." + ) if model_endpoint_drift_threshold: - context.logger.warning("The `model_endpoint_drift_threshold` parameter is deprecated and will be removed once the versioning mechanism is implemented. " - "if you are using mlrun<1.7.0, please import the previous version of this function, for example " - "'hub://batch_inference_v2:2.5.0'.") + context.logger.warning( + "The `model_endpoint_drift_threshold` parameter is deprecated and will be removed once the versioning mechanism is implemented. " + "if you are using mlrun<1.7.0, please import the previous version of this function, for example " + "'hub://batch_inference_v2:2.5.0'." + ) if model_endpoint_possible_drift_threshold: - context.logger.warning("The `model_endpoint_possible_drift_threshold` parameter is deprecated and will be removed once the versioning mechanism is implemented. " - "if you are using mlrun<1.7.0, please import the previous version of this function, for example " - "'hub://batch_inference_v2:2.5.0'.") + context.logger.warning( + "The `model_endpoint_possible_drift_threshold` parameter is deprecated and will be removed once the versioning mechanism is implemented. " + "if you are using mlrun<1.7.0, please import the previous version of this function, for example " + "'hub://batch_inference_v2:2.5.0'." + ) # Loading the model: - context.logger.info(f"Loading model...") + context.logger.info("Loading model...") if isinstance(model_path, mlrun.DataItem): model_path = model_path.artifact_url if not mlrun.datastore.is_store_uri(model_path): @@ -233,7 +255,7 @@ def infer( ] # Get dataset by object, URL or by FeatureVector: - context.logger.info(f"Loading data...") + context.logger.info("Loading data...") x, label_columns = mlrun.model_monitoring.api.read_dataset_as_dataframe( dataset=dataset, feature_columns=feature_columns, @@ -242,7 +264,7 @@ def infer( ) # Predict: - context.logger.info(f"Calculating prediction...") + context.logger.info("Calculating prediction...") y_pred = model_handler.model.predict(x, **predict_kwargs) # Prepare the result set: @@ -260,8 +282,8 @@ def infer( # Check for performing drift analysis if ( - perform_drift_analysis is None - and model_handler._model_artifact.spec.feature_stats is not None + perform_drift_analysis is None + and model_handler._model_artifact.spec.feature_stats is not None ): perform_drift_analysis = True if perform_drift_analysis: @@ -273,8 +295,11 @@ def infer( model_artifact_feature_stats=model_handler._model_artifact.spec.feature_stats, feature_columns=feature_columns, drop_columns=drop_columns, - label_columns=label_columns) - sample_set_statistics = mlrun.model_monitoring.api.get_sample_set_statistics(**statistics_input_filtered) + label_columns=label_columns, + ) + sample_set_statistics = mlrun.model_monitoring.api.get_sample_set_statistics( + **statistics_input_filtered + ) mlrun.model_monitoring.api.record_results( project=context.project, context=context, @@ -283,4 +308,4 @@ def infer( model_endpoint_name=model_endpoint_name, infer_results_df=result_set.copy(), sample_set_statistics=sample_set_statistics, - ) \ No newline at end of file + ) diff --git a/functions/src/batch_inference_v2/function.yaml b/functions/src/batch_inference_v2/function.yaml index 014cb2167..8c327e9d6 100644 --- a/functions/src/batch_inference_v2/function.yaml +++ b/functions/src/batch_inference_v2/function.yaml @@ -1,21 +1,32 @@ +metadata: + tag: '' + name: batch-inference-v2 + categories: + - model-serving verbose: false +kind: job spec: - default_handler: infer + image: mlrun/mlrun + disable_auto_mount: false + build: + origin_filename: '' + with_mlrun: false + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpmcm9tIGluc3BlY3QgaW1wb3J0IHNpZ25hdHVyZQpmcm9tIHR5cGluZyBpbXBvcnQgQW55CgppbXBvcnQgbWxydW4KCnRyeToKICAgIGltcG9ydCBtbHJ1bi5tb2RlbF9tb25pdG9yaW5nLmFwaQpleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvcjoKICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bk5vdEZvdW5kRXJyb3IoCiAgICAgICAgIlBsZWFzZSB1cGRhdGUgeW91ciBgbWxydW5gIHZlcnNpb24gdG8gPj0xLjUuMCBvciB1c2UgYW4gIgogICAgICAgICJvbGRlciB2ZXJzaW9uIG9mIHRoZSBiYXRjaCBpbmZlcmVuY2UgZnVuY3Rpb24uIgogICAgKQoKaW1wb3J0IG51bXB5IGFzIG5wCmltcG9ydCBwYW5kYXMgYXMgcGQKZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLmF1dG9fbWxydW4gaW1wb3J0IEF1dG9NTFJ1bgoKCmRlZiBfcHJlcGFyZV9yZXN1bHRfc2V0KAogICAgeDogcGQuRGF0YUZyYW1lLCBsYWJlbF9jb2x1bW5zOiBsaXN0W3N0cl0sIHlfcHJlZDogbnAubmRhcnJheQopIC0+IHBkLkRhdGFGcmFtZToKICAgICIiIgogICAgU2V0IGRlZmF1bHQgbGFiZWwgY29sdW1uIG5hbWVzIGFuZCB2YWxpZGF0ZSBnaXZlbiBuYW1lcyB0byBwcmVwYXJlIHRoZSByZXN1bHQgc2V0IC0gYSBjb25jYXRlbmF0aW9uIG9mIHRoZSBpbnB1dHMKICAgICh4KSBhbmQgdGhlIG1vZGVsIHByZWRpY3Rpb25zICh5X3ByZWQpLgoKICAgIDpwYXJhbSB4OiAgICAgICAgICAgICBUaGUgaW5wdXRzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6IEEgbGlzdCBvZiBzdHJpbmdzIHJlcHJlc2VudGluZyB0aGUgdGFyZ2V0IGNvbHVtbiBuYW1lcyB0byBhZGQgdG8gdGhlIHByZWRpY3Rpb25zLiBEZWZhdWx0IG5hbWUKICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIHVzZWQgaW4gY2FzZSB0aGUgbGlzdCBpcyBlbXB0eSAocHJlZGljdGVkX2xhYmVsX3tpfSkuCiAgICA6cGFyYW0geV9wcmVkOiAgICAgICAgVGhlIG1vZGVsIHByZWRpY3Rpb25zIG9uIHRoZSBpbnB1dHMuCgogICAgOnJldHVybnM6IFRoZSByZXN1bHQgc2V0LgoKICAgIHJhaXNlcyBNTFJ1bkludmFsaWRBcmd1bWVudEVycm9yOiBJZiB0aGUgbGFiZWxzIGNvbHVtbnMgYW1vdW50IGRvIG5vdCBtYXRjaCB0aGUgb3V0cHV0cyBvciBpZiBvbmUgb2YgdGhlIGxhYmVsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGNvbHVtbiBhbHJlYWR5IGV4aXN0cyBpbiB0aGUgZGF0YXNldC4KICAgICIiIgogICAgIyBQcmVwYXJlIGRlZmF1bHQgdGFyZ2V0IGNvbHVtbnMgbmFtZXMgaWYgbm90IHByb3ZpZGVkOgogICAgcHJlZGljdGlvbl9jb2x1bW5zX2Ftb3VudCA9IDEgaWYgbGVuKHlfcHJlZC5zaGFwZSkgPT0gMSBlbHNlIHlfcHJlZC5zaGFwZVsxXQogICAgaWYgbGVuKGxhYmVsX2NvbHVtbnMpID09IDA6CiAgICAgICAgIyBBZGQgZGVmYXVsdCBsYWJlbCBjb2x1bW4gbmFtZXM6CiAgICAgICAgaWYgcHJlZGljdGlvbl9jb2x1bW5zX2Ftb3VudCA9PSAxOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWyJwcmVkaWN0ZWRfbGFiZWwiXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbCiAgICAgICAgICAgICAgICBmInByZWRpY3RlZF9sYWJlbF97aX0iIGZvciBpIGluIHJhbmdlKHByZWRpY3Rpb25fY29sdW1uc19hbW91bnQpCiAgICAgICAgICAgIF0KCiAgICAjIFZhbGlkYXRlIHRoZSBsYWJlbCBjb2x1bW5zOgogICAgaWYgcHJlZGljdGlvbl9jb2x1bW5zX2Ftb3VudCAhPSBsZW4obGFiZWxfY29sdW1ucyk6CiAgICAgICAgIyBObyBlcXVhbGl0eSBiZXR3ZWVuIHByb3ZpZGVkIGxhYmVsIGNvbHVtbiBuYW1lcyBhbmQgb3V0cHV0cyBhbW91bnQ6CiAgICAgICAgcmFpc2UgbWxydW4uZXJyb3JzLk1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3IoCiAgICAgICAgICAgIGYiVGhlIG51bWJlciBvZiBwcmVkaWN0ZWQgbGFiZWxzOiB7cHJlZGljdGlvbl9jb2x1bW5zX2Ftb3VudH0gIgogICAgICAgICAgICBmImlzIG5vdCBlcXVhbCB0byB0aGUgZ2l2ZW4gbGFiZWwgY29sdW1uczoge2xlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgKQogICAgY29tbW9uX2xhYmVscyA9IHNldChsYWJlbF9jb2x1bW5zKSAmIHNldCh4LmNvbHVtbnMudG9saXN0KCkpCiAgICBpZiBjb21tb25fbGFiZWxzOgogICAgICAgICMgTGFiZWwgY29sdW1uIGV4aXN0IGluIHRoZSBvcmlnaW5hbCBpbnB1dHM6CiAgICAgICAgcmFpc2UgbWxydW4uZXJyb3JzLk1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3IoCiAgICAgICAgICAgIGYiVGhlIGxhYmVsczoge2NvbW1vbl9sYWJlbHN9IGFyZSBhbHJlYWR5IGV4aXN0ZWQgaW4gdGhlIGdpdmVuIGRhdGFzZXQuIgogICAgICAgICkKCiAgICByZXR1cm4gcGQuY29uY2F0KAogICAgICAgIFt4LCBwZC5EYXRhRnJhbWUoeV9wcmVkLCBjb2x1bW5zPWxhYmVsX2NvbHVtbnMsIGluZGV4PXguaW5kZXgpXSwgYXhpcz0xCiAgICApCgoKZGVmIF9nZXRfc2FtcGxlX3NldF9zdGF0aXN0aWNzX3BhcmFtZXRlcnMoCiAgICBjb250ZXh0OiBtbHJ1bi5NTENsaWVudEN0eCwKICAgIG1vZGVsX2VuZHBvaW50X3NhbXBsZV9zZXQ6IG1scnVuLkRhdGFJdGVtCiAgICB8IGxpc3QKICAgIHwgZGljdAogICAgfCBwZC5EYXRhRnJhbWUKICAgIHwgcGQuU2VyaWVzCiAgICB8IG5wLm5kYXJyYXksCiAgICBtb2RlbF9hcnRpZmFjdF9mZWF0dXJlX3N0YXRzOiBkaWN0LAogICAgZmVhdHVyZV9jb2x1bW5zOiBsaXN0IHwgTm9uZSwKICAgIGRyb3BfY29sdW1uczogbGlzdCB8IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBsaXN0IHwgTm9uZSwKKSAtPiBkaWN0W3N0ciwgQW55XToKICAgIHN0YXRpY3NfaW5wdXRfZnVsbF9kaWN0ID0gZGljdCgKICAgICAgICBzYW1wbGVfc2V0PW1vZGVsX2VuZHBvaW50X3NhbXBsZV9zZXQsCiAgICAgICAgbW9kZWxfYXJ0aWZhY3RfZmVhdHVyZV9zdGF0cz1tb2RlbF9hcnRpZmFjdF9mZWF0dXJlX3N0YXRzLAogICAgICAgIHNhbXBsZV9zZXRfY29sdW1ucz1mZWF0dXJlX2NvbHVtbnMsCiAgICAgICAgc2FtcGxlX3NldF9kcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgICAgIHNhbXBsZV9zZXRfbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgKQogICAgZ2V0X3NhbXBsZV9zdGF0aWNzX2Z1bmN0aW9uID0gbWxydW4ubW9kZWxfbW9uaXRvcmluZy5hcGkuZ2V0X3NhbXBsZV9zZXRfc3RhdGlzdGljcwogICAgc3RhdGljc19mdW5jdGlvbl9pbnB1dF9kaWN0ID0gc2lnbmF0dXJlKGdldF9zYW1wbGVfc3RhdGljc19mdW5jdGlvbikucGFyYW1ldGVycwogICAgIyAgQXMgYSByZXN1bHQgb2YgY2hhbmdlcyB0byBpbnB1dCBwYXJhbWV0ZXJzIGluIHRoZSBtbHJ1bi1nZXRfc2FtcGxlX3NldF9zdGF0aXN0aWNzIGZ1bmN0aW9uLAogICAgIyAgd2Ugd2lsbCBub3cgc2VuZCBvbmx5IHRoZSBwYXJhbWV0ZXJzIGl0IGV4cGVjdHMuCiAgICBzdGF0aXN0aWNzX2lucHV0X2ZpbHRlcmVkID0gewogICAgICAgIGtleTogc3RhdGljc19pbnB1dF9mdWxsX2RpY3Rba2V5XSBmb3Iga2V5IGluIHN0YXRpY3NfZnVuY3Rpb25faW5wdXRfZGljdAogICAgfQogICAgaWYgbGVuKHN0YXRpc3RpY3NfaW5wdXRfZmlsdGVyZWQpICE9IGxlbihzdGF0aWNzX2Z1bmN0aW9uX2lucHV0X2RpY3QpOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm5pbmcoCiAgICAgICAgICAgIGYiZ2V0X3NhbXBsZV9zZXRfc3RhdGlzdGljcyBpcyBpbiBhbiBvbGRlciB2ZXJzaW9uOyAiCiAgICAgICAgICAgICJzb21lIHBhcmFtZXRlcnMgd2lsbCBub3QgYmUgc2VudCB0byB0aGUgZnVuY3Rpb24uIgogICAgICAgICAgICBmIiBFeHBlY3RlZCBpbnB1dDoge2xpc3Qoc3RhdGljc19mdW5jdGlvbl9pbnB1dF9kaWN0LmtleXMoKSl9LCIKICAgICAgICAgICAgZiIgYWN0dWFsIGlucHV0OiB7bGlzdChzdGF0aXN0aWNzX2lucHV0X2ZpbHRlcmVkLmtleXMoKSl9IgogICAgICAgICkKICAgIHJldHVybiBzdGF0aXN0aWNzX2lucHV0X2ZpbHRlcmVkCgoKZGVmIGluZmVyKAogICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSB8IGxpc3QgfCBkaWN0IHwgcGQuRGF0YUZyYW1lIHwgcGQuU2VyaWVzIHwgbnAubmRhcnJheSwKICAgIG1vZGVsX3BhdGg6IHN0ciB8IG1scnVuLkRhdGFJdGVtLAogICAgZHJvcF9jb2x1bW5zOiBzdHIgfCBsaXN0W3N0cl0gfCBpbnQgfCBsaXN0W2ludF0gPSBOb25lLAogICAgbGFiZWxfY29sdW1uczogc3RyIHwgbGlzdFtzdHJdID0gTm9uZSwKICAgIGZlYXR1cmVfY29sdW1uczogc3RyIHwgbGlzdFtzdHJdID0gTm9uZSwKICAgIGxvZ19yZXN1bHRfc2V0OiBib29sID0gVHJ1ZSwKICAgIHJlc3VsdF9zZXRfbmFtZTogc3RyID0gInByZWRpY3Rpb24iLAogICAgYmF0Y2hfaWQ6IHN0ciA9IE5vbmUsCiAgICBhcnRpZmFjdHNfdGFnOiBzdHIgPSAiIiwKICAgICMgRHJpZnQgYW5hbHlzaXMgcGFyYW1ldGVycwogICAgcGVyZm9ybV9kcmlmdF9hbmFseXNpczogYm9vbCA9IE5vbmUsCiAgICBlbmRwb2ludF9pZDogc3RyID0gIiIsCiAgICAjIFRoZSBmb2xsb3dpbmcgbW9kZWwgZW5kcG9pbnQgcGFyYW1ldGVycyBhcmUgcmVsZXZhbnQgb25seSBpZjoKICAgICMgcGVyZm9ybSBkcmlmdCBhbmFseXNpcyBpcyBub3QgZGlzYWJsZWQKICAgICMgYSBuZXcgbW9kZWwgZW5kcG9pbnQgcmVjb3JkIGlzIGdvaW5nIHRvIGJlIGdlbmVyYXRlZAogICAgbW9kZWxfZW5kcG9pbnRfbmFtZTogc3RyID0gImJhdGNoLWluZmVyIiwKICAgIG1vZGVsX2VuZHBvaW50X3NhbXBsZV9zZXQ6IG1scnVuLkRhdGFJdGVtCiAgICB8IGxpc3QKICAgIHwgZGljdAogICAgfCBwZC5EYXRhRnJhbWUKICAgIHwgcGQuU2VyaWVzCiAgICB8IG5wLm5kYXJyYXkgPSBOb25lLAogICAgIyB0aGUgZm9sbG93aW5nIHBhcmFtZXRlcnMgYXJlIGRlcHJlY2F0ZWQgYW5kIHdpbGwgYmUgcmVtb3ZlZCBvbmNlIHRoZSB2ZXJzaW9uaW5nIG1lY2hhbmlzbSBpcyBpbXBsZW1lbnRlZAogICAgIyBUT0RPOiBSZW1vdmUgdGhlIGZvbGxvd2luZyBwYXJhbWV0ZXJzIG9uY2UgRkhVQi0xMyBpcyByZXNvbHZlZAogICAgdHJpZ2dlcl9tb25pdG9yaW5nX2pvYjogYm9vbCB8IE5vbmUgPSBOb25lLAogICAgYmF0Y2hfaW1hZ2Vfam9iOiBzdHIgfCBOb25lID0gTm9uZSwKICAgIG1vZGVsX2VuZHBvaW50X2RyaWZ0X3RocmVzaG9sZDogZmxvYXQgfCBOb25lID0gTm9uZSwKICAgIG1vZGVsX2VuZHBvaW50X3Bvc3NpYmxlX2RyaWZ0X3RocmVzaG9sZDogZmxvYXQgfCBOb25lID0gTm9uZSwKICAgICMgcHJlZGljdGlvbiBrd2FyZ3MgdG8gcGFzcyB0byB0aGUgbW9kZWwgcHJlZGljdCBmdW5jdGlvbgogICAgKipwcmVkaWN0X2t3YXJnczogZGljdFtzdHIsIEFueV0sCik6CiAgICAiIiIKICAgIFBlcmZvcm0gYSBwcmVkaWN0aW9uIG9uIHRoZSBwcm92aWRlZCBkYXRhc2V0IHVzaW5nIHRoZSBzcGVjaWZpZWQgbW9kZWwuCiAgICBFbnN1cmUgdGhhdCB0aGUgbW9kZWwgaGFzIGFscmVhZHkgYmVlbiBsb2dnZWQgdW5kZXIgdGhlIGN1cnJlbnQgcHJvamVjdC4KCiAgICBJZiB5b3Ugd2lzaCB0byBhcHBseSBtb25pdG9yaW5nIHRvb2xzIChlLmcuLCBkcmlmdCBhbmFseXNpcyksIHNldCB0aGUgcGVyZm9ybV9kcmlmdF9hbmFseXNpcyBwYXJhbWV0ZXIgdG8gVHJ1ZS4KICAgIFRoaXMgd2lsbCBjcmVhdGUgYSBuZXcgbW9kZWwgZW5kcG9pbnQgcmVjb3JkIHVuZGVyIHRoZSBzcGVjaWZpZWQgbW9kZWxfZW5kcG9pbnRfbmFtZS4KICAgIEFkZGl0aW9uYWxseSwgZW5zdXJlIHRoYXQgbW9kZWwgbW9uaXRvcmluZyBpcyBlbmFibGVkIGF0IHRoZSBwcm9qZWN0IGxldmVsIGJ5IGNhbGxpbmcgdGhlCiAgICBwcm9qZWN0LmVuYWJsZV9tb2RlbF9tb25pdG9yaW5nKCkgZnVuY3Rpb24uIFlvdSBjYW4gYWxzbyBhcHBseSBtb25pdG9yaW5nIHRvIGFuIGV4aXN0aW5nIG1vZGVsIGJ5IHByb3ZpZGluZyBpdHMKICAgIGVuZHBvaW50IGlkIG9yIG5hbWUsIGFuZCB0aGUgbW9uaXRvcmluZyB0b29scyB3aWxsIGJlIGFwcGxpZWQgdG8gdGhhdCBlbmRwb2ludC4KCiAgICBBdCB0aGUgbW9tZW50LCB0aGlzIGZ1bmN0aW9uIGlzIHN1cHBvcnRlZCBmb3IgYG1scnVuPj0xLjUuMGAgdmVyc2lvbnMuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIGluZmVyIHRocm91Z2ggdGhlIG1vZGVsLiBQcm92aWRlZCBhcyBhbiBpbnB1dCAoRGF0YUl0ZW0pCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGF0IHJlcHJlc2VudHMgRGF0YXNldCBhcnRpZmFjdCAvIEZlYXR1cmUgdmVjdG9yIFVSSS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIElmIHVzaW5nIE1MUnVuIFNESywgYGRhdGFzZXRgIGNhbiBhbHNvIGJlIHByb3ZpZGVkIGFzIGEgbGlzdCwgZGljdGlvbmFyeSBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbnVtcHkgYXJyYXkuCiAgICA6cGFyYW0gbW9kZWxfcGF0aDogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBNb2RlbCBzdG9yZSB1cmkgKHNob3VsZCBzdGFydCB3aXRoIHN0b3JlOi8vKS4gUHJvdmlkZWQgYXMgYW4gaW5wdXQgKERhdGFJdGVtKS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIElmIHVzaW5nIE1MUnVuIFNESywgYG1vZGVsX3BhdGhgIGNhbiBhbHNvIGJlIHByb3ZpZGVkIGFzIGEgcGFyYW1ldGVyIChzdHJpbmcpLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgVG8gZ2VuZXJhdGUgYSB2YWxpZCBtb2RlbCBzdG9yZSBVUkksIHBsZWFzZSBsb2cgdGhlIG1vZGVsIGJlZm9yZSBydW5uaW5nIHRoaXMgZnVuY3Rpb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBgZW5kcG9pbnRfaWRgIG9mIGV4aXN0aW5nIG1vZGVsIGVuZHBvaW50IGlzIHByb3ZpZGVkLCBtYWtlIHN1cmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoYXQgaXQgaGFzIGEgc2ltaWxhciBtb2RlbCBzdG9yZSBwYXRoLCBvdGhlcndpc2UgdGhlIGRyaWZ0IGFuYWx5c2lzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3b24ndCBiZSB0cmlnZ2VyZWQuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICAgICAgICAgICAgICAgICAgICAgICAgICBBIHN0cmluZyAvIGludGVnZXIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgLyBpbnRlZ2VycyB0aGF0IHJlcHJlc2VudCB0aGUgY29sdW1uIG5hbWVzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAvIGluZGljZXMgdG8gZHJvcC4gV2hlbiB0aGUgZGF0YXNldCBpcyBhIGxpc3Qgb3IgYSBudW1weSBhcnJheSB0aGlzIHBhcmFtZXRlciBtdXN0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSByZXByZXNlbnRlZCBieSBpbnRlZ2Vycy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldCBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2xhc3NpZmljYXRpb24gdGFza3MuIFRoZSBsYWJlbCBjb2x1bW4gY2FuIGJlIGFjY2Vzc2VkIGZyb20gdGhlIG1vZGVsIG9iamVjdCwgb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZSBmZWF0dXJlIHZlY3RvciBwcm92aWRlZCBpZiBhdmFpbGFibGUuCiAgICA6cGFyYW0gZmVhdHVyZV9jb2x1bW5zOiAgICAgICAgICAgICAgICAgICAgICAgICBMaXN0IG9mIGZlYXR1cmUgY29sdW1ucyB0aGF0IHdpbGwgYmUgdXNlZCB0byBidWlsZCB0aGUgZGF0YWZyYW1lIHdoZW4gZGF0YXNldCBpcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZnJvbSB0eXBlIGxpc3Qgb3IgbnVtcHkgYXJyYXkuCiAgICA6cGFyYW0gbG9nX3Jlc3VsdF9zZXQ6ICAgICAgICAgICAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIGxvZyB0aGUgcmVzdWx0IHNldCAtIGEgRGF0YUZyYW1lIG9mIHRoZSBnaXZlbiBpbnB1dHMgY29uY2F0ZW5hdGVkIHdpdGgKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZSBwcmVkaWN0aW9ucy4gRGVmYXVsdGVkIHRvIFRydWUuCiAgICA6cGFyYW0gcmVzdWx0X3NldF9uYW1lOiAgICAgICAgICAgICAgICAgICAgICAgICBUaGUgZGIga2V5IHRvIHNldCBuYW1lIG9mIHRoZSBwcmVkaWN0aW9uIHJlc3VsdCBhbmQgdGhlIGZpbGVuYW1lLiBEZWZhdWx0ZWQgdG8KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdwcmVkaWN0aW9uJy4KICAgIDpwYXJhbSBiYXRjaF9pZDogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSBJRCBvZiB0aGUgZ2l2ZW4gYmF0Y2ggKGluZmVyZW5jZSBkYXRhc2V0KS4gSWYgYE5vbmVgLCBpdCB3aWxsIGJlIGdlbmVyYXRlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFdpbGwgYmUgbG9nZ2VkIGFzIGEgcmVzdWx0IG9mIHRoZSBydW4uCiAgICA6cGFyYW0gYXJ0aWZhY3RzX3RhZzogICAgICAgICAgICAgICAgICAgICAgICAgICBUYWcgdG8gdXNlIGZvciBwcmVkaWN0aW9uIHNldCByZXN1bHQgYXJ0aWZhY3QuCiAgICA6cGFyYW0gcGVyZm9ybV9kcmlmdF9hbmFseXNpczogICAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHBlcmZvcm0gZHJpZnQgYW5hbHlzaXMgYmV0d2VlbiB0aGUgc2FtcGxlIHNldCBvZiB0aGUgbW9kZWwgb2JqZWN0IHRvIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGF0YXNldCBnaXZlbi4gQnkgZGVmYXVsdCwgTm9uZSwgd2hpY2ggbWVhbnMgaXQgd2lsbCBwZXJmb3JtIGRyaWZ0IGFuYWx5c2lzIGlmIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kZWwgYWxyZWFkeSBoYXMgZmVhdHVyZSBzdGF0cyB0aGF0IGFyZSBjb25zaWRlcmVkIGFzIGEgcmVmZXJlbmNlIHNhbXBsZSBzZXQuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBQZXJmb3JtaW5nIGRyaWZ0IGFuYWx5c2lzIG9uIGEgbmV3IGVuZHBvaW50IGlkIHdpbGwgZ2VuZXJhdGUgYSBuZXcgbW9kZWwgZW5kcG9pbnQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHJlY29yZC4KICAgIDpwYXJhbSBlbmRwb2ludF9pZDogICAgICAgICAgICAgICAgICAgICAgICAgICAgIE1vZGVsIGVuZHBvaW50IHVuaXF1ZSBJRC4gSWYgYHBlcmZvcm1fZHJpZnRfYW5hbHlzaXNgIHdhcyBzZXQsIHRoZSBlbmRwb2ludF9pZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSB1c2VkIHRvIHBlcmZvcm0gdGhlIGFuYWx5c2lzIG9uIGV4aXN0aW5nIG1vZGVsIGVuZHBvaW50LCBvciBpZiBpdCBkb2VzIG5vdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZXhpc3QgYSBuZXcgbW9kZWwgZW5kcG9pbnQgd2lsbCBiZSBjcmVhdGVkIHdpdGggYSBuZXdseSBnZW5lcmF0ZWQgSUQuCiAgICA6cGFyYW0gbW9kZWxfZW5kcG9pbnRfbmFtZTogICAgICAgICAgICAgICAgICAgICBJZiBhIG5ldyBtb2RlbCBlbmRwb2ludCBpcyBnZW5lcmF0ZWQsIHRoZSBtb2RlbCBuYW1lIHdpbGwgYmUgcHJlc2VudGVkIHVuZGVyIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGVuZHBvaW50LgogICAgOnBhcmFtIG1vZGVsX2VuZHBvaW50X3NhbXBsZV9zZXQ6ICAgICAgICAgICAgICAgQSBzYW1wbGUgZGF0YXNldCB0byBnaXZlIHRvIGNvbXBhcmUgdGhlIGlucHV0cyBpbiB0aGUgZHJpZnQgYW5hbHlzaXMuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgcHJvdmlkZWQgYXMgYW4gaW5wdXQgKERhdGFJdGVtKSBvciBhcyBhIHBhcmFtZXRlciAoZS5nLiBzdHJpbmcsIGxpc3QsIERhdGFGcmFtZSkuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBUaGUgZGVmYXVsdCBjaG9zZW4gc2FtcGxlIHNldCB3aWxsIGFsd2F5cyBiZSB0aGUgb25lIHdobyBpcyBzZXQgaW4gdGhlIG1vZGVsIGFydGlmYWN0IGl0c2VsZi4KICAgIDpwYXJhbSB0cmlnZ2VyX21vbml0b3Jpbmdfam9iOiAgICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gdHJpZ2dlciB0aGUgYmF0Y2ggZHJpZnQgYW5hbHlzaXMgYWZ0ZXIgdGhlIGluZmVyIGpvYi4KICAgIDpwYXJhbSBiYXRjaF9pbWFnZV9qb2I6ICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSBpbWFnZSB0aGF0IHdpbGwgYmUgdXNlZCB0byByZWdpc3RlciB0aGUgbW9uaXRvcmluZyBiYXRjaCBqb2IgaWYgbm90IGV4aXN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQnkgZGVmYXVsdCwgdGhlIGltYWdlIGlzIG1scnVuL21scnVuLgogICAgOnBhcmFtIG1vZGVsX2VuZHBvaW50X2RyaWZ0X3RocmVzaG9sZDogICAgICAgICAgVGhlIHRocmVzaG9sZCBvZiB3aGljaCB0byBtYXJrIGRyaWZ0cy4gRGVmYXVsdGVkIHRvIDAuNy4KICAgIDpwYXJhbSBtb2RlbF9lbmRwb2ludF9wb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQ6IFRoZSB0aHJlc2hvbGQgb2Ygd2hpY2ggdG8gbWFyayBwb3NzaWJsZSBkcmlmdHMuIERlZmF1bHRlZCB0byAwLjUuCgogICAgcmFpc2VzIE1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3I6IGlmIGJvdGggYG1vZGVsX3BhdGhgIGFuZCBgZW5kcG9pbnRfaWRgIGFyZSBub3QgcHJvdmlkZWQKICAgICIiIgoKICAgIGlmIHRyaWdnZXJfbW9uaXRvcmluZ19qb2I6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybmluZygKICAgICAgICAgICAgIlRoZSBgdHJpZ2dlcl9tb25pdG9yaW5nX2pvYmAgcGFyYW1ldGVyIGlzIGRlcHJlY2F0ZWQgYW5kIHdpbGwgYmUgcmVtb3ZlZCBvbmNlIHRoZSB2ZXJzaW9uaW5nIG1lY2hhbmlzbSBpcyBpbXBsZW1lbnRlZC4gIgogICAgICAgICAgICAiaWYgeW91IGFyZSB1c2luZyBtbHJ1bjwxLjcuMCwgcGxlYXNlIGltcG9ydCB0aGUgcHJldmlvdXMgdmVyc2lvbiBvZiB0aGlzIGZ1bmN0aW9uLCBmb3IgZXhhbXBsZSAiCiAgICAgICAgICAgICInaHViOi8vYmF0Y2hfaW5mZXJlbmNlX3YyOjIuNS4wJy4iCiAgICAgICAgKQogICAgaWYgYmF0Y2hfaW1hZ2Vfam9iOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm5pbmcoCiAgICAgICAgICAgICJUaGUgYGJhdGNoX2ltYWdlX2pvYmAgcGFyYW1ldGVyIGlzIGRlcHJlY2F0ZWQgYW5kIHdpbGwgYmUgcmVtb3ZlZCBvbmNlIHRoZSB2ZXJzaW9uaW5nIG1lY2hhbmlzbSBpcyBpbXBsZW1lbnRlZC4gIgogICAgICAgICAgICAiaWYgeW91IGFyZSB1c2luZyBtbHJ1bjwxLjcuMCwgcGxlYXNlIGltcG9ydCB0aGUgcHJldmlvdXMgdmVyc2lvbiBvZiB0aGlzIGZ1bmN0aW9uLCBmb3IgZXhhbXBsZSAiCiAgICAgICAgICAgICInaHViOi8vYmF0Y2hfaW5mZXJlbmNlX3YyOjIuNS4wJy4iCiAgICAgICAgKQogICAgaWYgbW9kZWxfZW5kcG9pbnRfZHJpZnRfdGhyZXNob2xkOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm5pbmcoCiAgICAgICAgICAgICJUaGUgYG1vZGVsX2VuZHBvaW50X2RyaWZ0X3RocmVzaG9sZGAgcGFyYW1ldGVyIGlzIGRlcHJlY2F0ZWQgYW5kIHdpbGwgYmUgcmVtb3ZlZCBvbmNlIHRoZSB2ZXJzaW9uaW5nIG1lY2hhbmlzbSBpcyBpbXBsZW1lbnRlZC4gIgogICAgICAgICAgICAiaWYgeW91IGFyZSB1c2luZyBtbHJ1bjwxLjcuMCwgcGxlYXNlIGltcG9ydCB0aGUgcHJldmlvdXMgdmVyc2lvbiBvZiB0aGlzIGZ1bmN0aW9uLCBmb3IgZXhhbXBsZSAiCiAgICAgICAgICAgICInaHViOi8vYmF0Y2hfaW5mZXJlbmNlX3YyOjIuNS4wJy4iCiAgICAgICAgKQogICAgaWYgbW9kZWxfZW5kcG9pbnRfcG9zc2libGVfZHJpZnRfdGhyZXNob2xkOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm5pbmcoCiAgICAgICAgICAgICJUaGUgYG1vZGVsX2VuZHBvaW50X3Bvc3NpYmxlX2RyaWZ0X3RocmVzaG9sZGAgcGFyYW1ldGVyIGlzIGRlcHJlY2F0ZWQgYW5kIHdpbGwgYmUgcmVtb3ZlZCBvbmNlIHRoZSB2ZXJzaW9uaW5nIG1lY2hhbmlzbSBpcyBpbXBsZW1lbnRlZC4gIgogICAgICAgICAgICAiaWYgeW91IGFyZSB1c2luZyBtbHJ1bjwxLjcuMCwgcGxlYXNlIGltcG9ydCB0aGUgcHJldmlvdXMgdmVyc2lvbiBvZiB0aGlzIGZ1bmN0aW9uLCBmb3IgZXhhbXBsZSAiCiAgICAgICAgICAgICInaHViOi8vYmF0Y2hfaW5mZXJlbmNlX3YyOjIuNS4wJy4iCiAgICAgICAgKQoKICAgICMgTG9hZGluZyB0aGUgbW9kZWw6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJMb2FkaW5nIG1vZGVsLi4uIikKICAgIGlmIGlzaW5zdGFuY2UobW9kZWxfcGF0aCwgbWxydW4uRGF0YUl0ZW0pOgogICAgICAgIG1vZGVsX3BhdGggPSBtb2RlbF9wYXRoLmFydGlmYWN0X3VybAogICAgaWYgbm90IG1scnVuLmRhdGFzdG9yZS5pc19zdG9yZV91cmkobW9kZWxfcGF0aCk6CiAgICAgICAgcmFpc2UgbWxydW4uZXJyb3JzLk1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3IoCiAgICAgICAgICAgIGYiVGhlIHByb3ZpZGVkIG1vZGVsIHBhdGggKHttb2RlbF9wYXRofSkgaXMgaW52YWxpZCAtIHNob3VsZCBzdGFydCB3aXRoIGBzdG9yZTovL2AuICIKICAgICAgICAgICAgZiJQbGVhc2UgbWFrZSBzdXJlIHRoYXQgeW91IGhhdmUgbG9nZ2VkIHRoZSBtb2RlbCB1c2luZyBgcHJvamVjdC5sb2dfbW9kZWwoKWAgIgogICAgICAgICAgICBmIndoaWNoIGdlbmVyYXRlcyBhIHVuaXF1ZSBzdG9yZSB1cmkgZm9yIHRoZSBsb2dnZWQgbW9kZWwuIgogICAgICAgICkKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbChtb2RlbF9wYXRoPW1vZGVsX3BhdGgsIGNvbnRleHQ9Y29udGV4dCkKCiAgICBpZiBsYWJlbF9jb2x1bW5zIGlzIE5vbmU6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFsKICAgICAgICAgICAgb3V0cHV0Lm5hbWUgZm9yIG91dHB1dCBpbiBtb2RlbF9oYW5kbGVyLl9tb2RlbF9hcnRpZmFjdC5zcGVjLm91dHB1dHMKICAgICAgICBdCgogICAgaWYgZmVhdHVyZV9jb2x1bW5zIGlzIE5vbmU6CiAgICAgICAgZmVhdHVyZV9jb2x1bW5zID0gWwogICAgICAgICAgICBpbnB1dC5uYW1lIGZvciBpbnB1dCBpbiBtb2RlbF9oYW5kbGVyLl9tb2RlbF9hcnRpZmFjdC5zcGVjLmlucHV0cwogICAgICAgIF0KCiAgICAjIEdldCBkYXRhc2V0IGJ5IG9iamVjdCwgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJMb2FkaW5nIGRhdGEuLi4iKQogICAgeCwgbGFiZWxfY29sdW1ucyA9IG1scnVuLm1vZGVsX21vbml0b3JpbmcuYXBpLnJlYWRfZGF0YXNldF9hc19kYXRhZnJhbWUoCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGZlYXR1cmVfY29sdW1ucz1mZWF0dXJlX2NvbHVtbnMsCiAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICApCgogICAgIyBQcmVkaWN0OgogICAgY29udGV4dC5sb2dnZXIuaW5mbygiQ2FsY3VsYXRpbmcgcHJlZGljdGlvbi4uLiIpCiAgICB5X3ByZWQgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoeCwgKipwcmVkaWN0X2t3YXJncykKCiAgICAjIFByZXBhcmUgdGhlIHJlc3VsdCBzZXQ6CiAgICByZXN1bHRfc2V0ID0gX3ByZXBhcmVfcmVzdWx0X3NldCh4PXgsIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywgeV9wcmVkPXlfcHJlZCkKCiAgICAjIENoZWNrIGZvciBsb2dnaW5nIHRoZSByZXN1bHQgc2V0OgogICAgaWYgbG9nX3Jlc3VsdF9zZXQ6CiAgICAgICAgbWxydW4ubW9kZWxfbW9uaXRvcmluZy5hcGkubG9nX3Jlc3VsdCgKICAgICAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgICAgICByZXN1bHRfc2V0X25hbWU9cmVzdWx0X3NldF9uYW1lLAogICAgICAgICAgICByZXN1bHRfc2V0PXJlc3VsdF9zZXQsCiAgICAgICAgICAgIGFydGlmYWN0c190YWc9YXJ0aWZhY3RzX3RhZywKICAgICAgICAgICAgYmF0Y2hfaWQ9YmF0Y2hfaWQsCiAgICAgICAgKQoKICAgICMgQ2hlY2sgZm9yIHBlcmZvcm1pbmcgZHJpZnQgYW5hbHlzaXMKICAgIGlmICgKICAgICAgICBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzIGlzIE5vbmUKICAgICAgICBhbmQgbW9kZWxfaGFuZGxlci5fbW9kZWxfYXJ0aWZhY3Quc3BlYy5mZWF0dXJlX3N0YXRzIGlzIG5vdCBOb25lCiAgICApOgogICAgICAgIHBlcmZvcm1fZHJpZnRfYW5hbHlzaXMgPSBUcnVlCiAgICBpZiBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIlBlcmZvcm1pbmcgZHJpZnQgYW5hbHlzaXMuLi4iKQogICAgICAgICMgR2V0IHRoZSBzYW1wbGUgc2V0IHN0YXRpc3RpY3MgKGVpdGhlciBmcm9tIHRoZSBzYW1wbGUgc2V0IG9yIGZyb20gdGhlIHN0YXRpc3RpY3MgbG9nZ2VkIHdpdGggdGhlIG1vZGVsKQogICAgICAgIHN0YXRpc3RpY3NfaW5wdXRfZmlsdGVyZWQgPSBfZ2V0X3NhbXBsZV9zZXRfc3RhdGlzdGljc19wYXJhbWV0ZXJzKAogICAgICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgICAgIG1vZGVsX2VuZHBvaW50X3NhbXBsZV9zZXQ9bW9kZWxfZW5kcG9pbnRfc2FtcGxlX3NldCwKICAgICAgICAgICAgbW9kZWxfYXJ0aWZhY3RfZmVhdHVyZV9zdGF0cz1tb2RlbF9oYW5kbGVyLl9tb2RlbF9hcnRpZmFjdC5zcGVjLmZlYXR1cmVfc3RhdHMsCiAgICAgICAgICAgIGZlYXR1cmVfY29sdW1ucz1mZWF0dXJlX2NvbHVtbnMsCiAgICAgICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICApCiAgICAgICAgc2FtcGxlX3NldF9zdGF0aXN0aWNzID0gbWxydW4ubW9kZWxfbW9uaXRvcmluZy5hcGkuZ2V0X3NhbXBsZV9zZXRfc3RhdGlzdGljcygKICAgICAgICAgICAgKipzdGF0aXN0aWNzX2lucHV0X2ZpbHRlcmVkCiAgICAgICAgKQogICAgICAgIG1scnVuLm1vZGVsX21vbml0b3JpbmcuYXBpLnJlY29yZF9yZXN1bHRzKAogICAgICAgICAgICBwcm9qZWN0PWNvbnRleHQucHJvamVjdCwKICAgICAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgICAgICBlbmRwb2ludF9pZD1lbmRwb2ludF9pZCwKICAgICAgICAgICAgbW9kZWxfcGF0aD1tb2RlbF9wYXRoLAogICAgICAgICAgICBtb2RlbF9lbmRwb2ludF9uYW1lPW1vZGVsX2VuZHBvaW50X25hbWUsCiAgICAgICAgICAgIGluZmVyX3Jlc3VsdHNfZGY9cmVzdWx0X3NldC5jb3B5KCksCiAgICAgICAgICAgIHNhbXBsZV9zZXRfc3RhdGlzdGljcz1zYW1wbGVfc2V0X3N0YXRpc3RpY3MsCiAgICAgICAgKQo= + code_origin: '' + auto_build: false + allow_empty_resources: true + filename: batch_inference_v2.py entry_points: infer: - lineno: 102 - name: infer parameters: - name: context type: MLClientCtx doc: MLRun context. - name: dataset - type: Union[DataItem, list, dict, DataFrame, Series, ndarray] doc: The dataset to infer through the model. Provided as an input (DataItem) that represents Dataset artifact / Feature vector URI. If using MLRun SDK, `dataset` can also be provided as a list, dictionary or numpy array. - name: model_path - type: Union[str, DataItem] doc: Model store uri (should start with store://). Provided as an input (DataItem). If using MLRun SDK, `model_path` can also be provided as a parameter (string). To generate a valid model store URI, please log the model before running @@ -23,19 +34,16 @@ spec: make sure that it has a similar model store path, otherwise the drift analysis won't be triggered. - name: drop_columns - type: Union[str, List[str], int, List[int]] doc: A string / integer or a list of strings / integers that represent the column names / indices to drop. When the dataset is a list or a numpy array this parameter must be represented by integers. default: null - name: label_columns - type: Union[str, List[str]] doc: The target label(s) of the column(s) in the dataset for Regression or Classification tasks. The label column can be accessed from the model object, or the feature vector provided if available. default: null - name: feature_columns - type: Union[str, List[str]] doc: List of feature columns that will be used to build the dataframe when dataset is from type list or numpy array. default: null @@ -69,8 +77,9 @@ spec: - name: endpoint_id type: str doc: Model endpoint unique ID. If `perform_drift_analysis` was set, the endpoint_id - will be used either to perform the analysis on existing model endpoint or - to generate a new model endpoint record. + will be used to perform the analysis on existing model endpoint, or if it + does not exist a new model endpoint will be created with a newly generated + ID. default: '' - name: model_endpoint_name type: str @@ -78,31 +87,25 @@ spec: under this endpoint. default: batch-infer - name: model_endpoint_sample_set - type: Union[DataItem, list, dict, DataFrame, Series, ndarray] doc: A sample dataset to give to compare the inputs in the drift analysis. Can be provided as an input (DataItem) or as a parameter (e.g. string, list, DataFrame). The default chosen sample set will always be the one who is set in the model artifact itself. default: null - name: trigger_monitoring_job - type: Optional[bool] doc: Whether to trigger the batch drift analysis after the infer job. default: null - name: batch_image_job - type: Optional[str] doc: The image that will be used to register the monitoring batch job if not exist. By default, the image is mlrun/mlrun. default: null - name: model_endpoint_drift_threshold - type: Optional[float] doc: The threshold of which to mark drifts. Defaulted to 0.7. default: null - name: model_endpoint_possible_drift_threshold - type: Optional[float] doc: The threshold of which to mark possible drifts. Defaulted to 0.5. default: null - has_kwargs: true - has_varargs: false + name: infer doc: 'Perform a prediction on the provided dataset using the specified model. Ensure that the model has already been logged under the current project. @@ -123,21 +126,10 @@ spec: At the moment, this function is supported for `mlrun>=1.5.0` versions.' + has_kwargs: true + has_varargs: false + lineno: 117 command: '' - build: - with_mlrun: false - code_origin: '' - origin_filename: '' - auto_build: false - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCgpmcm9tIGluc3BlY3QgaW1wb3J0IHNpZ25hdHVyZQpmcm9tIHR5cGluZyBpbXBvcnQgQW55LCBEaWN0LCBMaXN0LCBVbmlvbiwgT3B0aW9uYWwKaW1wb3J0IG1scnVuCgp0cnk6CiAgICBpbXBvcnQgbWxydW4ubW9kZWxfbW9uaXRvcmluZy5hcGkKZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3I6CiAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5Ob3RGb3VuZEVycm9yKAogICAgICAgIGYiUGxlYXNlIHVwZGF0ZSB5b3VyIGBtbHJ1bmAgdmVyc2lvbiB0byA+PTEuNS4wIG9yIHVzZSBhbiAiCiAgICAgICAgZiJvbGRlciB2ZXJzaW9uIG9mIHRoZSBiYXRjaCBpbmZlcmVuY2UgZnVuY3Rpb24uIgogICAgKQoKaW1wb3J0IG51bXB5IGFzIG5wCmltcG9ydCBwYW5kYXMgYXMgcGQKZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLmF1dG9fbWxydW4gaW1wb3J0IEF1dG9NTFJ1bgoKCmRlZiBfcHJlcGFyZV9yZXN1bHRfc2V0KHg6IHBkLkRhdGFGcmFtZSwgbGFiZWxfY29sdW1uczogTGlzdFtzdHJdLCB5X3ByZWQ6IG5wLm5kYXJyYXkpIC0+IHBkLkRhdGFGcmFtZToKICAgICIiIgogICAgU2V0IGRlZmF1bHQgbGFiZWwgY29sdW1uIG5hbWVzIGFuZCB2YWxpZGF0ZSBnaXZlbiBuYW1lcyB0byBwcmVwYXJlIHRoZSByZXN1bHQgc2V0IC0gYSBjb25jYXRlbmF0aW9uIG9mIHRoZSBpbnB1dHMKICAgICh4KSBhbmQgdGhlIG1vZGVsIHByZWRpY3Rpb25zICh5X3ByZWQpLgoKICAgIDpwYXJhbSB4OiAgICAgICAgICAgICBUaGUgaW5wdXRzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6IEEgbGlzdCBvZiBzdHJpbmdzIHJlcHJlc2VudGluZyB0aGUgdGFyZ2V0IGNvbHVtbiBuYW1lcyB0byBhZGQgdG8gdGhlIHByZWRpY3Rpb25zLiBEZWZhdWx0IG5hbWUKICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIHVzZWQgaW4gY2FzZSB0aGUgbGlzdCBpcyBlbXB0eSAocHJlZGljdGVkX2xhYmVsX3tpfSkuCiAgICA6cGFyYW0geV9wcmVkOiAgICAgICAgVGhlIG1vZGVsIHByZWRpY3Rpb25zIG9uIHRoZSBpbnB1dHMuCgogICAgOnJldHVybnM6IFRoZSByZXN1bHQgc2V0LgoKICAgIHJhaXNlcyBNTFJ1bkludmFsaWRBcmd1bWVudEVycm9yOiBJZiB0aGUgbGFiZWxzIGNvbHVtbnMgYW1vdW50IGRvIG5vdCBtYXRjaCB0aGUgb3V0cHV0cyBvciBpZiBvbmUgb2YgdGhlIGxhYmVsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGNvbHVtbiBhbHJlYWR5IGV4aXN0cyBpbiB0aGUgZGF0YXNldC4KICAgICIiIgogICAgIyBQcmVwYXJlIGRlZmF1bHQgdGFyZ2V0IGNvbHVtbnMgbmFtZXMgaWYgbm90IHByb3ZpZGVkOgogICAgcHJlZGljdGlvbl9jb2x1bW5zX2Ftb3VudCA9IDEgaWYgbGVuKHlfcHJlZC5zaGFwZSkgPT0gMSBlbHNlIHlfcHJlZC5zaGFwZVsxXQogICAgaWYgbGVuKGxhYmVsX2NvbHVtbnMpID09IDA6CiAgICAgICAgIyBBZGQgZGVmYXVsdCBsYWJlbCBjb2x1bW4gbmFtZXM6CiAgICAgICAgaWYgcHJlZGljdGlvbl9jb2x1bW5zX2Ftb3VudCA9PSAxOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWyJwcmVkaWN0ZWRfbGFiZWwiXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbCiAgICAgICAgICAgICAgICBmInByZWRpY3RlZF9sYWJlbF97aX0iIGZvciBpIGluIHJhbmdlKHByZWRpY3Rpb25fY29sdW1uc19hbW91bnQpCiAgICAgICAgICAgIF0KCiAgICAjIFZhbGlkYXRlIHRoZSBsYWJlbCBjb2x1bW5zOgogICAgaWYgcHJlZGljdGlvbl9jb2x1bW5zX2Ftb3VudCAhPSBsZW4obGFiZWxfY29sdW1ucyk6CiAgICAgICAgIyBObyBlcXVhbGl0eSBiZXR3ZWVuIHByb3ZpZGVkIGxhYmVsIGNvbHVtbiBuYW1lcyBhbmQgb3V0cHV0cyBhbW91bnQ6CiAgICAgICAgcmFpc2UgbWxydW4uZXJyb3JzLk1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3IoCiAgICAgICAgICAgIGYiVGhlIG51bWJlciBvZiBwcmVkaWN0ZWQgbGFiZWxzOiB7cHJlZGljdGlvbl9jb2x1bW5zX2Ftb3VudH0gIgogICAgICAgICAgICBmImlzIG5vdCBlcXVhbCB0byB0aGUgZ2l2ZW4gbGFiZWwgY29sdW1uczoge2xlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgKQogICAgY29tbW9uX2xhYmVscyA9IHNldChsYWJlbF9jb2x1bW5zKSAmIHNldCh4LmNvbHVtbnMudG9saXN0KCkpCiAgICBpZiBjb21tb25fbGFiZWxzOgogICAgICAgICMgTGFiZWwgY29sdW1uIGV4aXN0IGluIHRoZSBvcmlnaW5hbCBpbnB1dHM6CiAgICAgICAgcmFpc2UgbWxydW4uZXJyb3JzLk1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3IoCiAgICAgICAgICAgIGYiVGhlIGxhYmVsczoge2NvbW1vbl9sYWJlbHN9IGFyZSBhbHJlYWR5IGV4aXN0ZWQgaW4gdGhlIGdpdmVuIGRhdGFzZXQuIgogICAgICAgICkKCiAgICByZXR1cm4gcGQuY29uY2F0KAogICAgICAgIFt4LCBwZC5EYXRhRnJhbWUoeV9wcmVkLCBjb2x1bW5zPWxhYmVsX2NvbHVtbnMsIGluZGV4PXguaW5kZXgpXSwgYXhpcz0xCiAgICApCgoKZGVmIF9nZXRfc2FtcGxlX3NldF9zdGF0aXN0aWNzX3BhcmFtZXRlcnMoY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsX2VuZHBvaW50X3NhbXBsZV9zZXQ6IFVuaW9uWwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWxydW4uRGF0YUl0ZW0sIGxpc3QsIGRpY3QsIHBkLkRhdGFGcmFtZSwgcGQuU2VyaWVzLCBucC5uZGFycmF5XSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kZWxfYXJ0aWZhY3RfZmVhdHVyZV9zdGF0czogZGljdCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZmVhdHVyZV9jb2x1bW5zOiBPcHRpb25hbFtMaXN0XSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZHJvcF9jb2x1bW5zOiBPcHRpb25hbFtMaXN0XSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGFiZWxfY29sdW1uczogT3B0aW9uYWxbTGlzdF0pIC0+IERpY3Rbc3RyLCBBbnldOgogICAgc3RhdGljc19pbnB1dF9mdWxsX2RpY3QgPSBkaWN0KHNhbXBsZV9zZXQ9bW9kZWxfZW5kcG9pbnRfc2FtcGxlX3NldCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlbF9hcnRpZmFjdF9mZWF0dXJlX3N0YXRzPW1vZGVsX2FydGlmYWN0X2ZlYXR1cmVfc3RhdHMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2FtcGxlX3NldF9jb2x1bW5zPWZlYXR1cmVfY29sdW1ucywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGVfc2V0X2Ryb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2FtcGxlX3NldF9sYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMpCiAgICBnZXRfc2FtcGxlX3N0YXRpY3NfZnVuY3Rpb24gPSBtbHJ1bi5tb2RlbF9tb25pdG9yaW5nLmFwaS5nZXRfc2FtcGxlX3NldF9zdGF0aXN0aWNzCiAgICBzdGF0aWNzX2Z1bmN0aW9uX2lucHV0X2RpY3QgPSBzaWduYXR1cmUoZ2V0X3NhbXBsZV9zdGF0aWNzX2Z1bmN0aW9uKS5wYXJhbWV0ZXJzCiAgICAjICBBcyBhIHJlc3VsdCBvZiBjaGFuZ2VzIHRvIGlucHV0IHBhcmFtZXRlcnMgaW4gdGhlIG1scnVuLWdldF9zYW1wbGVfc2V0X3N0YXRpc3RpY3MgZnVuY3Rpb24sCiAgICAjICB3ZSB3aWxsIG5vdyBzZW5kIG9ubHkgdGhlIHBhcmFtZXRlcnMgaXQgZXhwZWN0cy4KICAgIHN0YXRpc3RpY3NfaW5wdXRfZmlsdGVyZWQgPSB7a2V5OiBzdGF0aWNzX2lucHV0X2Z1bGxfZGljdFtrZXldIGZvciBrZXkgaW4gc3RhdGljc19mdW5jdGlvbl9pbnB1dF9kaWN0fQogICAgaWYgbGVuKHN0YXRpc3RpY3NfaW5wdXRfZmlsdGVyZWQpICE9IGxlbihzdGF0aWNzX2Z1bmN0aW9uX2lucHV0X2RpY3QpOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm5pbmcoZiJnZXRfc2FtcGxlX3NldF9zdGF0aXN0aWNzIGlzIGluIGFuIG9sZGVyIHZlcnNpb247ICIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzb21lIHBhcmFtZXRlcnMgd2lsbCBub3QgYmUgc2VudCB0byB0aGUgZnVuY3Rpb24uIgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZiIgRXhwZWN0ZWQgaW5wdXQ6IHtsaXN0KHN0YXRpY3NfZnVuY3Rpb25faW5wdXRfZGljdC5rZXlzKCkpfSwiCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmIiBhY3R1YWwgaW5wdXQ6IHtsaXN0KHN0YXRpc3RpY3NfaW5wdXRfZmlsdGVyZWQua2V5cygpKX0iKQogICAgcmV0dXJuIHN0YXRpc3RpY3NfaW5wdXRfZmlsdGVyZWQKCgpkZWYgaW5mZXIoCiAgICAgICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICAgICAgZGF0YXNldDogVW5pb25bbWxydW4uRGF0YUl0ZW0sIGxpc3QsIGRpY3QsIHBkLkRhdGFGcmFtZSwgcGQuU2VyaWVzLCBucC5uZGFycmF5XSwKICAgICAgICBtb2RlbF9wYXRoOiBVbmlvbltzdHIsIG1scnVuLkRhdGFJdGVtXSwKICAgICAgICBkcm9wX2NvbHVtbnM6IFVuaW9uW3N0ciwgTGlzdFtzdHJdLCBpbnQsIExpc3RbaW50XV0gPSBOb25lLAogICAgICAgIGxhYmVsX2NvbHVtbnM6IFVuaW9uW3N0ciwgTGlzdFtzdHJdXSA9IE5vbmUsCiAgICAgICAgZmVhdHVyZV9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXV0gPSBOb25lLAogICAgICAgIGxvZ19yZXN1bHRfc2V0OiBib29sID0gVHJ1ZSwKICAgICAgICByZXN1bHRfc2V0X25hbWU6IHN0ciA9ICJwcmVkaWN0aW9uIiwKICAgICAgICBiYXRjaF9pZDogc3RyID0gTm9uZSwKICAgICAgICBhcnRpZmFjdHNfdGFnOiBzdHIgPSAiIiwKICAgICAgICAjIERyaWZ0IGFuYWx5c2lzIHBhcmFtZXRlcnMKICAgICAgICBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzOiBib29sID0gTm9uZSwKICAgICAgICBlbmRwb2ludF9pZDogc3RyID0gIiIsCiAgICAgICAgIyBUaGUgZm9sbG93aW5nIG1vZGVsIGVuZHBvaW50IHBhcmFtZXRlcnMgYXJlIHJlbGV2YW50IG9ubHkgaWY6CiAgICAgICAgIyBwZXJmb3JtIGRyaWZ0IGFuYWx5c2lzIGlzIG5vdCBkaXNhYmxlZAogICAgICAgICMgYSBuZXcgbW9kZWwgZW5kcG9pbnQgcmVjb3JkIGlzIGdvaW5nIHRvIGJlIGdlbmVyYXRlZAogICAgICAgIG1vZGVsX2VuZHBvaW50X25hbWU6IHN0ciA9ICJiYXRjaC1pbmZlciIsCiAgICAgICAgbW9kZWxfZW5kcG9pbnRfc2FtcGxlX3NldDogVW5pb25bCiAgICAgICAgICAgIG1scnVuLkRhdGFJdGVtLCBsaXN0LCBkaWN0LCBwZC5EYXRhRnJhbWUsIHBkLlNlcmllcywgbnAubmRhcnJheQogICAgICAgIF0gPSBOb25lLAoKICAgICAgICAjIHRoZSBmb2xsb3dpbmcgcGFyYW1ldGVycyBhcmUgZGVwcmVjYXRlZCBhbmQgd2lsbCBiZSByZW1vdmVkIG9uY2UgdGhlIHZlcnNpb25pbmcgbWVjaGFuaXNtIGlzIGltcGxlbWVudGVkCiAgICAgICAgIyBUT0RPOiBSZW1vdmUgdGhlIGZvbGxvd2luZyBwYXJhbWV0ZXJzIG9uY2UgRkhVQi0xMyBpcyByZXNvbHZlZAogICAgICAgIHRyaWdnZXJfbW9uaXRvcmluZ19qb2I6IE9wdGlvbmFsW2Jvb2xdID0gTm9uZSwKICAgICAgICBiYXRjaF9pbWFnZV9qb2I6IE9wdGlvbmFsW3N0cl0gPSBOb25lLAogICAgICAgIG1vZGVsX2VuZHBvaW50X2RyaWZ0X3RocmVzaG9sZDogT3B0aW9uYWxbZmxvYXRdID0gTm9uZSwKICAgICAgICBtb2RlbF9lbmRwb2ludF9wb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQ6IE9wdGlvbmFsW2Zsb2F0XSA9IE5vbmUsCgogICAgICAgICMgcHJlZGljdGlvbiBrd2FyZ3MgdG8gcGFzcyB0byB0aGUgbW9kZWwgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICoqcHJlZGljdF9rd2FyZ3M6IERpY3Rbc3RyLCBBbnldLAoKKToKICAgICIiIgogICAgUGVyZm9ybSBhIHByZWRpY3Rpb24gb24gdGhlIHByb3ZpZGVkIGRhdGFzZXQgdXNpbmcgdGhlIHNwZWNpZmllZCBtb2RlbC4KICAgIEVuc3VyZSB0aGF0IHRoZSBtb2RlbCBoYXMgYWxyZWFkeSBiZWVuIGxvZ2dlZCB1bmRlciB0aGUgY3VycmVudCBwcm9qZWN0LgoKICAgIElmIHlvdSB3aXNoIHRvIGFwcGx5IG1vbml0b3JpbmcgdG9vbHMgKGUuZy4sIGRyaWZ0IGFuYWx5c2lzKSwgc2V0IHRoZSBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzIHBhcmFtZXRlciB0byBUcnVlLgogICAgVGhpcyB3aWxsIGNyZWF0ZSBhIG5ldyBtb2RlbCBlbmRwb2ludCByZWNvcmQgdW5kZXIgdGhlIHNwZWNpZmllZCBtb2RlbF9lbmRwb2ludF9uYW1lLgogICAgQWRkaXRpb25hbGx5LCBlbnN1cmUgdGhhdCBtb2RlbCBtb25pdG9yaW5nIGlzIGVuYWJsZWQgYXQgdGhlIHByb2plY3QgbGV2ZWwgYnkgY2FsbGluZyB0aGUKICAgIHByb2plY3QuZW5hYmxlX21vZGVsX21vbml0b3JpbmcoKSBmdW5jdGlvbi4gWW91IGNhbiBhbHNvIGFwcGx5IG1vbml0b3JpbmcgdG8gYW4gZXhpc3RpbmcgbW9kZWwgYnkgcHJvdmlkaW5nIGl0cwogICAgZW5kcG9pbnQgaWQgb3IgbmFtZSwgYW5kIHRoZSBtb25pdG9yaW5nIHRvb2xzIHdpbGwgYmUgYXBwbGllZCB0byB0aGF0IGVuZHBvaW50LgoKICAgIEF0IHRoZSBtb21lbnQsIHRoaXMgZnVuY3Rpb24gaXMgc3VwcG9ydGVkIGZvciBgbWxydW4+PTEuNS4wYCB2ZXJzaW9ucy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gaW5mZXIgdGhyb3VnaCB0aGUgbW9kZWwuIFByb3ZpZGVkIGFzIGFuIGlucHV0IChEYXRhSXRlbSkKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoYXQgcmVwcmVzZW50cyBEYXRhc2V0IGFydGlmYWN0IC8gRmVhdHVyZSB2ZWN0b3IgVVJJLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgdXNpbmcgTUxSdW4gU0RLLCBgZGF0YXNldGAgY2FuIGFsc28gYmUgcHJvdmlkZWQgYXMgYSBsaXN0LCBkaWN0aW9uYXJ5IG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBudW1weSBhcnJheS4KICAgIDpwYXJhbSBtb2RlbF9wYXRoOiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE1vZGVsIHN0b3JlIHVyaSAoc2hvdWxkIHN0YXJ0IHdpdGggc3RvcmU6Ly8pLiBQcm92aWRlZCBhcyBhbiBpbnB1dCAoRGF0YUl0ZW0pLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgdXNpbmcgTUxSdW4gU0RLLCBgbW9kZWxfcGF0aGAgY2FuIGFsc28gYmUgcHJvdmlkZWQgYXMgYSBwYXJhbWV0ZXIgKHN0cmluZykuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBUbyBnZW5lcmF0ZSBhIHZhbGlkIG1vZGVsIHN0b3JlIFVSSSwgcGxlYXNlIGxvZyB0aGUgbW9kZWwgYmVmb3JlIHJ1bm5pbmcgdGhpcyBmdW5jdGlvbi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIElmIGBlbmRwb2ludF9pZGAgb2YgZXhpc3RpbmcgbW9kZWwgZW5kcG9pbnQgaXMgcHJvdmlkZWQsIG1ha2Ugc3VyZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCBpdCBoYXMgYSBzaW1pbGFyIG1vZGVsIHN0b3JlIHBhdGgsIG90aGVyd2lzZSB0aGUgZHJpZnQgYW5hbHlzaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHdvbid0IGJlIHRyaWdnZXJlZC4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgICAgICAgICAgICAgICAgIEEgc3RyaW5nIC8gaW50ZWdlciBvciBhIGxpc3Qgb2Ygc3RyaW5ncyAvIGludGVnZXJzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC8gaW5kaWNlcyB0byBkcm9wLiBXaGVuIHRoZSBkYXRhc2V0IGlzIGEgbGlzdCBvciBhIG51bXB5IGFycmF5IHRoaXMgcGFyYW1ldGVyIG11c3QKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJlIHJlcHJlc2VudGVkIGJ5IGludGVnZXJzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICAgICAgICAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0IGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gVGhlIGxhYmVsIGNvbHVtbiBjYW4gYmUgYWNjZXNzZWQgZnJvbSB0aGUgbW9kZWwgb2JqZWN0LCBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhlIGZlYXR1cmUgdmVjdG9yIHByb3ZpZGVkIGlmIGF2YWlsYWJsZS4KICAgIDpwYXJhbSBmZWF0dXJlX2NvbHVtbnM6ICAgICAgICAgICAgICAgICAgICAgICAgIExpc3Qgb2YgZmVhdHVyZSBjb2x1bW5zIHRoYXQgd2lsbCBiZSB1c2VkIHRvIGJ1aWxkIHRoZSBkYXRhZnJhbWUgd2hlbiBkYXRhc2V0IGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmcm9tIHR5cGUgbGlzdCBvciBudW1weSBhcnJheS4KICAgIDpwYXJhbSBsb2dfcmVzdWx0X3NldDogICAgICAgICAgICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gbG9nIHRoZSByZXN1bHQgc2V0IC0gYSBEYXRhRnJhbWUgb2YgdGhlIGdpdmVuIGlucHV0cyBjb25jYXRlbmF0ZWQgd2l0aAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhlIHByZWRpY3Rpb25zLiBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgIDpwYXJhbSByZXN1bHRfc2V0X25hbWU6ICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSBkYiBrZXkgdG8gc2V0IG5hbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0IGFuZCB0aGUgZmlsZW5hbWUuIERlZmF1bHRlZCB0bwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ3ByZWRpY3Rpb24nLgogICAgOnBhcmFtIGJhdGNoX2lkOiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgVGhlIElEIG9mIHRoZSBnaXZlbiBiYXRjaCAoaW5mZXJlbmNlIGRhdGFzZXQpLiBJZiBgTm9uZWAsIGl0IHdpbGwgYmUgZ2VuZXJhdGVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV2lsbCBiZSBsb2dnZWQgYXMgYSByZXN1bHQgb2YgdGhlIHJ1bi4KICAgIDpwYXJhbSBhcnRpZmFjdHNfdGFnOiAgICAgICAgICAgICAgICAgICAgICAgICAgIFRhZyB0byB1c2UgZm9yIHByZWRpY3Rpb24gc2V0IHJlc3VsdCBhcnRpZmFjdC4KICAgIDpwYXJhbSBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzOiAgICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gcGVyZm9ybSBkcmlmdCBhbmFseXNpcyBiZXR3ZWVuIHRoZSBzYW1wbGUgc2V0IG9mIHRoZSBtb2RlbCBvYmplY3QgdG8gdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkYXRhc2V0IGdpdmVuLiBCeSBkZWZhdWx0LCBOb25lLCB3aGljaCBtZWFucyBpdCB3aWxsIHBlcmZvcm0gZHJpZnQgYW5hbHlzaXMgaWYgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlbCBhbHJlYWR5IGhhcyBmZWF0dXJlIHN0YXRzIHRoYXQgYXJlIGNvbnNpZGVyZWQgYXMgYSByZWZlcmVuY2Ugc2FtcGxlIHNldC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFBlcmZvcm1pbmcgZHJpZnQgYW5hbHlzaXMgb24gYSBuZXcgZW5kcG9pbnQgaWQgd2lsbCBnZW5lcmF0ZSBhIG5ldyBtb2RlbCBlbmRwb2ludAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcmVjb3JkLgogICAgOnBhcmFtIGVuZHBvaW50X2lkOiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTW9kZWwgZW5kcG9pbnQgdW5pcXVlIElELiBJZiBgcGVyZm9ybV9kcmlmdF9hbmFseXNpc2Agd2FzIHNldCwgdGhlIGVuZHBvaW50X2lkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIHVzZWQgZWl0aGVyIHRvIHBlcmZvcm0gdGhlIGFuYWx5c2lzIG9uIGV4aXN0aW5nIG1vZGVsIGVuZHBvaW50IG9yIHRvCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBnZW5lcmF0ZSBhIG5ldyBtb2RlbCBlbmRwb2ludCByZWNvcmQuCiAgICA6cGFyYW0gbW9kZWxfZW5kcG9pbnRfbmFtZTogICAgICAgICAgICAgICAgICAgICBJZiBhIG5ldyBtb2RlbCBlbmRwb2ludCBpcyBnZW5lcmF0ZWQsIHRoZSBtb2RlbCBuYW1lIHdpbGwgYmUgcHJlc2VudGVkIHVuZGVyIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGVuZHBvaW50LgogICAgOnBhcmFtIG1vZGVsX2VuZHBvaW50X3NhbXBsZV9zZXQ6ICAgICAgICAgICAgICAgQSBzYW1wbGUgZGF0YXNldCB0byBnaXZlIHRvIGNvbXBhcmUgdGhlIGlucHV0cyBpbiB0aGUgZHJpZnQgYW5hbHlzaXMuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgcHJvdmlkZWQgYXMgYW4gaW5wdXQgKERhdGFJdGVtKSBvciBhcyBhIHBhcmFtZXRlciAoZS5nLiBzdHJpbmcsIGxpc3QsIERhdGFGcmFtZSkuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBUaGUgZGVmYXVsdCBjaG9zZW4gc2FtcGxlIHNldCB3aWxsIGFsd2F5cyBiZSB0aGUgb25lIHdobyBpcyBzZXQgaW4gdGhlIG1vZGVsIGFydGlmYWN0IGl0c2VsZi4KICAgIDpwYXJhbSB0cmlnZ2VyX21vbml0b3Jpbmdfam9iOiAgICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gdHJpZ2dlciB0aGUgYmF0Y2ggZHJpZnQgYW5hbHlzaXMgYWZ0ZXIgdGhlIGluZmVyIGpvYi4KICAgIDpwYXJhbSBiYXRjaF9pbWFnZV9qb2I6ICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSBpbWFnZSB0aGF0IHdpbGwgYmUgdXNlZCB0byByZWdpc3RlciB0aGUgbW9uaXRvcmluZyBiYXRjaCBqb2IgaWYgbm90IGV4aXN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQnkgZGVmYXVsdCwgdGhlIGltYWdlIGlzIG1scnVuL21scnVuLgogICAgOnBhcmFtIG1vZGVsX2VuZHBvaW50X2RyaWZ0X3RocmVzaG9sZDogICAgICAgICAgVGhlIHRocmVzaG9sZCBvZiB3aGljaCB0byBtYXJrIGRyaWZ0cy4gRGVmYXVsdGVkIHRvIDAuNy4KICAgIDpwYXJhbSBtb2RlbF9lbmRwb2ludF9wb3NzaWJsZV9kcmlmdF90aHJlc2hvbGQ6IFRoZSB0aHJlc2hvbGQgb2Ygd2hpY2ggdG8gbWFyayBwb3NzaWJsZSBkcmlmdHMuIERlZmF1bHRlZCB0byAwLjUuCgogICAgcmFpc2VzIE1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3I6IGlmIGJvdGggYG1vZGVsX3BhdGhgIGFuZCBgZW5kcG9pbnRfaWRgIGFyZSBub3QgcHJvdmlkZWQKICAgICIiIgoKCiAgICBpZiB0cmlnZ2VyX21vbml0b3Jpbmdfam9iOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm5pbmcoIlRoZSBgdHJpZ2dlcl9tb25pdG9yaW5nX2pvYmAgcGFyYW1ldGVyIGlzIGRlcHJlY2F0ZWQgYW5kIHdpbGwgYmUgcmVtb3ZlZCBvbmNlIHRoZSB2ZXJzaW9uaW5nIG1lY2hhbmlzbSBpcyBpbXBsZW1lbnRlZC4gIgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImlmIHlvdSBhcmUgdXNpbmcgbWxydW48MS43LjAsIHBsZWFzZSBpbXBvcnQgdGhlIHByZXZpb3VzIHZlcnNpb24gb2YgdGhpcyBmdW5jdGlvbiwgZm9yIGV4YW1wbGUgIgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIidodWI6Ly9iYXRjaF9pbmZlcmVuY2VfdjI6Mi41LjAnLiIpCiAgICBpZiBiYXRjaF9pbWFnZV9qb2I6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybmluZygiVGhlIGBiYXRjaF9pbWFnZV9qb2JgIHBhcmFtZXRlciBpcyBkZXByZWNhdGVkIGFuZCB3aWxsIGJlIHJlbW92ZWQgb25jZSB0aGUgdmVyc2lvbmluZyBtZWNoYW5pc20gaXMgaW1wbGVtZW50ZWQuICIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJpZiB5b3UgYXJlIHVzaW5nIG1scnVuPDEuNy4wLCBwbGVhc2UgaW1wb3J0IHRoZSBwcmV2aW91cyB2ZXJzaW9uIG9mIHRoaXMgZnVuY3Rpb24sIGZvciBleGFtcGxlICIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICInaHViOi8vYmF0Y2hfaW5mZXJlbmNlX3YyOjIuNS4wJy4iKQogICAgaWYgbW9kZWxfZW5kcG9pbnRfZHJpZnRfdGhyZXNob2xkOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm5pbmcoIlRoZSBgbW9kZWxfZW5kcG9pbnRfZHJpZnRfdGhyZXNob2xkYCBwYXJhbWV0ZXIgaXMgZGVwcmVjYXRlZCBhbmQgd2lsbCBiZSByZW1vdmVkIG9uY2UgdGhlIHZlcnNpb25pbmcgbWVjaGFuaXNtIGlzIGltcGxlbWVudGVkLiAiCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiaWYgeW91IGFyZSB1c2luZyBtbHJ1bjwxLjcuMCwgcGxlYXNlIGltcG9ydCB0aGUgcHJldmlvdXMgdmVyc2lvbiBvZiB0aGlzIGZ1bmN0aW9uLCBmb3IgZXhhbXBsZSAiCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiJ2h1YjovL2JhdGNoX2luZmVyZW5jZV92MjoyLjUuMCcuIikKICAgIGlmIG1vZGVsX2VuZHBvaW50X3Bvc3NpYmxlX2RyaWZ0X3RocmVzaG9sZDoKICAgICAgICBjb250ZXh0LmxvZ2dlci53YXJuaW5nKCJUaGUgYG1vZGVsX2VuZHBvaW50X3Bvc3NpYmxlX2RyaWZ0X3RocmVzaG9sZGAgcGFyYW1ldGVyIGlzIGRlcHJlY2F0ZWQgYW5kIHdpbGwgYmUgcmVtb3ZlZCBvbmNlIHRoZSB2ZXJzaW9uaW5nIG1lY2hhbmlzbSBpcyBpbXBsZW1lbnRlZC4gIgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImlmIHlvdSBhcmUgdXNpbmcgbWxydW48MS43LjAsIHBsZWFzZSBpbXBvcnQgdGhlIHByZXZpb3VzIHZlcnNpb24gb2YgdGhpcyBmdW5jdGlvbiwgZm9yIGV4YW1wbGUgIgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIidodWI6Ly9iYXRjaF9pbmZlcmVuY2VfdjI6Mi41LjAnLiIpCgogICAgIyBMb2FkaW5nIHRoZSBtb2RlbDoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJMb2FkaW5nIG1vZGVsLi4uIikKICAgIGlmIGlzaW5zdGFuY2UobW9kZWxfcGF0aCwgbWxydW4uRGF0YUl0ZW0pOgogICAgICAgIG1vZGVsX3BhdGggPSBtb2RlbF9wYXRoLmFydGlmYWN0X3VybAogICAgaWYgbm90IG1scnVuLmRhdGFzdG9yZS5pc19zdG9yZV91cmkobW9kZWxfcGF0aCk6CiAgICAgICAgcmFpc2UgbWxydW4uZXJyb3JzLk1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3IoCiAgICAgICAgICAgIGYiVGhlIHByb3ZpZGVkIG1vZGVsIHBhdGggKHttb2RlbF9wYXRofSkgaXMgaW52YWxpZCAtIHNob3VsZCBzdGFydCB3aXRoIGBzdG9yZTovL2AuICIKICAgICAgICAgICAgZiJQbGVhc2UgbWFrZSBzdXJlIHRoYXQgeW91IGhhdmUgbG9nZ2VkIHRoZSBtb2RlbCB1c2luZyBgcHJvamVjdC5sb2dfbW9kZWwoKWAgIgogICAgICAgICAgICBmIndoaWNoIGdlbmVyYXRlcyBhIHVuaXF1ZSBzdG9yZSB1cmkgZm9yIHRoZSBsb2dnZWQgbW9kZWwuIgogICAgICAgICkKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbChtb2RlbF9wYXRoPW1vZGVsX3BhdGgsIGNvbnRleHQ9Y29udGV4dCkKCiAgICBpZiBsYWJlbF9jb2x1bW5zIGlzIE5vbmU6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFsKICAgICAgICAgICAgb3V0cHV0Lm5hbWUgZm9yIG91dHB1dCBpbiBtb2RlbF9oYW5kbGVyLl9tb2RlbF9hcnRpZmFjdC5zcGVjLm91dHB1dHMKICAgICAgICBdCgogICAgaWYgZmVhdHVyZV9jb2x1bW5zIGlzIE5vbmU6CiAgICAgICAgZmVhdHVyZV9jb2x1bW5zID0gWwogICAgICAgICAgICBpbnB1dC5uYW1lIGZvciBpbnB1dCBpbiBtb2RlbF9oYW5kbGVyLl9tb2RlbF9hcnRpZmFjdC5zcGVjLmlucHV0cwogICAgICAgIF0KCiAgICAjIEdldCBkYXRhc2V0IGJ5IG9iamVjdCwgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiTG9hZGluZyBkYXRhLi4uIikKICAgIHgsIGxhYmVsX2NvbHVtbnMgPSBtbHJ1bi5tb2RlbF9tb25pdG9yaW5nLmFwaS5yZWFkX2RhdGFzZXRfYXNfZGF0YWZyYW1lKAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBmZWF0dXJlX2NvbHVtbnM9ZmVhdHVyZV9jb2x1bW5zLAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgUHJlZGljdDoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJDYWxjdWxhdGluZyBwcmVkaWN0aW9uLi4uIikKICAgIHlfcHJlZCA9IG1vZGVsX2hhbmRsZXIubW9kZWwucHJlZGljdCh4LCAqKnByZWRpY3Rfa3dhcmdzKQoKICAgICMgUHJlcGFyZSB0aGUgcmVzdWx0IHNldDoKICAgIHJlc3VsdF9zZXQgPSBfcHJlcGFyZV9yZXN1bHRfc2V0KHg9eCwgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLCB5X3ByZWQ9eV9wcmVkKQoKICAgICMgQ2hlY2sgZm9yIGxvZ2dpbmcgdGhlIHJlc3VsdCBzZXQ6CiAgICBpZiBsb2dfcmVzdWx0X3NldDoKICAgICAgICBtbHJ1bi5tb2RlbF9tb25pdG9yaW5nLmFwaS5sb2dfcmVzdWx0KAogICAgICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgICAgIHJlc3VsdF9zZXRfbmFtZT1yZXN1bHRfc2V0X25hbWUsCiAgICAgICAgICAgIHJlc3VsdF9zZXQ9cmVzdWx0X3NldCwKICAgICAgICAgICAgYXJ0aWZhY3RzX3RhZz1hcnRpZmFjdHNfdGFnLAogICAgICAgICAgICBiYXRjaF9pZD1iYXRjaF9pZCwKICAgICAgICApCgogICAgIyBDaGVjayBmb3IgcGVyZm9ybWluZyBkcmlmdCBhbmFseXNpcwogICAgaWYgKAogICAgICAgICAgICBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzIGlzIE5vbmUKICAgICAgICAgICAgYW5kIG1vZGVsX2hhbmRsZXIuX21vZGVsX2FydGlmYWN0LnNwZWMuZmVhdHVyZV9zdGF0cyBpcyBub3QgTm9uZQogICAgKToKICAgICAgICBwZXJmb3JtX2RyaWZ0X2FuYWx5c2lzID0gVHJ1ZQogICAgaWYgcGVyZm9ybV9kcmlmdF9hbmFseXNpczoKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJQZXJmb3JtaW5nIGRyaWZ0IGFuYWx5c2lzLi4uIikKICAgICAgICAjIEdldCB0aGUgc2FtcGxlIHNldCBzdGF0aXN0aWNzIChlaXRoZXIgZnJvbSB0aGUgc2FtcGxlIHNldCBvciBmcm9tIHRoZSBzdGF0aXN0aWNzIGxvZ2dlZCB3aXRoIHRoZSBtb2RlbCkKICAgICAgICBzdGF0aXN0aWNzX2lucHV0X2ZpbHRlcmVkID0gX2dldF9zYW1wbGVfc2V0X3N0YXRpc3RpY3NfcGFyYW1ldGVycygKICAgICAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgICAgICBtb2RlbF9lbmRwb2ludF9zYW1wbGVfc2V0PW1vZGVsX2VuZHBvaW50X3NhbXBsZV9zZXQsCiAgICAgICAgICAgIG1vZGVsX2FydGlmYWN0X2ZlYXR1cmVfc3RhdHM9bW9kZWxfaGFuZGxlci5fbW9kZWxfYXJ0aWZhY3Quc3BlYy5mZWF0dXJlX3N0YXRzLAogICAgICAgICAgICBmZWF0dXJlX2NvbHVtbnM9ZmVhdHVyZV9jb2x1bW5zLAogICAgICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMpCiAgICAgICAgc2FtcGxlX3NldF9zdGF0aXN0aWNzID0gbWxydW4ubW9kZWxfbW9uaXRvcmluZy5hcGkuZ2V0X3NhbXBsZV9zZXRfc3RhdGlzdGljcygqKnN0YXRpc3RpY3NfaW5wdXRfZmlsdGVyZWQpCiAgICAgICAgbWxydW4ubW9kZWxfbW9uaXRvcmluZy5hcGkucmVjb3JkX3Jlc3VsdHMoCiAgICAgICAgICAgIHByb2plY3Q9Y29udGV4dC5wcm9qZWN0LAogICAgICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgICAgIGVuZHBvaW50X2lkPWVuZHBvaW50X2lkLAogICAgICAgICAgICBtb2RlbF9wYXRoPW1vZGVsX3BhdGgsCiAgICAgICAgICAgIG1vZGVsX2VuZHBvaW50X25hbWU9bW9kZWxfZW5kcG9pbnRfbmFtZSwKICAgICAgICAgICAgaW5mZXJfcmVzdWx0c19kZj1yZXN1bHRfc2V0LmNvcHkoKSwKICAgICAgICAgICAgc2FtcGxlX3NldF9zdGF0aXN0aWNzPXNhbXBsZV9zZXRfc3RhdGlzdGljcywKICAgICAgICAp - allow_empty_resources: true - disable_auto_mount: false - image: mlrun/mlrun description: Batch inference (also knows as prediction) for the common ML frameworks (SciKit-Learn, XGBoost and LightGBM) while performing data drift analysis. -metadata: - tag: '' - categories: - - model-serving - name: batch-inference-v2 -kind: job + default_handler: infer diff --git a/functions/src/batch_inference_v2/item.yaml b/functions/src/batch_inference_v2/item.yaml index 8b8f01df0..62738b1ec 100644 --- a/functions/src/batch_inference_v2/item.yaml +++ b/functions/src/batch_inference_v2/item.yaml @@ -12,7 +12,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.0-rc51 +mlrunVersion: 1.7.0 name: batch_inference_v2 platformVersion: 3.6.0 spec: diff --git a/functions/src/batch_inference_v2/test_batch_inference_v2.py b/functions/src/batch_inference_v2/test_batch_inference_v2.py index 6fa657a0d..e34433076 100644 --- a/functions/src/batch_inference_v2/test_batch_inference_v2.py +++ b/functions/src/batch_inference_v2/test_batch_inference_v2.py @@ -13,25 +13,27 @@ # limitations under the License. # +import datetime import os import pickle +import shutil import time import uuid + +import mlrun +import mlrun.common.schemas import numpy as np import pandas as pd import pytest +from batch_inference_v2 import infer +from mlrun.frameworks.sklearn import apply_mlrun +from mlrun.model_monitoring.api import get_or_create_model_endpoint +from mlrun.projects import get_or_create_project from sklearn.datasets import make_classification -from sklearn.tree import DecisionTreeClassifier -import datetime from sklearn.model_selection import train_test_split +from sklearn.tree import DecisionTreeClassifier from xgboost import XGBClassifier -from mlrun.frameworks.sklearn import apply_mlrun -from mlrun.projects import get_or_create_project -import mlrun -import mlrun.common.schemas -from batch_inference_v2 import infer -import shutil -from mlrun.model_monitoring.api import get_or_create_model_endpoint + REQUIRED_ENV_VARS = [ "MLRUN_DBPATH", "V3IO_USERNAME", @@ -39,6 +41,7 @@ "V3IO_ACCESS_KEY", ] + def _validate_environment_variables() -> bool: """ Checks that all required Environment variables are set. @@ -52,7 +55,7 @@ def generate_data(n_samples: int = 5000, n_features: int = 20): x, y = make_classification(n_samples=n_samples, n_features=n_features, n_classes=2) # Split the data into a training set and a prediction set: - x_train, x_prediction = x[: n_samples // 2], x[n_samples // 2:] + x_train, x_prediction = x[: n_samples // 2], x[n_samples // 2 :] y_train = y[: n_samples // 2] # Randomly drift some features: @@ -86,17 +89,27 @@ def train(training_set: pd.DataFrame): model.fit(training_set, labels) -def assert_batch_predict(n_features, batch_inference_run, with_monitoring=False, project_name="batch-infer-test"): +def assert_batch_predict( + n_features, + batch_inference_run, + with_monitoring=False, + project_name="batch-infer-test", +): # Check the logged results: assert "batch_id" in batch_inference_run.status.results assert len(batch_inference_run.status.artifacts) == 1 - assert len(batch_inference_run.artifact("prediction").as_df().columns) == n_features + 1 + assert ( + len(batch_inference_run.artifact("prediction").as_df().columns) + == n_features + 1 + ) if with_monitoring: # Check that the drift analysis was performed: time.sleep(60) # Retrieve the model endpoint project = get_or_create_project(project_name) - endpoint = get_or_create_model_endpoint(project=project.name, model_endpoint_name="my_cool_endpoint") + endpoint = get_or_create_model_endpoint( + project=project.name, model_endpoint_name="my_cool_endpoint" + ) # Validate that the artifacts were logged in the project artifacts = project.list_artifacts( @@ -119,9 +132,7 @@ def assert_batch_predict(n_features, batch_inference_run, with_monitoring=False, reason="Project's environment variables are not set", ) def test_batch_predict(): - project = get_or_create_project( - "batch-infer-test", context="./", user_project=True - ) + project = get_or_create_project("batch-infer-test", context="./", user_project=True) # Configure test: n_samples = 5000 n_features = 20 @@ -157,19 +168,23 @@ def test_batch_predict(): # Enable model monitoring project.set_model_monitoring_credentials( - endpoint_store_connection="v3io", - tsdb_connection="v3io", - stream_path="v3io") + endpoint_store_connection="v3io", tsdb_connection="v3io", stream_path="v3io" + ) # Deploy model monitoring infrastructure project.enable_model_monitoring(wait_for_deployment=True, base_period=1) # Wait until the monitoring application is triggered import time + time.sleep(60) # Check the logged results: - assert_batch_predict(n_features=n_features, batch_inference_run=batch_inference_run, with_monitoring=True) + assert_batch_predict( + n_features=n_features, + batch_inference_run=batch_inference_run, + with_monitoring=True, + ) # Clean resources _delete_project(project=project.metadata.name) @@ -190,7 +205,9 @@ def setup_method(self): current_datetime = datetime.datetime.now() datetime_str = current_datetime.strftime("%Y%m%d_%H%M%S") mlrun.runtimes.utils.global_context.set(None) - self.context = mlrun.get_or_create_ctx(datetime_str, project=self.project.metadata.name, upload_artifacts=True) + self.context = mlrun.get_or_create_ctx( + datetime_str, project=self.project.metadata.name, upload_artifacts=True + ) self.context.artifact_path = self.infer_artifact_path def teardown_method(self): @@ -209,43 +226,70 @@ def _get_model_endpoint_sample_set(self, sample_type, n_features: int = 20): elif sample_type == list: return data.values.tolist() elif sample_type == dict: - return data.to_dict(orient='list') + return data.to_dict(orient="list") elif sample_type == pd.DataFrame: return data elif sample_type == np.ndarray: return data.values - @pytest.mark.parametrize("sample_type", [mlrun.DataItem, list, dict, pd.DataFrame, np.ndarray]) + @pytest.mark.parametrize( + "sample_type", [mlrun.DataItem, list, dict, pd.DataFrame, np.ndarray] + ) def test_infer_sample_types(self, sample_type): n_features = 10 training_set, prediction_set = generate_data(n_features=n_features) - clf = XGBClassifier(n_estimators=2, max_depth=2, learning_rate=1, objective="binary:logistic") - x, y = prediction_set, training_set['target_label'] - x_train, x_test, y_train, y_test = train_test_split(x, y, train_size=0.8, test_size=0.2, random_state=0) + clf = XGBClassifier( + n_estimators=2, max_depth=2, learning_rate=1, objective="binary:logistic" + ) + x, y = prediction_set, training_set["target_label"] + x_train, x_test, y_train, y_test = train_test_split( + x, y, train_size=0.8, test_size=0.2, random_state=0 + ) clf.fit(x_train, y_train) train_set_to_log = x_train.join(y_train) - model = self.project.log_model(f"model-{uuid.uuid4()}", body=pickle.dumps(clf), - model_file=f"model-{uuid.uuid4()}.pkl", framework="xgboost", - training_set=train_set_to_log, label_column="target_label") + model = self.project.log_model( + f"model-{uuid.uuid4()}", + body=pickle.dumps(clf), + model_file=f"model-{uuid.uuid4()}.pkl", + framework="xgboost", + training_set=train_set_to_log, + label_column="target_label", + ) dataset = self.project.log_dataset(f"dataset-{uuid.uuid4()}", df=x_test) z_test = train_set_to_log * 5 - model_endpoint_sample_set = self.project.log_dataset(f"model-endpoint-sample-set{uuid.uuid4()}", df=z_test) + model_endpoint_sample_set = self.project.log_dataset( + f"model-endpoint-sample-set{uuid.uuid4()}", df=z_test + ) sample = self._get_model_endpoint_sample_set( - sample_type=sample_type, n_features=n_features) - infer(context=self.context, - dataset=dataset.to_dataitem().as_df(), model_path=model.uri, - model_endpoint_sample_set=sample, - feature_columns=list(model_endpoint_sample_set.to_dataitem().as_df().columns), - label_columns="target_label", - model_endpoint_name=f"model-endpoint-name-{uuid.uuid4()}", - trigger_monitoring_job=True, - perform_drift_analysis=True) + sample_type=sample_type, n_features=n_features + ) + infer( + context=self.context, + dataset=dataset.to_dataitem().as_df(), + model_path=model.uri, + model_endpoint_sample_set=sample, + feature_columns=list( + model_endpoint_sample_set.to_dataitem().as_df().columns + ), + label_columns="target_label", + model_endpoint_name=f"model-endpoint-name-{uuid.uuid4()}", + trigger_monitoring_job=True, + perform_drift_analysis=True, + ) # a workaround until ML-4636 will be solved. - batch_inference_run = self.project.list_runs(name=self.context.name).to_objects()[0] - mlrun.get_run_db().update_run(updates={"status.state": "completed"}, uid=batch_inference_run.uid()) - assert_batch_predict(n_features=n_features, batch_inference_run=batch_inference_run, project_name=self.project_name) + batch_inference_run = self.project.list_runs( + name=self.context.name + ).to_objects()[0] + mlrun.get_run_db().update_run( + updates={"status.state": "completed"}, uid=batch_inference_run.uid() + ) + assert_batch_predict( + n_features=n_features, + batch_inference_run=batch_inference_run, + project_name=self.project_name, + ) def _delete_project(project: str): diff --git a/functions/src/describe/describe.py b/functions/src/describe/describe.py index 27d789f5b..ac8a744dc 100644 --- a/functions/src/describe/describe.py +++ b/functions/src/describe/describe.py @@ -15,7 +15,6 @@ # Generated by nuclio.export.NuclioExporter import warnings -from typing import Union import mlrun import numpy as np @@ -46,7 +45,7 @@ def analyze( context: MLClientCtx, name: str = "dataset", - table: Union[FeatureSet, DataItem] = None, + table: FeatureSet | DataItem = None, label_column: str = None, plots_dest: str = "plots", random_state: int = 1, @@ -129,7 +128,7 @@ def analyze( ) df = feature_set.to_dataframe() else: - context.logger.error(f"Wrong table type.") + context.logger.error("Wrong table type.") return if df.size > MAX_SIZE_OF_DF: @@ -320,8 +319,8 @@ def _create_features_histogram_artifacts( ) fig.update_layout(title_text=f"Histograms of {first_feature_name}") - extra_data[f"histograms"] = context.log_artifact( - PlotlyArtifact(key=f"histograms", figure=fig), + extra_data["histograms"] = context.log_artifact( + PlotlyArtifact(key="histograms", figure=fig), local_path=f"{plots_dest}/histograms.html", ) @@ -431,9 +430,9 @@ def _create_features_2d_scatter_artifacts( template="plotly_white", ) - fig.update_layout(title_text=f"Scatter-2d") - extra_data[f"scatter-2d"] = context.log_artifact( - PlotlyArtifact(key=f"scatter-2d", figure=fig), + fig.update_layout(title_text="Scatter-2d") + extra_data["scatter-2d"] = context.log_artifact( + PlotlyArtifact(key="scatter-2d", figure=fig), local_path=f"{plots_dest}/scatter-2d.html", ) @@ -540,7 +539,7 @@ def _create_corr_artifact( ) z = tblcorr.values.tolist() - z_text = [["{:.2f}".format(y) for y in x] for x in z] + z_text = [[f"{y:.2f}" for y in x] for x in z] fig = ff.create_annotated_heatmap( z, x=list(tblcorr.columns), diff --git a/functions/src/describe/function.yaml b/functions/src/describe/function.yaml index a11461774..679eea213 100644 --- a/functions/src/describe/function.yaml +++ b/functions/src/describe/function.yaml @@ -1,7 +1,20 @@ +metadata: + tag: '' + name: describe + categories: + - data-analysis +verbose: false +kind: job spec: + image: mlrun/mlrun + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgppbXBvcnQgbWxydW4KaW1wb3J0IG51bXB5IGFzIG5wCgp3YXJuaW5ncy5zaW1wbGVmaWx0ZXIoYWN0aW9uPSJpZ25vcmUiLCBjYXRlZ29yeT1GdXR1cmVXYXJuaW5nKQoKaW1wb3J0IG1scnVuLmZlYXR1cmVfc3RvcmUgYXMgZnN0b3JlCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IHBsb3RseS5leHByZXNzIGFzIHB4CmltcG9ydCBwbG90bHkuZmlndXJlX2ZhY3RvcnkgYXMgZmYKaW1wb3J0IHBsb3RseS5ncmFwaF9vYmplY3RzIGFzIGdvCmZyb20gbWxydW4uYXJ0aWZhY3RzIGltcG9ydCAoCiAgICBBcnRpZmFjdCwKICAgIERhdGFzZXRBcnRpZmFjdCwKICAgIFBsb3RseUFydGlmYWN0LAogICAgVGFibGVBcnRpZmFjdCwKICAgIHVwZGF0ZV9kYXRhc2V0X21ldGEsCikKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZlYXR1cmVfc3RvcmUgaW1wb3J0IEZlYXR1cmVTZXQKZnJvbSBwbG90bHkuc3VicGxvdHMgaW1wb3J0IG1ha2Vfc3VicGxvdHMKCnBkLnNldF9vcHRpb24oImRpc3BsYXkuZmxvYXRfZm9ybWF0IiwgbGFtYmRhIHg6ICIlLjJmIiAlIHgpCk1BWF9TSVpFX09GX0RGID0gNTAwMDAwCgoKZGVmIGFuYWx5emUoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG5hbWU6IHN0ciA9ICJkYXRhc2V0IiwKICAgIHRhYmxlOiBGZWF0dXJlU2V0IHwgRGF0YUl0ZW0gPSBOb25lLAogICAgbGFiZWxfY29sdW1uOiBzdHIgPSBOb25lLAogICAgcGxvdHNfZGVzdDogc3RyID0gInBsb3RzIiwKICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gMSwKICAgIHByb2JsZW1fdHlwZTogc3RyID0gImNsYXNzaWZpY2F0aW9uIiwKICAgIGRhc2tfa2V5OiBzdHIgPSAiZGFza19rZXkiLAogICAgZGFza19mdW5jdGlvbjogc3RyID0gTm9uZSwKICAgIGRhc2tfY2xpZW50PU5vbmUsCikgLT4gTm9uZToKICAgICIiIgogICAgVGhlIGZ1bmN0aW9uIHdpbGwgb3V0cHV0IHRoZSBmb2xsb3dpbmcgYXJ0aWZhY3RzIHBlcgogICAgY29sdW1uIHdpdGhpbiB0aGUgZGF0YSBmcmFtZSAoYmFzZWQgb24gZGF0YSB0eXBlcykKICAgIElmIHRoZSBkYXRhIGhhcyBtb3JlIHRoYW4gNTAwLDAwMCBzYW1wbGUgd2UKICAgIHNhbXBsZSByYW5kb21seSA1MDAsMDAwIHNhbXBsZXM6CgogICAgZGVzY3JpYmUgY3N2CiAgICBoaXN0b2dyYW1zCiAgICBzY2F0dGVyLTJkCiAgICB2aW9saW4gY2hhcnQKICAgIGNvcnJlbGF0aW9uLW1hdHJpeCBjaGFydAogICAgY29ycmVsYXRpb24tbWF0cml4IGNzdgogICAgaW1iYWxhbmNlIHBpZSBjaGFydAogICAgaW1iYWxhbmNlLXdlaWdodHMtdmVjIGNzdgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgVGhlIGZ1bmN0aW9uIGNvbnRleHQKICAgIDpwYXJhbSBuYW1lOiAgICAgICAgICAgICAgICAgICAgS2V5IG9mIGRhdGFzZXQgdG8gZGF0YWJhc2UgKCJkYXRhc2V0IiBmb3IgZGVmYXVsdCkKICAgIDpwYXJhbSB0YWJsZTogICAgICAgICAgICAgICAgICAgTUxSdW4gaW5wdXQgcG9pbnRpbmcgdG8gcGFuZGFzIGRhdGFmcmFtZSAoY3N2L3BhcnF1ZXQgZmlsZSBwYXRoKSBvciBGZWF0dXJlU2V0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGFzIHBhcmFtCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uOiAgICAgICAgICAgIEdyb3VuZCB0cnV0aCBjb2x1bW4gbGFiZWwKICAgIDpwYXJhbSBwbG90c19kZXN0OiAgICAgICAgICAgICAgRGVzdGluYXRpb24gZm9sZGVyIG9mIHN1bW1hcnkgcGxvdHMgKHJlbGF0aXZlIHRvIGFydGlmYWN0X3BhdGgpCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICgicGxvdHMiIGZvciBkZWZhdWx0KQogICAgOnBhcmFtIHJhbmRvbV9zdGF0ZTogICAgICAgICAgICBXaGVuIHRoZSB0YWJsZSBoYXMgbW9yZSB0aGFuIDUwMCwwMDAgc2FtcGxlcywgd2Ugc2FtcGxlIHJhbmRvbWx5IDUwMCwwMDAgc2FtcGxlcwogICAgOnBhcmFtIHByb2JsZW1fdHlwZSAgICAgICAgICAgICBUaGUgdHlwZSBvZiB0aGUgTUwgcHJvYmxlbSB0aGUgZGF0YSBmYWNpbmcgLSByZWdyZXNzaW9uLCBjbGFzc2lmaWNhdGlvbiBvciBOb25lCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChjbGFzc2lmaWNhdGlvbiBmb3IgZGVmYXVsdCkKICAgIDpwYXJhbSBkYXNrX2tleTogICAgICAgICAgICAgICAgS2V5IG9mIGRhdGFmcmFtZSBpbiBkYXNrIGNsaWVudCAiZGF0YXNldHMiIGF0dHJpYnV0ZQogICAgOnBhcmFtIGRhc2tfZnVuY3Rpb246ICAgICAgICAgICBEYXNrIGZ1bmN0aW9uIHVybCAoZGI6Ly8uLikKICAgIDpwYXJhbSBkYXNrX2NsaWVudDogICAgICAgICAgICAgRGFzayBjbGllbnQgb2JqZWN0CiAgICAiIiIKICAgIGRhdGFfaXRlbSwgZmVhdHVyZXNldCwgY3JlYXQsIHVwZGF0ZSA9IEZhbHNlLCBGYWxzZSwgRmFsc2UsIEZhbHNlCiAgICBnZXRfZnJvbV90YWJsZSA9IFRydWUKICAgIGlmIGRhc2tfZnVuY3Rpb24gb3IgZGFza19jbGllbnQ6CiAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCA9IFRydWUsIFRydWUKICAgICAgICBpZiBkYXNrX2Z1bmN0aW9uOgogICAgICAgICAgICBjbGllbnQgPSBtbHJ1bi5pbXBvcnRfZnVuY3Rpb24oZGFza19mdW5jdGlvbikuY2xpZW50CiAgICAgICAgZWxpZiBkYXNrX2NsaWVudDoKICAgICAgICAgICAgY2xpZW50ID0gZGFza19jbGllbnQKICAgICAgICBlbHNlOgogICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJkYXNrIGNsaWVudCB3YXMgbm90IHByb3ZpZGVkIikKCiAgICAgICAgaWYgZGFza19rZXkgaW4gY2xpZW50LmRhdGFzZXRzOgogICAgICAgICAgICBkZiA9IGNsaWVudC5nZXRfZGF0YXNldChkYXNrX2tleSkKICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgZ2V0X2Zyb21fdGFibGUgPSBUcnVlLCBUcnVlLCBGYWxzZQogICAgICAgIGVsaWYgdGFibGU6CiAgICAgICAgICAgIGdldF9mcm9tX3RhYmxlID0gVHJ1ZQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICBmIm9ubHkgdGhlc2UgZGF0YXNldHMgYXJlIGF2YWlsYWJsZSB7Y2xpZW50LmRhdGFzZXRzfSBpbiBjbGllbnQge2NsaWVudH0iCiAgICAgICAgICAgICkKICAgICAgICAgICAgcmFpc2UgRXhjZXB0aW9uKCJkYXRhc2V0IG5vdCBmb3VuZCBvbiBkYXNrIGNsdXN0ZXIiKQoKICAgIGlmIGdldF9mcm9tX3RhYmxlOgogICAgICAgIGlmIHR5cGUodGFibGUpID09IERhdGFJdGVtOgogICAgICAgICAgICBpZiB0YWJsZS5tZXRhIGlzIE5vbmU6CiAgICAgICAgICAgICAgICBkYXRhX2l0ZW0sIGNyZWF0LCB1cGRhdGUgPSBUcnVlLCBUcnVlLCBGYWxzZQogICAgICAgICAgICBlbGlmIHRhYmxlLm1ldGEua2luZCA9PSAiZGF0YXNldCI6CiAgICAgICAgICAgICAgICBkYXRhX2l0ZW0sIGNyZWF0LCB1cGRhdGUgPSBUcnVlLCBGYWxzZSwgVHJ1ZQogICAgICAgICAgICBlbGlmIHRhYmxlLm1ldGEua2luZCA9PSAiRmVhdHVyZVZlY3RvciI6CiAgICAgICAgICAgICAgICBkYXRhX2l0ZW0sIGNyZWF0LCB1cGRhdGUgPSBUcnVlLCBGYWxzZSwgRmFsc2UKICAgICAgICAgICAgZWxpZiB0YWJsZS5tZXRhLmtpbmQgPT0gIkZlYXR1cmVTZXQiOgogICAgICAgICAgICAgICAgZmVhdHVyZXNldCwgY3JlYXQsIHVwZGF0ZSA9IFRydWUsIEZhbHNlLCBGYWxzZQoKICAgICAgICBpZiBkYXRhX2l0ZW06CiAgICAgICAgICAgIGRmID0gdGFibGUuYXNfZGYoKQogICAgICAgIGVsaWYgZmVhdHVyZXNldDoKICAgICAgICAgICAgcHJvamVjdF9uYW1lLCBzZXRfbmFtZSA9ICgKICAgICAgICAgICAgICAgIHRhYmxlLl9wYXRoLnNwbGl0KCIvIilbMl0sCiAgICAgICAgICAgICAgICB0YWJsZS5fcGF0aC5zcGxpdCgiLyIpWzRdLAogICAgICAgICAgICApCiAgICAgICAgICAgIGZlYXR1cmVfc2V0ID0gZnN0b3JlLmdldF9mZWF0dXJlX3NldCgKICAgICAgICAgICAgICAgIGYic3RvcmU6Ly9mZWF0dXJlLXNldHMve3Byb2plY3RfbmFtZX0ve3NldF9uYW1lfSIKICAgICAgICAgICAgKQogICAgICAgICAgICBkZiA9IGZlYXR1cmVfc2V0LnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoIldyb25nIHRhYmxlIHR5cGUuIikKICAgICAgICAgICAgcmV0dXJuCgogICAgaWYgZGYuc2l6ZSA+IE1BWF9TSVpFX09GX0RGOgogICAgICAgIGRmID0gZGYuc2FtcGxlKG49aW50KE1BWF9TSVpFX09GX0RGIC8gZGYuc2hhcGVbMV0pLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlKQogICAgZXh0cmFfZGF0YSA9IHt9CgogICAgaWYgbGFiZWxfY29sdW1uIG5vdCBpbiBkZi5jb2x1bW5zOgogICAgICAgIGxhYmVsX2NvbHVtbiA9IE5vbmUKCiAgICBleHRyYV9kYXRhWyJkZXNjcmliZSBjc3YiXSA9IGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgIFRhYmxlQXJ0aWZhY3QoImRlc2NyaWJlLWNzdiIsIGRmPWRmLmRlc2NyaWJlKCkpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vZGVzY3JpYmUuY3N2IiwKICAgICkKCiAgICB0cnk6CiAgICAgICAgX2NyZWF0ZV9oaXN0b2dyYW1fbWF0X2FydGlmYWN0KAogICAgICAgICAgICBjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0CiAgICAgICAgKQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIGhpc3RvZ3JhbSBtYXRyaXggYXJ0aWZhY3QgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfZmVhdHVyZXNfaGlzdG9ncmFtX2FydGlmYWN0cygKICAgICAgICAgICAgY29udGV4dCwgZGYsIGV4dHJhX2RhdGEsIGxhYmVsX2NvbHVtbiwgcGxvdHNfZGVzdCwgcHJvYmxlbV90eXBlCiAgICAgICAgKQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIHBhaXJwbG90IGhpc3RvZ3JhbXMgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfZmVhdHVyZXNfMmRfc2NhdHRlcl9hcnRpZmFjdHMoCiAgICAgICAgICAgIGNvbnRleHQsIGRmLCBleHRyYV9kYXRhLCBsYWJlbF9jb2x1bW4sIHBsb3RzX2Rlc3QsIHByb2JsZW1fdHlwZQogICAgICAgICkKICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICBjb250ZXh0LmxvZ2dlci53YXJuKGYiRmFpbGVkIHRvIGNyZWF0ZSBwYWlycGxvdCAyZF9zY2F0dGVyIGR1ZSB0bzoge2V9IikKICAgIHRyeToKICAgICAgICBfY3JlYXRlX3Zpb2xpbl9hcnRpZmFjdChjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgcGxvdHNfZGVzdCkKICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICBjb250ZXh0LmxvZ2dlci53YXJuKGYiRmFpbGVkIHRvIGNyZWF0ZSB2aW9saW4gZGlzdHJpYnV0aW9uIHBsb3RzIGR1ZSB0bzoge2V9IikKICAgIHRyeToKICAgICAgICBfY3JlYXRlX2ltYmFsYW5jZV9hcnRpZmFjdCgKICAgICAgICAgICAgY29udGV4dCwgZGYsIGV4dHJhX2RhdGEsIGxhYmVsX2NvbHVtbiwgcGxvdHNfZGVzdCwgcHJvYmxlbV90eXBlCiAgICAgICAgKQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIGNsYXNzIGltYmFsYW5jZSBwbG90IGR1ZSB0bzoge2V9IikKICAgIHRyeToKICAgICAgICBfY3JlYXRlX2NvcnJfYXJ0aWZhY3QoY29udGV4dCwgZGYsIGV4dHJhX2RhdGEsIGxhYmVsX2NvbHVtbiwgcGxvdHNfZGVzdCkKICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICBjb250ZXh0LmxvZ2dlci53YXJuKGYiRmFpbGVkIHRvIGNyZWF0ZSBmZWF0dXJlcyBjb3JyZWxhdGlvbiBwbG90IGR1ZSB0bzoge2V9IikKCiAgICBpZiBub3QgZGF0YV9pdGVtOgogICAgICAgIHJldHVybgoKICAgIGFydGlmYWN0ID0gdGFibGUuYXJ0aWZhY3RfdXJsCiAgICBpZiBjcmVhdDogICMgZGF0YXNldCBub3Qgc3RvcmVkCiAgICAgICAgYXJ0aWZhY3QgPSBEYXRhc2V0QXJ0aWZhY3QoCiAgICAgICAgICAgIGtleT0iZGF0YXNldCIsIHN0YXRzPVRydWUsIGRmPWRmLCBleHRyYV9kYXRhPWV4dHJhX2RhdGEKICAgICAgICApCiAgICAgICAgYXJ0aWZhY3QgPSBjb250ZXh0LmxvZ19hcnRpZmFjdChhcnRpZmFjdCwgZGJfa2V5PW5hbWUpCiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIlRoZSBkYXRhIHNldCBpcyBsb2dnZWQgdG8gdGhlIHByb2plY3QgdW5kZXIge25hbWV9IG5hbWUiKQoKICAgIGlmIHVwZGF0ZToKICAgICAgICB1cGRhdGVfZGF0YXNldF9tZXRhKGFydGlmYWN0LCBleHRyYV9kYXRhPWV4dHJhX2RhdGEpCiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIlRoZSBkYXRhIHNldCBuYW1lZCB7bmFtZX0gaXMgdXBkYXRlZCIpCgogICAgIyBUT0RPIDogMy1EIHBsb3Qgb24gb24gc2VsZWN0ZWQgZmVhdHVyZXMuCiAgICAjIFRPRE8gOiBSZWludGVncmF0aW9uIHBsb3Qgb24gb24gc2VsZWN0ZWQgZmVhdHVyZXMuCiAgICAjIFRPRE8gOiBQQ0EgcGxvdCAod2l0aCBvcHRpb25zKQoKCmRlZiBfY3JlYXRlX2hpc3RvZ3JhbV9tYXRfYXJ0aWZhY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIGRmOiBwZC5EYXRhRnJhbWUsCiAgICBleHRyYV9kYXRhOiBkaWN0LAogICAgbGFiZWxfY29sdW1uOiBzdHIsCiAgICBwbG90c19kZXN0OiBzdHIsCik6CiAgICAiIiIKICAgIENyZWF0ZSBhbmQgbG9nIGEgaGlzdG9ncmFtIG1hdHJpeCBhcnRpZmFjdAogICAgIiIiCiAgICBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBpdGVtPUFydGlmYWN0KAogICAgICAgICAgICBrZXk9Imhpc3QiLAogICAgICAgICAgICBib2R5PWIiPGI+IERlcHJlY2F0ZWQsIHNlZSB0aGUgYXJ0aWZhY3RzIHNjYXR0ZXItMmQgIgogICAgICAgICAgICBiImFuZCBoaXN0b2dyYW1zIGluc3RlYWQ8Yj4iLAogICAgICAgICksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9oaXN0Lmh0bWwiLAogICAgKQoKCmRlZiBfY3JlYXRlX2ZlYXR1cmVzX2hpc3RvZ3JhbV9hcnRpZmFjdHMoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIGRmOiBwZC5EYXRhRnJhbWUsCiAgICBleHRyYV9kYXRhOiBkaWN0LAogICAgbGFiZWxfY29sdW1uOiBzdHIsCiAgICBwbG90c19kZXN0OiBzdHIsCiAgICBwcm9ibGVtX3R5cGU6IHN0ciwKKToKICAgICIiIgogICAgQ3JlYXRlIGFuZCBsb2cgYSBoaXN0b2dyYW0gYXJ0aWZhY3QgZm9yIGVhY2ggZmVhdHVyZQogICAgIiIiCgogICAgZmlncyA9IGRpY3QoKQogICAgZmlyc3RfZmVhdHVyZV9uYW1lID0gIiIKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZSBhbmQgcHJvYmxlbV90eXBlID09ICJjbGFzc2lmaWNhdGlvbiI6CiAgICAgICAgYWxsX2xhYmVscyA9IGRmW2xhYmVsX2NvbHVtbl0udW5pcXVlKCkKICAgIHZpc2libGUgPSBUcnVlCiAgICBmb3IgY29sdW1uX25hbWUgaW4gZGYuY29sdW1uczoKICAgICAgICBpZiBjb2x1bW5fbmFtZSA9PSBsYWJlbF9jb2x1bW46CiAgICAgICAgICAgIGNvbnRpbnVlCgogICAgICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZSBhbmQgcHJvYmxlbV90eXBlID09ICJjbGFzc2lmaWNhdGlvbiI6CiAgICAgICAgICAgIGZvciBsYWJlbCBpbiBhbGxfbGFiZWxzOgogICAgICAgICAgICAgICAgc3ViX2ZpZyA9IGdvLkhpc3RvZ3JhbSgKICAgICAgICAgICAgICAgICAgICBoaXN0ZnVuYz0iY291bnQiLAogICAgICAgICAgICAgICAgICAgIHg9ZGYubG9jW2RmW2xhYmVsX2NvbHVtbl0gPT0gbGFiZWxdW2NvbHVtbl9uYW1lXSwKICAgICAgICAgICAgICAgICAgICBuYW1lPXN0cihsYWJlbCksCiAgICAgICAgICAgICAgICAgICAgdmlzaWJsZT12aXNpYmxlLAogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZmlnc1tmIntjb2x1bW5fbmFtZX1AP0B7bGFiZWx9Il0gPSBzdWJfZmlnCiAgICAgICAgZWxzZToKICAgICAgICAgICAgc3ViX2ZpZyA9IGdvLkhpc3RvZ3JhbShoaXN0ZnVuYz0iY291bnQiLCB4PWRmW2NvbHVtbl9uYW1lXSwgdmlzaWJsZT12aXNpYmxlKQogICAgICAgICAgICBmaWdzW2Yie2NvbHVtbl9uYW1lfUA/QHsxfSJdID0gc3ViX2ZpZwogICAgICAgIGlmIHZpc2libGU6CiAgICAgICAgICAgIGZpcnN0X2ZlYXR1cmVfbmFtZSA9IGNvbHVtbl9uYW1lCiAgICAgICAgdmlzaWJsZSA9IEZhbHNlCgogICAgZmlnID0gZ28uRmlndXJlKCkKICAgIGZvciBrIGluIGZpZ3Mua2V5cygpOgogICAgICAgIGZpZy5hZGRfdHJhY2UoZmlnc1trXSkKCiAgICBmaWcudXBkYXRlX2xheW91dCgKICAgICAgICB1cGRhdGVtZW51cz1bCiAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICJidXR0b25zIjogWwogICAgICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAgICAgImxhYmVsIjogY29sdW1uX25hbWUsCiAgICAgICAgICAgICAgICAgICAgICAgICJtZXRob2QiOiAidXBkYXRlIiwKICAgICAgICAgICAgICAgICAgICAgICAgImFyZ3MiOiBbCiAgICAgICAgICAgICAgICAgICAgICAgICAgICB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInZpc2libGUiOiBbCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGtleS5zcGxpdCgiQD9AIilbMF0gPT0gY29sdW1uX25hbWUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZm9yIGtleSBpbiBmaWdzLmtleXMoKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgInhheGlzIjogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAicmFuZ2UiOiBbCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtaW4oZGZbY29sdW1uX25hbWVdKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1heChkZltjb2x1bW5fbmFtZV0pLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICB7InRpdGxlIjogZiI8aT48Yj5IaXN0b2dyYW0gb2Yge2NvbHVtbl9uYW1lfTwvYj48L2k+In0sCiAgICAgICAgICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgIGZvciBjb2x1bW5fbmFtZSBpbiBkZi5jb2x1bW5zCiAgICAgICAgICAgICAgICAgICAgaWYgY29sdW1uX25hbWUgIT0gbGFiZWxfY29sdW1uCiAgICAgICAgICAgICAgICBdLAogICAgICAgICAgICAgICAgImRpcmVjdGlvbiI6ICJkb3duIiwKICAgICAgICAgICAgICAgICJwYWQiOiB7InIiOiAxMCwgInQiOiAxMH0sCiAgICAgICAgICAgICAgICAic2hvd2FjdGl2ZSI6IFRydWUsCiAgICAgICAgICAgICAgICAieCI6IDAuMjUsCiAgICAgICAgICAgICAgICAieGFuY2hvciI6ICJsZWZ0IiwKICAgICAgICAgICAgICAgICJ5IjogMS4xLAogICAgICAgICAgICAgICAgInlhbmNob3IiOiAidG9wIiwKICAgICAgICAgICAgfQogICAgICAgIF0sCiAgICAgICAgYW5ub3RhdGlvbnM9WwogICAgICAgICAgICBkaWN0KAogICAgICAgICAgICAgICAgdGV4dD0iU2VsZWN0IEZlYXR1cmUgTmFtZSAiLAogICAgICAgICAgICAgICAgc2hvd2Fycm93PUZhbHNlLAogICAgICAgICAgICAgICAgeD0wLAogICAgICAgICAgICAgICAgeT0xLjA1LAogICAgICAgICAgICAgICAgeXJlZj0icGFwZXIiLAogICAgICAgICAgICAgICAgeHJlZj0icGFwZXIiLAogICAgICAgICAgICAgICAgYWxpZ249ImxlZnQiLAogICAgICAgICAgICAgICAgeGFuY2hvcj0ibGVmdCIsCiAgICAgICAgICAgICAgICB5YW5jaG9yPSJ0b3AiLAogICAgICAgICAgICAgICAgZm9udD17CiAgICAgICAgICAgICAgICAgICAgImNvbG9yIjogImJsdWUiLAogICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgKQogICAgICAgIF0sCiAgICApCgogICAgZmlnLnVwZGF0ZV9sYXlvdXQoCiAgICAgICAgd2lkdGg9NjAwLAogICAgICAgIGhlaWdodD00MDAsCiAgICAgICAgYXV0b3NpemU9RmFsc2UsCiAgICAgICAgbWFyZ2luPWRpY3QodD0xMDAsIGI9MCwgbD0wLCByPTApLAogICAgICAgIHRlbXBsYXRlPSJwbG90bHlfd2hpdGUiLAogICAgKQoKICAgIGZpZy51cGRhdGVfbGF5b3V0KHRpdGxlX3RleHQ9ZiI8aT48Yj5IaXN0b2dyYW1zIG9mIHtmaXJzdF9mZWF0dXJlX25hbWV9PC9iPjwvaT4iKQogICAgZXh0cmFfZGF0YVsiaGlzdG9ncmFtcyJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJoaXN0b2dyYW1zIiwgZmlndXJlPWZpZyksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9oaXN0b2dyYW1zLmh0bWwiLAogICAgKQoKCmRlZiBfY3JlYXRlX2ZlYXR1cmVzXzJkX3NjYXR0ZXJfYXJ0aWZhY3RzKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkZjogcGQuRGF0YUZyYW1lLAogICAgZXh0cmFfZGF0YTogZGljdCwKICAgIGxhYmVsX2NvbHVtbjogc3RyLAogICAgcGxvdHNfZGVzdDogc3RyLAogICAgcHJvYmxlbV90eXBlOiBzdHIsCik6CiAgICAiIiIKICAgIENyZWF0ZSBhbmQgbG9nIGEgc2NhdHRlci0yZCBhcnRpZmFjdCBmb3IgZWFjaCBjb3VwbGUgb2YgZmVhdHVyZXMKICAgICIiIgogICAgZmVhdHVyZXMgPSBbCiAgICAgICAgY29sdW1uX25hbWUgZm9yIGNvbHVtbl9uYW1lIGluIGRmLmNvbHVtbnMgaWYgY29sdW1uX25hbWUgIT0gbGFiZWxfY29sdW1uCiAgICBdCiAgICBtYXhfZmVhdHVyZV9sZW4gPSBmbG9hdChtYXgobGVuKGVsZW0pIGZvciBlbGVtIGluIGZlYXR1cmVzKSkKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZToKICAgICAgICBsYWJlbHMgPSBzb3J0ZWQoZGZbbGFiZWxfY29sdW1uXS51bmlxdWUoKSkKICAgIGVsc2U6CiAgICAgICAgbGFiZWxzID0gW05vbmVdCiAgICBmaWcgPSBnby5GaWd1cmUoKQogICAgaWYgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lIGFuZCBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICBmb3IgbCBpbiBsYWJlbHM6CiAgICAgICAgICAgIGZpZy5hZGRfdHJhY2UoCiAgICAgICAgICAgICAgICBnby5TY2F0dGVyKAogICAgICAgICAgICAgICAgICAgIHg9ZGYubG9jW2RmW2xhYmVsX2NvbHVtbl0gPT0gbF1bZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgICAgIHk9ZGYubG9jW2RmW2xhYmVsX2NvbHVtbl0gPT0gbF1bZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgICAgIG1vZGU9Im1hcmtlcnMiLAogICAgICAgICAgICAgICAgICAgIHZpc2libGU9VHJ1ZSwKICAgICAgICAgICAgICAgICAgICBzaG93bGVnZW5kPVRydWUsCiAgICAgICAgICAgICAgICAgICAgbmFtZT1zdHIobCksCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICkKICAgIGVsaWYgbGFiZWxfY29sdW1uIGlzIE5vbmU6CiAgICAgICAgZmlnLmFkZF90cmFjZSgKICAgICAgICAgICAgZ28uU2NhdHRlcigKICAgICAgICAgICAgICAgIHg9ZGZbZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgeT1kZltmZWF0dXJlc1swXV0sCiAgICAgICAgICAgICAgICBtb2RlPSJtYXJrZXJzIiwKICAgICAgICAgICAgICAgIHZpc2libGU9VHJ1ZSwKICAgICAgICAgICAgKQogICAgICAgICkKICAgIGVsaWYgcHJvYmxlbV90eXBlID09ICJyZWdyZXNzaW9uIjoKICAgICAgICBmaWcuYWRkX3RyYWNlKAogICAgICAgICAgICBnby5TY2F0dGVyKAogICAgICAgICAgICAgICAgeD1kZltmZWF0dXJlc1swXV0sCiAgICAgICAgICAgICAgICB5PWRmW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgIG1vZGU9Im1hcmtlcnMiLAogICAgICAgICAgICAgICAgbWFya2VyPWRpY3QoCiAgICAgICAgICAgICAgICAgICAgY29sb3I9ZGZbbGFiZWxfY29sdW1uXSwgY29sb3JzY2FsZT0iVmlyaWRpcyIsIHNob3dzY2FsZT1UcnVlCiAgICAgICAgICAgICAgICApLAogICAgICAgICAgICAgICAgdmlzaWJsZT1UcnVlLAogICAgICAgICAgICApCiAgICAgICAgKQoKICAgIHhfYnV0dG9ucyA9IFtdCiAgICB5X2J1dHRvbnMgPSBbXQoKICAgIGZvciBuY29sIGluIGZlYXR1cmVzOgogICAgICAgIGlmIHByb2JsZW1fdHlwZSA9PSAiY2xhc3NpZmljYXRpb24iIGFuZCBsYWJlbF9jb2x1bW4gaXMgbm90IE5vbmU6CiAgICAgICAgICAgIHhfYnV0dG9ucy5hcHBlbmQoCiAgICAgICAgICAgICAgICBkaWN0KAogICAgICAgICAgICAgICAgICAgIG1ldGhvZD0idXBkYXRlIiwKICAgICAgICAgICAgICAgICAgICBsYWJlbD1uY29sLAogICAgICAgICAgICAgICAgICAgIGFyZ3M9WwogICAgICAgICAgICAgICAgICAgICAgICB7IngiOiBbZGYubG9jW2RmW2xhYmVsX2NvbHVtbl0gPT0gbF1bbmNvbF0gZm9yIGwgaW4gbGFiZWxzXX0sCiAgICAgICAgICAgICAgICAgICAgICAgIG5wLmFyYW5nZShsZW4obGFiZWxzKSkudG9saXN0KCksCiAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgKQoKICAgICAgICAgICAgeV9idXR0b25zLmFwcGVuZCgKICAgICAgICAgICAgICAgIGRpY3QoCiAgICAgICAgICAgICAgICAgICAgbWV0aG9kPSJ1cGRhdGUiLAogICAgICAgICAgICAgICAgICAgIGxhYmVsPW5jb2wsCiAgICAgICAgICAgICAgICAgICAgYXJncz1bCiAgICAgICAgICAgICAgICAgICAgICAgIHsieSI6IFtkZi5sb2NbZGZbbGFiZWxfY29sdW1uXSA9PSBsXVtuY29sXSBmb3IgbCBpbiBsYWJlbHNdfSwKICAgICAgICAgICAgICAgICAgICAgICAgbnAuYXJhbmdlKGxlbihsYWJlbHMpKS50b2xpc3QoKSwKICAgICAgICAgICAgICAgICAgICBdLAogICAgICAgICAgICAgICAgKQogICAgICAgICAgICApCiAgICAgICAgZWxzZToKICAgICAgICAgICAgeF9idXR0b25zLmFwcGVuZCgKICAgICAgICAgICAgICAgIGRpY3QobWV0aG9kPSJ1cGRhdGUiLCBsYWJlbD1uY29sLCBhcmdzPVt7IngiOiBbZGZbbmNvbF1dfV0pCiAgICAgICAgICAgICkKCiAgICAgICAgICAgIHlfYnV0dG9ucy5hcHBlbmQoCiAgICAgICAgICAgICAgICBkaWN0KG1ldGhvZD0idXBkYXRlIiwgbGFiZWw9bmNvbCwgYXJncz1beyJ5IjogW2RmW25jb2xdXX1dKQogICAgICAgICAgICApCgogICAgIyBQYXNzIGJ1dHRvbnMgdG8gdGhlIHVwZGF0ZW1lbnVzIGFyZ3VtZW50CiAgICBmaWcudXBkYXRlX2xheW91dCgKICAgICAgICB1cGRhdGVtZW51cz1bCiAgICAgICAgICAgIGRpY3QoYnV0dG9ucz14X2J1dHRvbnMsIGRpcmVjdGlvbj0idXAiLCB4PTAuNSwgeT0tMC4xKSwKICAgICAgICAgICAgZGljdChidXR0b25zPXlfYnV0dG9ucywgZGlyZWN0aW9uPSJkb3duIiwgeD0tbWF4X2ZlYXR1cmVfbGVuIC8gMTAwLCB5PTAuNSksCiAgICAgICAgXQogICAgKQoKICAgIGZpZy51cGRhdGVfbGF5b3V0KAogICAgICAgIHdpZHRoPTYwMCwKICAgICAgICBoZWlnaHQ9NDAwLAogICAgICAgIGF1dG9zaXplPUZhbHNlLAogICAgICAgIG1hcmdpbj1kaWN0KHQ9MTAwLCBiPTAsIGw9MCwgcj0wKSwKICAgICAgICB0ZW1wbGF0ZT0icGxvdGx5X3doaXRlIiwKICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dCh0aXRsZV90ZXh0PSI8aT48Yj5TY2F0dGVyLTJkPC9iPjwvaT4iKQogICAgZXh0cmFfZGF0YVsic2NhdHRlci0yZCJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJzY2F0dGVyLTJkIiwgZmlndXJlPWZpZyksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9zY2F0dGVyLTJkLmh0bWwiLAogICAgKQoKCmRlZiBfY3JlYXRlX3Zpb2xpbl9hcnRpZmFjdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LCBkZjogcGQuRGF0YUZyYW1lLCBleHRyYV9kYXRhOiBkaWN0LCBwbG90c19kZXN0OiBzdHIKKToKICAgICIiIgogICAgQ3JlYXRlIGFuZCBsb2cgYSB2aW9saW4gYXJ0aWZhY3QKICAgICIiIgogICAgY29scyA9IDUKICAgIHJvd3MgPSAoZGYuc2hhcGVbMV0gLy8gY29scykgKyAxCiAgICBmaWcgPSBtYWtlX3N1YnBsb3RzKHJvd3M9cm93cywgY29scz1jb2xzKQoKICAgIHBsb3RfbnVtID0gMAoKICAgIGZvciBjb2x1bW5fbmFtZSBpbiBkZi5jb2x1bW5zOgogICAgICAgIGNvbHVtbl9kYXRhID0gZGZbY29sdW1uX25hbWVdCiAgICAgICAgdmlvbGluID0gZ28uVmlvbGluKAogICAgICAgICAgICB4PVtjb2x1bW5fbmFtZV0gKiBjb2x1bW5fZGF0YS5zaGFwZVswXSwKICAgICAgICAgICAgeT1jb2x1bW5fZGF0YSwKICAgICAgICAgICAgbmFtZT1jb2x1bW5fbmFtZSwKICAgICAgICApCgogICAgICAgIGZpZy5hZGRfdHJhY2UoCiAgICAgICAgICAgIHZpb2xpbiwKICAgICAgICAgICAgcm93PShwbG90X251bSAvLyBjb2xzKSArIDEsCiAgICAgICAgICAgIGNvbD0ocGxvdF9udW0gJSBjb2xzKSArIDEsCiAgICAgICAgKQoKICAgICAgICBwbG90X251bSArPSAxCgogICAgZmlnWyJsYXlvdXQiXS51cGRhdGUoCiAgICAgICAgaGVpZ2h0PShyb3dzICsgMSkgKiAyMDAsCiAgICAgICAgd2lkdGg9KGNvbHMgKyAxKSAqIDIwMCwKICAgICAgICB0aXRsZT0iPGk+PGI+VmlvbGluIFBsb3RzPC9iPjwvaT4iLAogICAgKQoKICAgIGZpZy51cGRhdGVfbGF5b3V0KHNob3dsZWdlbmQ9RmFsc2UpCiAgICBleHRyYV9kYXRhWyJ2aW9saW4iXSA9IGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgIFBsb3RseUFydGlmYWN0KGtleT0idmlvbGluIiwgZmlndXJlPWZpZyksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS92aW9saW4uaHRtbCIsCiAgICApCgoKZGVmIF9jcmVhdGVfaW1iYWxhbmNlX2FydGlmYWN0KAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkZjogcGQuRGF0YUZyYW1lLAogICAgZXh0cmFfZGF0YTogZGljdCwKICAgIGxhYmVsX2NvbHVtbjogc3RyLAogICAgcGxvdHNfZGVzdDogc3RyLAogICAgcHJvYmxlbV90eXBlOiBzdHIsCik6CiAgICAiIiIKICAgIENyZWF0ZSBhbmQgbG9nIGFuIGltYmFsYW5jZSBjbGFzcyBhcnRpZmFjdCAoY3N2ICsgcGxvdCkKICAgICIiIgogICAgaWYgbGFiZWxfY29sdW1uOgogICAgICAgIGlmIHByb2JsZW1fdHlwZSA9PSAiY2xhc3NpZmljYXRpb24iOgogICAgICAgICAgICB2YWx1ZXNfY29sdW1uID0gImNvdW50IgogICAgICAgICAgICBsYWJlbHNfY291bnQgPSBkZltsYWJlbF9jb2x1bW5dLnZhbHVlX2NvdW50cygpLnNvcnRfaW5kZXgoKQogICAgICAgICAgICBkZl9sYWJlbHNfY291bnQgPSBwZC5EYXRhRnJhbWUobGFiZWxzX2NvdW50KQogICAgICAgICAgICBkZl9sYWJlbHNfY291bnRbbGFiZWxfY29sdW1uXSA9IGxhYmVsc19jb3VudC5pbmRleAogICAgICAgICAgICBkZl9sYWJlbHNfY291bnQucmVuYW1lKGNvbHVtbnM9eyIiOiB2YWx1ZXNfY29sdW1ufSwgaW5wbGFjZT1UcnVlKQogICAgICAgICAgICBkZl9sYWJlbHNfY291bnRbdmFsdWVzX2NvbHVtbl0gPSBkZl9sYWJlbHNfY291bnRbdmFsdWVzX2NvbHVtbl0gLyBzdW0oCiAgICAgICAgICAgICAgICBkZl9sYWJlbHNfY291bnRbdmFsdWVzX2NvbHVtbl0KICAgICAgICAgICAgKQogICAgICAgICAgICBmaWcgPSBweC5waWUoZGZfbGFiZWxzX2NvdW50LCBuYW1lcz1sYWJlbF9jb2x1bW4sIHZhbHVlcz12YWx1ZXNfY29sdW1uKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGZpZyA9IHB4Lmhpc3RvZ3JhbSgKICAgICAgICAgICAgICAgIGhpc3RmdW5jPSJjb3VudCIsCiAgICAgICAgICAgICAgICB4PWRmW2xhYmVsX2NvbHVtbl0sCiAgICAgICAgICAgICkKICAgICAgICAgICAgaGlzdCA9IG5wLmhpc3RvZ3JhbShkZltsYWJlbF9jb2x1bW5dKQogICAgICAgICAgICBkZl9sYWJlbHNfY291bnQgPSBwZC5EYXRhRnJhbWUoCiAgICAgICAgICAgICAgICB7Im1pbl92YWwiOiBoaXN0WzFdLCAiY291bnQiOiBoaXN0WzBdLnRvbGlzdCgpICsgWzBdfQogICAgICAgICAgICApCiAgICAgICAgZmlnLnVwZGF0ZV9sYXlvdXQodGl0bGVfdGV4dD0iPGk+PGI+TGFiZWxzIEltYmFsYW5jZTwvYj48L2k+IikKICAgICAgICBleHRyYV9kYXRhWyJpbWJhbGFuY2UiXSA9IGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgICAgICBQbG90bHlBcnRpZmFjdChrZXk9ImltYmFsYW5jZSIsIGZpZ3VyZT1maWcpLAogICAgICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2ltYmFsYW5jZS5odG1sIiwKICAgICAgICApCiAgICAgICAgZXh0cmFfZGF0YVsiaW1iYWxhbmNlLWNzdiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgICAgIFRhYmxlQXJ0aWZhY3QoImltYmFsYW5jZS13ZWlnaHRzLXZlYyIsIGRmPWRmX2xhYmVsc19jb3VudCksCiAgICAgICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vaW1iYWxhbmNlLXdlaWdodHMtdmVjLmNzdiIsCiAgICAgICAgKQoKCmRlZiBfY3JlYXRlX2NvcnJfYXJ0aWZhY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIGRmOiBwZC5EYXRhRnJhbWUsCiAgICBleHRyYV9kYXRhOiBkaWN0LAogICAgbGFiZWxfY29sdW1uOiBzdHIsCiAgICBwbG90c19kZXN0OiBzdHIsCik6CiAgICAiIiIKICAgIENyZWF0ZSBhbmQgbG9nIGFuIGNvcnJlbGF0aW9uLW1hdHJpeCBhcnRpZmFjdCAoY3N2ICsgcGxvdCkKICAgICIiIgogICAgaWYgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lOgogICAgICAgIGRmID0gZGYuZHJvcChbbGFiZWxfY29sdW1uXSwgYXhpcz0xKQogICAgdGJsY29yciA9IGRmLmNvcnIobnVtZXJpY19vbmx5PVRydWUpCiAgICBleHRyYV9kYXRhWyJjb3JyZWxhdGlvbi1tYXRyaXgtY3N2Il0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBUYWJsZUFydGlmYWN0KCJjb3JyZWxhdGlvbi1tYXRyaXgtY3N2IiwgZGY9dGJsY29yciwgdmlzaWJsZT1UcnVlKSwKICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2NvcnJlbGF0aW9uLW1hdHJpeC5jc3YiLAogICAgKQoKICAgIHogPSB0Ymxjb3JyLnZhbHVlcy50b2xpc3QoKQogICAgel90ZXh0ID0gW1tmInt5Oi4yZn0iIGZvciB5IGluIHhdIGZvciB4IGluIHpdCiAgICBmaWcgPSBmZi5jcmVhdGVfYW5ub3RhdGVkX2hlYXRtYXAoCiAgICAgICAgeiwKICAgICAgICB4PWxpc3QodGJsY29yci5jb2x1bW5zKSwKICAgICAgICB5PWxpc3QodGJsY29yci5jb2x1bW5zKSwKICAgICAgICBhbm5vdGF0aW9uX3RleHQ9el90ZXh0LAogICAgICAgIGNvbG9yc2NhbGU9ImFnc3Vuc2V0IiwKICAgICkKICAgIGZpZ1sibGF5b3V0Il1bInlheGlzIl1bImF1dG9yYW5nZSJdID0gInJldmVyc2VkIiAgIyBsIC0+IHIKICAgIGZpZy51cGRhdGVfbGF5b3V0KHRpdGxlX3RleHQ9IjxpPjxiPkNvcnJlbGF0aW9uIG1hdHJpeDwvYj48L2k+IikKICAgIGZpZ1siZGF0YSJdWzBdWyJzaG93c2NhbGUiXSA9IFRydWUKCiAgICBleHRyYV9kYXRhWyJjb3JyZWxhdGlvbiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJjb3JyZWxhdGlvbiIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vY29ycmVsYXRpb24uaHRtbCIsCiAgICApCg== + code_origin: '' + filename: describe.py entry_points: analyze: - has_varargs: false outputs: - type: None parameters: @@ -13,7 +26,6 @@ spec: doc: Key of dataset to database ("dataset" for default) default: dataset - name: table - type: Union[FeatureSet, DataItem] doc: MLRun input pointing to pandas dataframe (csv/parquet file path) or FeatureSet as param default: null @@ -45,6 +57,7 @@ spec: - name: dask_client doc: Dask client object default: null + name: analyze doc: 'The function will output the following artifacts per column within the data frame (based on data types) @@ -70,21 +83,8 @@ spec: imbalance-weights-vec csv' has_kwargs: false - name: analyze - lineno: 46 - image: mlrun/mlrun + has_varargs: false + lineno: 45 command: '' - build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCmZyb20gdHlwaW5nIGltcG9ydCBVbmlvbgoKaW1wb3J0IG1scnVuCmltcG9ydCBudW1weSBhcyBucAoKd2FybmluZ3Muc2ltcGxlZmlsdGVyKGFjdGlvbj0iaWdub3JlIiwgY2F0ZWdvcnk9RnV0dXJlV2FybmluZykKCmltcG9ydCBtbHJ1bi5mZWF0dXJlX3N0b3JlIGFzIGZzdG9yZQppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCBwbG90bHkuZXhwcmVzcyBhcyBweAppbXBvcnQgcGxvdGx5LmZpZ3VyZV9mYWN0b3J5IGFzIGZmCmltcG9ydCBwbG90bHkuZ3JhcGhfb2JqZWN0cyBhcyBnbwpmcm9tIG1scnVuLmFydGlmYWN0cyBpbXBvcnQgKAogICAgQXJ0aWZhY3QsCiAgICBEYXRhc2V0QXJ0aWZhY3QsCiAgICBQbG90bHlBcnRpZmFjdCwKICAgIFRhYmxlQXJ0aWZhY3QsCiAgICB1cGRhdGVfZGF0YXNldF9tZXRhLAopCmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKZnJvbSBtbHJ1bi5mZWF0dXJlX3N0b3JlIGltcG9ydCBGZWF0dXJlU2V0CmZyb20gcGxvdGx5LnN1YnBsb3RzIGltcG9ydCBtYWtlX3N1YnBsb3RzCgpwZC5zZXRfb3B0aW9uKCJkaXNwbGF5LmZsb2F0X2Zvcm1hdCIsIGxhbWJkYSB4OiAiJS4yZiIgJSB4KQpNQVhfU0laRV9PRl9ERiA9IDUwMDAwMAoKCmRlZiBhbmFseXplKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBuYW1lOiBzdHIgPSAiZGF0YXNldCIsCiAgICB0YWJsZTogVW5pb25bRmVhdHVyZVNldCwgRGF0YUl0ZW1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gTm9uZSwKICAgIHBsb3RzX2Rlc3Q6IHN0ciA9ICJwbG90cyIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDEsCiAgICBwcm9ibGVtX3R5cGU6IHN0ciA9ICJjbGFzc2lmaWNhdGlvbiIsCiAgICBkYXNrX2tleTogc3RyID0gImRhc2tfa2V5IiwKICAgIGRhc2tfZnVuY3Rpb246IHN0ciA9IE5vbmUsCiAgICBkYXNrX2NsaWVudD1Ob25lLAopIC0+IE5vbmU6CiAgICAiIiIKICAgIFRoZSBmdW5jdGlvbiB3aWxsIG91dHB1dCB0aGUgZm9sbG93aW5nIGFydGlmYWN0cyBwZXIKICAgIGNvbHVtbiB3aXRoaW4gdGhlIGRhdGEgZnJhbWUgKGJhc2VkIG9uIGRhdGEgdHlwZXMpCiAgICBJZiB0aGUgZGF0YSBoYXMgbW9yZSB0aGFuIDUwMCwwMDAgc2FtcGxlIHdlCiAgICBzYW1wbGUgcmFuZG9tbHkgNTAwLDAwMCBzYW1wbGVzOgoKICAgIGRlc2NyaWJlIGNzdgogICAgaGlzdG9ncmFtcwogICAgc2NhdHRlci0yZAogICAgdmlvbGluIGNoYXJ0CiAgICBjb3JyZWxhdGlvbi1tYXRyaXggY2hhcnQKICAgIGNvcnJlbGF0aW9uLW1hdHJpeCBjc3YKICAgIGltYmFsYW5jZSBwaWUgY2hhcnQKICAgIGltYmFsYW5jZS13ZWlnaHRzLXZlYyBjc3YKCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIFRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbmFtZTogICAgICAgICAgICAgICAgICAgIEtleSBvZiBkYXRhc2V0IHRvIGRhdGFiYXNlICgiZGF0YXNldCIgZm9yIGRlZmF1bHQpCiAgICA6cGFyYW0gdGFibGU6ICAgICAgICAgICAgICAgICAgIE1MUnVuIGlucHV0IHBvaW50aW5nIHRvIHBhbmRhcyBkYXRhZnJhbWUgKGNzdi9wYXJxdWV0IGZpbGUgcGF0aCkgb3IgRmVhdHVyZVNldAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhcyBwYXJhbQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICAgICAgICBHcm91bmQgdHJ1dGggY29sdW1uIGxhYmVsCiAgICA6cGFyYW0gcGxvdHNfZGVzdDogICAgICAgICAgICAgIERlc3RpbmF0aW9uIGZvbGRlciBvZiBzdW1tYXJ5IHBsb3RzIChyZWxhdGl2ZSB0byBhcnRpZmFjdF9wYXRoKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoInBsb3RzIiBmb3IgZGVmYXVsdCkKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgV2hlbiB0aGUgdGFibGUgaGFzIG1vcmUgdGhhbiA1MDAsMDAwIHNhbXBsZXMsIHdlIHNhbXBsZSByYW5kb21seSA1MDAsMDAwIHNhbXBsZXMKICAgIDpwYXJhbSBwcm9ibGVtX3R5cGUgICAgICAgICAgICAgVGhlIHR5cGUgb2YgdGhlIE1MIHByb2JsZW0gdGhlIGRhdGEgZmFjaW5nIC0gcmVncmVzc2lvbiwgY2xhc3NpZmljYXRpb24gb3IgTm9uZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoY2xhc3NpZmljYXRpb24gZm9yIGRlZmF1bHQpCiAgICA6cGFyYW0gZGFza19rZXk6ICAgICAgICAgICAgICAgIEtleSBvZiBkYXRhZnJhbWUgaW4gZGFzayBjbGllbnQgImRhdGFzZXRzIiBhdHRyaWJ1dGUKICAgIDpwYXJhbSBkYXNrX2Z1bmN0aW9uOiAgICAgICAgICAgRGFzayBmdW5jdGlvbiB1cmwgKGRiOi8vLi4pCiAgICA6cGFyYW0gZGFza19jbGllbnQ6ICAgICAgICAgICAgIERhc2sgY2xpZW50IG9iamVjdAogICAgIiIiCiAgICBkYXRhX2l0ZW0sIGZlYXR1cmVzZXQsIGNyZWF0LCB1cGRhdGUgPSBGYWxzZSwgRmFsc2UsIEZhbHNlLCBGYWxzZQogICAgZ2V0X2Zyb21fdGFibGUgPSBUcnVlCiAgICBpZiBkYXNrX2Z1bmN0aW9uIG9yIGRhc2tfY2xpZW50OgogICAgICAgIGRhdGFfaXRlbSwgY3JlYXQgPSBUcnVlLCBUcnVlCiAgICAgICAgaWYgZGFza19mdW5jdGlvbjoKICAgICAgICAgICAgY2xpZW50ID0gbWxydW4uaW1wb3J0X2Z1bmN0aW9uKGRhc2tfZnVuY3Rpb24pLmNsaWVudAogICAgICAgIGVsaWYgZGFza19jbGllbnQ6CiAgICAgICAgICAgIGNsaWVudCA9IGRhc2tfY2xpZW50CiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigiZGFzayBjbGllbnQgd2FzIG5vdCBwcm92aWRlZCIpCgogICAgICAgIGlmIGRhc2tfa2V5IGluIGNsaWVudC5kYXRhc2V0czoKICAgICAgICAgICAgZGYgPSBjbGllbnQuZ2V0X2RhdGFzZXQoZGFza19rZXkpCiAgICAgICAgICAgIGRhdGFfaXRlbSwgY3JlYXQsIGdldF9mcm9tX3RhYmxlID0gVHJ1ZSwgVHJ1ZSwgRmFsc2UKICAgICAgICBlbGlmIHRhYmxlOgogICAgICAgICAgICBnZXRfZnJvbV90YWJsZSA9IFRydWUKICAgICAgICBlbHNlOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgZiJvbmx5IHRoZXNlIGRhdGFzZXRzIGFyZSBhdmFpbGFibGUge2NsaWVudC5kYXRhc2V0c30gaW4gY2xpZW50IHtjbGllbnR9IgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIEV4Y2VwdGlvbigiZGF0YXNldCBub3QgZm91bmQgb24gZGFzayBjbHVzdGVyIikKCiAgICBpZiBnZXRfZnJvbV90YWJsZToKICAgICAgICBpZiB0eXBlKHRhYmxlKSA9PSBEYXRhSXRlbToKICAgICAgICAgICAgaWYgdGFibGUubWV0YSBpcyBOb25lOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgVHJ1ZSwgRmFsc2UKICAgICAgICAgICAgZWxpZiB0YWJsZS5tZXRhLmtpbmQgPT0gImRhdGFzZXQiOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgRmFsc2UsIFRydWUKICAgICAgICAgICAgZWxpZiB0YWJsZS5tZXRhLmtpbmQgPT0gIkZlYXR1cmVWZWN0b3IiOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgRmFsc2UsIEZhbHNlCiAgICAgICAgICAgIGVsaWYgdGFibGUubWV0YS5raW5kID09ICJGZWF0dXJlU2V0IjoKICAgICAgICAgICAgICAgIGZlYXR1cmVzZXQsIGNyZWF0LCB1cGRhdGUgPSBUcnVlLCBGYWxzZSwgRmFsc2UKCiAgICAgICAgaWYgZGF0YV9pdGVtOgogICAgICAgICAgICBkZiA9IHRhYmxlLmFzX2RmKCkKICAgICAgICBlbGlmIGZlYXR1cmVzZXQ6CiAgICAgICAgICAgIHByb2plY3RfbmFtZSwgc2V0X25hbWUgPSAoCiAgICAgICAgICAgICAgICB0YWJsZS5fcGF0aC5zcGxpdCgiLyIpWzJdLAogICAgICAgICAgICAgICAgdGFibGUuX3BhdGguc3BsaXQoIi8iKVs0XSwKICAgICAgICAgICAgKQogICAgICAgICAgICBmZWF0dXJlX3NldCA9IGZzdG9yZS5nZXRfZmVhdHVyZV9zZXQoCiAgICAgICAgICAgICAgICBmInN0b3JlOi8vZmVhdHVyZS1zZXRzL3twcm9qZWN0X25hbWV9L3tzZXRfbmFtZX0iCiAgICAgICAgICAgICkKICAgICAgICAgICAgZGYgPSBmZWF0dXJlX3NldC50b19kYXRhZnJhbWUoKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKGYiV3JvbmcgdGFibGUgdHlwZS4iKQogICAgICAgICAgICByZXR1cm4KCiAgICBpZiBkZi5zaXplID4gTUFYX1NJWkVfT0ZfREY6CiAgICAgICAgZGYgPSBkZi5zYW1wbGUobj1pbnQoTUFYX1NJWkVfT0ZfREYgLyBkZi5zaGFwZVsxXSksIHJhbmRvbV9zdGF0ZT1yYW5kb21fc3RhdGUpCiAgICBleHRyYV9kYXRhID0ge30KCiAgICBpZiBsYWJlbF9jb2x1bW4gbm90IGluIGRmLmNvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1uID0gTm9uZQoKICAgIGV4dHJhX2RhdGFbImRlc2NyaWJlIGNzdiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgVGFibGVBcnRpZmFjdCgiZGVzY3JpYmUtY3N2IiwgZGY9ZGYuZGVzY3JpYmUoKSksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9kZXNjcmliZS5jc3YiLAogICAgKQoKICAgIHRyeToKICAgICAgICBfY3JlYXRlX2hpc3RvZ3JhbV9tYXRfYXJ0aWZhY3QoCiAgICAgICAgICAgIGNvbnRleHQsIGRmLCBleHRyYV9kYXRhLCBsYWJlbF9jb2x1bW4sIHBsb3RzX2Rlc3QKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgaGlzdG9ncmFtIG1hdHJpeCBhcnRpZmFjdCBkdWUgdG86IHtlfSIpCiAgICB0cnk6CiAgICAgICAgX2NyZWF0ZV9mZWF0dXJlc19oaXN0b2dyYW1fYXJ0aWZhY3RzKAogICAgICAgICAgICBjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0LCBwcm9ibGVtX3R5cGUKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgcGFpcnBsb3QgaGlzdG9ncmFtcyBkdWUgdG86IHtlfSIpCiAgICB0cnk6CiAgICAgICAgX2NyZWF0ZV9mZWF0dXJlc18yZF9zY2F0dGVyX2FydGlmYWN0cygKICAgICAgICAgICAgY29udGV4dCwgZGYsIGV4dHJhX2RhdGEsIGxhYmVsX2NvbHVtbiwgcGxvdHNfZGVzdCwgcHJvYmxlbV90eXBlCiAgICAgICAgKQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIHBhaXJwbG90IDJkX3NjYXR0ZXIgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfdmlvbGluX2FydGlmYWN0KGNvbnRleHQsIGRmLCBleHRyYV9kYXRhLCBwbG90c19kZXN0KQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIHZpb2xpbiBkaXN0cmlidXRpb24gcGxvdHMgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfaW1iYWxhbmNlX2FydGlmYWN0KAogICAgICAgICAgICBjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0LCBwcm9ibGVtX3R5cGUKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgY2xhc3MgaW1iYWxhbmNlIHBsb3QgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfY29ycl9hcnRpZmFjdChjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0KQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIGZlYXR1cmVzIGNvcnJlbGF0aW9uIHBsb3QgZHVlIHRvOiB7ZX0iKQoKICAgIGlmIG5vdCBkYXRhX2l0ZW06CiAgICAgICAgcmV0dXJuCgogICAgYXJ0aWZhY3QgPSB0YWJsZS5hcnRpZmFjdF91cmwKICAgIGlmIGNyZWF0OiAgIyBkYXRhc2V0IG5vdCBzdG9yZWQKICAgICAgICBhcnRpZmFjdCA9IERhdGFzZXRBcnRpZmFjdCgKICAgICAgICAgICAga2V5PSJkYXRhc2V0Iiwgc3RhdHM9VHJ1ZSwgZGY9ZGYsIGV4dHJhX2RhdGE9ZXh0cmFfZGF0YQogICAgICAgICkKICAgICAgICBhcnRpZmFjdCA9IGNvbnRleHQubG9nX2FydGlmYWN0KGFydGlmYWN0LCBkYl9rZXk9bmFtZSkKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVGhlIGRhdGEgc2V0IGlzIGxvZ2dlZCB0byB0aGUgcHJvamVjdCB1bmRlciB7bmFtZX0gbmFtZSIpCgogICAgaWYgdXBkYXRlOgogICAgICAgIHVwZGF0ZV9kYXRhc2V0X21ldGEoYXJ0aWZhY3QsIGV4dHJhX2RhdGE9ZXh0cmFfZGF0YSkKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVGhlIGRhdGEgc2V0IG5hbWVkIHtuYW1lfSBpcyB1cGRhdGVkIikKCiAgICAjIFRPRE8gOiAzLUQgcGxvdCBvbiBvbiBzZWxlY3RlZCBmZWF0dXJlcy4KICAgICMgVE9ETyA6IFJlaW50ZWdyYXRpb24gcGxvdCBvbiBvbiBzZWxlY3RlZCBmZWF0dXJlcy4KICAgICMgVE9ETyA6IFBDQSBwbG90ICh3aXRoIG9wdGlvbnMpCgoKZGVmIF9jcmVhdGVfaGlzdG9ncmFtX21hdF9hcnRpZmFjdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKKToKICAgICIiIgogICAgQ3JlYXRlIGFuZCBsb2cgYSBoaXN0b2dyYW0gbWF0cml4IGFydGlmYWN0CiAgICAiIiIKICAgIGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgIGl0ZW09QXJ0aWZhY3QoCiAgICAgICAgICAgIGtleT0iaGlzdCIsCiAgICAgICAgICAgIGJvZHk9YiI8Yj4gRGVwcmVjYXRlZCwgc2VlIHRoZSBhcnRpZmFjdHMgc2NhdHRlci0yZCAiCiAgICAgICAgICAgIGIiYW5kIGhpc3RvZ3JhbXMgaW5zdGVhZDxiPiIsCiAgICAgICAgKSwKICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2hpc3QuaHRtbCIsCiAgICApCgoKZGVmIF9jcmVhdGVfZmVhdHVyZXNfaGlzdG9ncmFtX2FydGlmYWN0cygKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhIGhpc3RvZ3JhbSBhcnRpZmFjdCBmb3IgZWFjaCBmZWF0dXJlCiAgICAiIiIKCiAgICBmaWdzID0gZGljdCgpCiAgICBmaXJzdF9mZWF0dXJlX25hbWUgPSAiIgogICAgaWYgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lIGFuZCBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICBhbGxfbGFiZWxzID0gZGZbbGFiZWxfY29sdW1uXS51bmlxdWUoKQogICAgdmlzaWJsZSA9IFRydWUKICAgIGZvciBjb2x1bW5fbmFtZSBpbiBkZi5jb2x1bW5zOgogICAgICAgIGlmIGNvbHVtbl9uYW1lID09IGxhYmVsX2NvbHVtbjoKICAgICAgICAgICAgY29udGludWUKCiAgICAgICAgaWYgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lIGFuZCBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICAgICAgZm9yIGxhYmVsIGluIGFsbF9sYWJlbHM6CiAgICAgICAgICAgICAgICBzdWJfZmlnID0gZ28uSGlzdG9ncmFtKAogICAgICAgICAgICAgICAgICAgIGhpc3RmdW5jPSJjb3VudCIsCiAgICAgICAgICAgICAgICAgICAgeD1kZi5sb2NbZGZbbGFiZWxfY29sdW1uXSA9PSBsYWJlbF1bY29sdW1uX25hbWVdLAogICAgICAgICAgICAgICAgICAgIG5hbWU9c3RyKGxhYmVsKSwKICAgICAgICAgICAgICAgICAgICB2aXNpYmxlPXZpc2libGUsCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBmaWdzW2Yie2NvbHVtbl9uYW1lfUA/QHtsYWJlbH0iXSA9IHN1Yl9maWcKICAgICAgICBlbHNlOgogICAgICAgICAgICBzdWJfZmlnID0gZ28uSGlzdG9ncmFtKGhpc3RmdW5jPSJjb3VudCIsIHg9ZGZbY29sdW1uX25hbWVdLCB2aXNpYmxlPXZpc2libGUpCiAgICAgICAgICAgIGZpZ3NbZiJ7Y29sdW1uX25hbWV9QD9AezF9Il0gPSBzdWJfZmlnCiAgICAgICAgaWYgdmlzaWJsZToKICAgICAgICAgICAgZmlyc3RfZmVhdHVyZV9uYW1lID0gY29sdW1uX25hbWUKICAgICAgICB2aXNpYmxlID0gRmFsc2UKCiAgICBmaWcgPSBnby5GaWd1cmUoKQogICAgZm9yIGsgaW4gZmlncy5rZXlzKCk6CiAgICAgICAgZmlnLmFkZF90cmFjZShmaWdzW2tdKQoKICAgIGZpZy51cGRhdGVfbGF5b3V0KAogICAgICAgIHVwZGF0ZW1lbnVzPVsKICAgICAgICAgICAgewogICAgICAgICAgICAgICAgImJ1dHRvbnMiOiBbCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAibGFiZWwiOiBjb2x1bW5fbmFtZSwKICAgICAgICAgICAgICAgICAgICAgICAgIm1ldGhvZCI6ICJ1cGRhdGUiLAogICAgICAgICAgICAgICAgICAgICAgICAiYXJncyI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidmlzaWJsZSI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAga2V5LnNwbGl0KCJAP0AiKVswXSA9PSBjb2x1bW5fbmFtZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb3Iga2V5IGluIGZpZ3Mua2V5cygpCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAieGF4aXMiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJyYW5nZSI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1pbihkZltjb2x1bW5fbmFtZV0pLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWF4KGRmW2NvbHVtbl9uYW1lXSksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHsidGl0bGUiOiBmIjxpPjxiPkhpc3RvZ3JhbSBvZiB7Y29sdW1uX25hbWV9PC9iPjwvaT4ifSwKICAgICAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICAgICAgZm9yIGNvbHVtbl9uYW1lIGluIGRmLmNvbHVtbnMKICAgICAgICAgICAgICAgICAgICBpZiBjb2x1bW5fbmFtZSAhPSBsYWJlbF9jb2x1bW4KICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICAiZGlyZWN0aW9uIjogImRvd24iLAogICAgICAgICAgICAgICAgInBhZCI6IHsiciI6IDEwLCAidCI6IDEwfSwKICAgICAgICAgICAgICAgICJzaG93YWN0aXZlIjogVHJ1ZSwKICAgICAgICAgICAgICAgICJ4IjogMC4yNSwKICAgICAgICAgICAgICAgICJ4YW5jaG9yIjogImxlZnQiLAogICAgICAgICAgICAgICAgInkiOiAxLjEsCiAgICAgICAgICAgICAgICAieWFuY2hvciI6ICJ0b3AiLAogICAgICAgICAgICB9CiAgICAgICAgXSwKICAgICAgICBhbm5vdGF0aW9ucz1bCiAgICAgICAgICAgIGRpY3QoCiAgICAgICAgICAgICAgICB0ZXh0PSJTZWxlY3QgRmVhdHVyZSBOYW1lICIsCiAgICAgICAgICAgICAgICBzaG93YXJyb3c9RmFsc2UsCiAgICAgICAgICAgICAgICB4PTAsCiAgICAgICAgICAgICAgICB5PTEuMDUsCiAgICAgICAgICAgICAgICB5cmVmPSJwYXBlciIsCiAgICAgICAgICAgICAgICB4cmVmPSJwYXBlciIsCiAgICAgICAgICAgICAgICBhbGlnbj0ibGVmdCIsCiAgICAgICAgICAgICAgICB4YW5jaG9yPSJsZWZ0IiwKICAgICAgICAgICAgICAgIHlhbmNob3I9InRvcCIsCiAgICAgICAgICAgICAgICBmb250PXsKICAgICAgICAgICAgICAgICAgICAiY29sb3IiOiAiYmx1ZSIsCiAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICApCiAgICAgICAgXSwKICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dCgKICAgICAgICB3aWR0aD02MDAsCiAgICAgICAgaGVpZ2h0PTQwMCwKICAgICAgICBhdXRvc2l6ZT1GYWxzZSwKICAgICAgICBtYXJnaW49ZGljdCh0PTEwMCwgYj0wLCBsPTAsIHI9MCksCiAgICAgICAgdGVtcGxhdGU9InBsb3RseV93aGl0ZSIsCiAgICApCgogICAgZmlnLnVwZGF0ZV9sYXlvdXQodGl0bGVfdGV4dD1mIjxpPjxiPkhpc3RvZ3JhbXMgb2Yge2ZpcnN0X2ZlYXR1cmVfbmFtZX08L2I+PC9pPiIpCiAgICBleHRyYV9kYXRhW2YiaGlzdG9ncmFtcyJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PWYiaGlzdG9ncmFtcyIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vaGlzdG9ncmFtcy5odG1sIiwKICAgICkKCgpkZWYgX2NyZWF0ZV9mZWF0dXJlc18yZF9zY2F0dGVyX2FydGlmYWN0cygKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhIHNjYXR0ZXItMmQgYXJ0aWZhY3QgZm9yIGVhY2ggY291cGxlIG9mIGZlYXR1cmVzCiAgICAiIiIKICAgIGZlYXR1cmVzID0gWwogICAgICAgIGNvbHVtbl9uYW1lIGZvciBjb2x1bW5fbmFtZSBpbiBkZi5jb2x1bW5zIGlmIGNvbHVtbl9uYW1lICE9IGxhYmVsX2NvbHVtbgogICAgXQogICAgbWF4X2ZlYXR1cmVfbGVuID0gZmxvYXQobWF4KGxlbihlbGVtKSBmb3IgZWxlbSBpbiBmZWF0dXJlcykpCiAgICBpZiBsYWJlbF9jb2x1bW4gaXMgbm90IE5vbmU6CiAgICAgICAgbGFiZWxzID0gc29ydGVkKGRmW2xhYmVsX2NvbHVtbl0udW5pcXVlKCkpCiAgICBlbHNlOgogICAgICAgIGxhYmVscyA9IFtOb25lXQogICAgZmlnID0gZ28uRmlndXJlKCkKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZSBhbmQgcHJvYmxlbV90eXBlID09ICJjbGFzc2lmaWNhdGlvbiI6CiAgICAgICAgZm9yIGwgaW4gbGFiZWxzOgogICAgICAgICAgICBmaWcuYWRkX3RyYWNlKAogICAgICAgICAgICAgICAgZ28uU2NhdHRlcigKICAgICAgICAgICAgICAgICAgICB4PWRmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgICAgICB5PWRmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgICAgICBtb2RlPSJtYXJrZXJzIiwKICAgICAgICAgICAgICAgICAgICB2aXNpYmxlPVRydWUsCiAgICAgICAgICAgICAgICAgICAgc2hvd2xlZ2VuZD1UcnVlLAogICAgICAgICAgICAgICAgICAgIG5hbWU9c3RyKGwpLAogICAgICAgICAgICAgICAgKQogICAgICAgICAgICApCiAgICBlbGlmIGxhYmVsX2NvbHVtbiBpcyBOb25lOgogICAgICAgIGZpZy5hZGRfdHJhY2UoCiAgICAgICAgICAgIGdvLlNjYXR0ZXIoCiAgICAgICAgICAgICAgICB4PWRmW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgIHk9ZGZbZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgbW9kZT0ibWFya2VycyIsCiAgICAgICAgICAgICAgICB2aXNpYmxlPVRydWUsCiAgICAgICAgICAgICkKICAgICAgICApCiAgICBlbGlmIHByb2JsZW1fdHlwZSA9PSAicmVncmVzc2lvbiI6CiAgICAgICAgZmlnLmFkZF90cmFjZSgKICAgICAgICAgICAgZ28uU2NhdHRlcigKICAgICAgICAgICAgICAgIHg9ZGZbZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgeT1kZltmZWF0dXJlc1swXV0sCiAgICAgICAgICAgICAgICBtb2RlPSJtYXJrZXJzIiwKICAgICAgICAgICAgICAgIG1hcmtlcj1kaWN0KAogICAgICAgICAgICAgICAgICAgIGNvbG9yPWRmW2xhYmVsX2NvbHVtbl0sIGNvbG9yc2NhbGU9IlZpcmlkaXMiLCBzaG93c2NhbGU9VHJ1ZQogICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgIHZpc2libGU9VHJ1ZSwKICAgICAgICAgICAgKQogICAgICAgICkKCiAgICB4X2J1dHRvbnMgPSBbXQogICAgeV9idXR0b25zID0gW10KCiAgICBmb3IgbmNvbCBpbiBmZWF0dXJlczoKICAgICAgICBpZiBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIiBhbmQgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lOgogICAgICAgICAgICB4X2J1dHRvbnMuYXBwZW5kKAogICAgICAgICAgICAgICAgZGljdCgKICAgICAgICAgICAgICAgICAgICBtZXRob2Q9InVwZGF0ZSIsCiAgICAgICAgICAgICAgICAgICAgbGFiZWw9bmNvbCwKICAgICAgICAgICAgICAgICAgICBhcmdzPVsKICAgICAgICAgICAgICAgICAgICAgICAgeyJ4IjogW2RmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW25jb2xdIGZvciBsIGluIGxhYmVsc119LAogICAgICAgICAgICAgICAgICAgICAgICBucC5hcmFuZ2UobGVuKGxhYmVscykpLnRvbGlzdCgpLAogICAgICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICkKCiAgICAgICAgICAgIHlfYnV0dG9ucy5hcHBlbmQoCiAgICAgICAgICAgICAgICBkaWN0KAogICAgICAgICAgICAgICAgICAgIG1ldGhvZD0idXBkYXRlIiwKICAgICAgICAgICAgICAgICAgICBsYWJlbD1uY29sLAogICAgICAgICAgICAgICAgICAgIGFyZ3M9WwogICAgICAgICAgICAgICAgICAgICAgICB7InkiOiBbZGYubG9jW2RmW2xhYmVsX2NvbHVtbl0gPT0gbF1bbmNvbF0gZm9yIGwgaW4gbGFiZWxzXX0sCiAgICAgICAgICAgICAgICAgICAgICAgIG5wLmFyYW5nZShsZW4obGFiZWxzKSkudG9saXN0KCksCiAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHhfYnV0dG9ucy5hcHBlbmQoCiAgICAgICAgICAgICAgICBkaWN0KG1ldGhvZD0idXBkYXRlIiwgbGFiZWw9bmNvbCwgYXJncz1beyJ4IjogW2RmW25jb2xdXX1dKQogICAgICAgICAgICApCgogICAgICAgICAgICB5X2J1dHRvbnMuYXBwZW5kKAogICAgICAgICAgICAgICAgZGljdChtZXRob2Q9InVwZGF0ZSIsIGxhYmVsPW5jb2wsIGFyZ3M9W3sieSI6IFtkZltuY29sXV19XSkKICAgICAgICAgICAgKQoKICAgICMgUGFzcyBidXR0b25zIHRvIHRoZSB1cGRhdGVtZW51cyBhcmd1bWVudAogICAgZmlnLnVwZGF0ZV9sYXlvdXQoCiAgICAgICAgdXBkYXRlbWVudXM9WwogICAgICAgICAgICBkaWN0KGJ1dHRvbnM9eF9idXR0b25zLCBkaXJlY3Rpb249InVwIiwgeD0wLjUsIHk9LTAuMSksCiAgICAgICAgICAgIGRpY3QoYnV0dG9ucz15X2J1dHRvbnMsIGRpcmVjdGlvbj0iZG93biIsIHg9LW1heF9mZWF0dXJlX2xlbiAvIDEwMCwgeT0wLjUpLAogICAgICAgIF0KICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dCgKICAgICAgICB3aWR0aD02MDAsCiAgICAgICAgaGVpZ2h0PTQwMCwKICAgICAgICBhdXRvc2l6ZT1GYWxzZSwKICAgICAgICBtYXJnaW49ZGljdCh0PTEwMCwgYj0wLCBsPTAsIHI9MCksCiAgICAgICAgdGVtcGxhdGU9InBsb3RseV93aGl0ZSIsCiAgICApCgogICAgZmlnLnVwZGF0ZV9sYXlvdXQodGl0bGVfdGV4dD1mIjxpPjxiPlNjYXR0ZXItMmQ8L2I+PC9pPiIpCiAgICBleHRyYV9kYXRhW2Yic2NhdHRlci0yZCJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PWYic2NhdHRlci0yZCIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vc2NhdHRlci0yZC5odG1sIiwKICAgICkKCgpkZWYgX2NyZWF0ZV92aW9saW5fYXJ0aWZhY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwgZGY6IHBkLkRhdGFGcmFtZSwgZXh0cmFfZGF0YTogZGljdCwgcGxvdHNfZGVzdDogc3RyCik6CiAgICAiIiIKICAgIENyZWF0ZSBhbmQgbG9nIGEgdmlvbGluIGFydGlmYWN0CiAgICAiIiIKICAgIGNvbHMgPSA1CiAgICByb3dzID0gKGRmLnNoYXBlWzFdIC8vIGNvbHMpICsgMQogICAgZmlnID0gbWFrZV9zdWJwbG90cyhyb3dzPXJvd3MsIGNvbHM9Y29scykKCiAgICBwbG90X251bSA9IDAKCiAgICBmb3IgY29sdW1uX25hbWUgaW4gZGYuY29sdW1uczoKICAgICAgICBjb2x1bW5fZGF0YSA9IGRmW2NvbHVtbl9uYW1lXQogICAgICAgIHZpb2xpbiA9IGdvLlZpb2xpbigKICAgICAgICAgICAgeD1bY29sdW1uX25hbWVdICogY29sdW1uX2RhdGEuc2hhcGVbMF0sCiAgICAgICAgICAgIHk9Y29sdW1uX2RhdGEsCiAgICAgICAgICAgIG5hbWU9Y29sdW1uX25hbWUsCiAgICAgICAgKQoKICAgICAgICBmaWcuYWRkX3RyYWNlKAogICAgICAgICAgICB2aW9saW4sCiAgICAgICAgICAgIHJvdz0ocGxvdF9udW0gLy8gY29scykgKyAxLAogICAgICAgICAgICBjb2w9KHBsb3RfbnVtICUgY29scykgKyAxLAogICAgICAgICkKCiAgICAgICAgcGxvdF9udW0gKz0gMQoKICAgIGZpZ1sibGF5b3V0Il0udXBkYXRlKAogICAgICAgIGhlaWdodD0ocm93cyArIDEpICogMjAwLAogICAgICAgIHdpZHRoPShjb2xzICsgMSkgKiAyMDAsCiAgICAgICAgdGl0bGU9IjxpPjxiPlZpb2xpbiBQbG90czwvYj48L2k+IiwKICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dChzaG93bGVnZW5kPUZhbHNlKQogICAgZXh0cmFfZGF0YVsidmlvbGluIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBQbG90bHlBcnRpZmFjdChrZXk9InZpb2xpbiIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vdmlvbGluLmh0bWwiLAogICAgKQoKCmRlZiBfY3JlYXRlX2ltYmFsYW5jZV9hcnRpZmFjdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhbiBpbWJhbGFuY2UgY2xhc3MgYXJ0aWZhY3QgKGNzdiArIHBsb3QpCiAgICAiIiIKICAgIGlmIGxhYmVsX2NvbHVtbjoKICAgICAgICBpZiBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICAgICAgdmFsdWVzX2NvbHVtbiA9ICJjb3VudCIKICAgICAgICAgICAgbGFiZWxzX2NvdW50ID0gZGZbbGFiZWxfY29sdW1uXS52YWx1ZV9jb3VudHMoKS5zb3J0X2luZGV4KCkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50ID0gcGQuRGF0YUZyYW1lKGxhYmVsc19jb3VudCkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W2xhYmVsX2NvbHVtbl0gPSBsYWJlbHNfY291bnQuaW5kZXgKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50LnJlbmFtZShjb2x1bW5zPXsiIjogdmFsdWVzX2NvbHVtbn0sIGlucGxhY2U9VHJ1ZSkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dID0gZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dIC8gc3VtKAogICAgICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dCiAgICAgICAgICAgICkKICAgICAgICAgICAgZmlnID0gcHgucGllKGRmX2xhYmVsc19jb3VudCwgbmFtZXM9bGFiZWxfY29sdW1uLCB2YWx1ZXM9dmFsdWVzX2NvbHVtbikKICAgICAgICBlbHNlOgogICAgICAgICAgICBmaWcgPSBweC5oaXN0b2dyYW0oCiAgICAgICAgICAgICAgICBoaXN0ZnVuYz0iY291bnQiLAogICAgICAgICAgICAgICAgeD1kZltsYWJlbF9jb2x1bW5dLAogICAgICAgICAgICApCiAgICAgICAgICAgIGhpc3QgPSBucC5oaXN0b2dyYW0oZGZbbGFiZWxfY29sdW1uXSkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50ID0gcGQuRGF0YUZyYW1lKAogICAgICAgICAgICAgICAgeyJtaW5fdmFsIjogaGlzdFsxXSwgImNvdW50IjogaGlzdFswXS50b2xpc3QoKSArIFswXX0KICAgICAgICAgICAgKQogICAgICAgIGZpZy51cGRhdGVfbGF5b3V0KHRpdGxlX3RleHQ9IjxpPjxiPkxhYmVscyBJbWJhbGFuY2U8L2I+PC9pPiIpCiAgICAgICAgZXh0cmFfZGF0YVsiaW1iYWxhbmNlIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJpbWJhbGFuY2UiLCBmaWd1cmU9ZmlnKSwKICAgICAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9pbWJhbGFuY2UuaHRtbCIsCiAgICAgICAgKQogICAgICAgIGV4dHJhX2RhdGFbImltYmFsYW5jZS1jc3YiXSA9IGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgICAgICBUYWJsZUFydGlmYWN0KCJpbWJhbGFuY2Utd2VpZ2h0cy12ZWMiLCBkZj1kZl9sYWJlbHNfY291bnQpLAogICAgICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2ltYmFsYW5jZS13ZWlnaHRzLXZlYy5jc3YiLAogICAgICAgICkKCgpkZWYgX2NyZWF0ZV9jb3JyX2FydGlmYWN0KAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkZjogcGQuRGF0YUZyYW1lLAogICAgZXh0cmFfZGF0YTogZGljdCwKICAgIGxhYmVsX2NvbHVtbjogc3RyLAogICAgcGxvdHNfZGVzdDogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhbiBjb3JyZWxhdGlvbi1tYXRyaXggYXJ0aWZhY3QgKGNzdiArIHBsb3QpCiAgICAiIiIKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZToKICAgICAgICBkZiA9IGRmLmRyb3AoW2xhYmVsX2NvbHVtbl0sIGF4aXM9MSkKICAgIHRibGNvcnIgPSBkZi5jb3JyKG51bWVyaWNfb25seT1UcnVlKQogICAgZXh0cmFfZGF0YVsiY29ycmVsYXRpb24tbWF0cml4LWNzdiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgVGFibGVBcnRpZmFjdCgiY29ycmVsYXRpb24tbWF0cml4LWNzdiIsIGRmPXRibGNvcnIsIHZpc2libGU9VHJ1ZSksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9jb3JyZWxhdGlvbi1tYXRyaXguY3N2IiwKICAgICkKCiAgICB6ID0gdGJsY29yci52YWx1ZXMudG9saXN0KCkKICAgIHpfdGV4dCA9IFtbIns6LjJmfSIuZm9ybWF0KHkpIGZvciB5IGluIHhdIGZvciB4IGluIHpdCiAgICBmaWcgPSBmZi5jcmVhdGVfYW5ub3RhdGVkX2hlYXRtYXAoCiAgICAgICAgeiwKICAgICAgICB4PWxpc3QodGJsY29yci5jb2x1bW5zKSwKICAgICAgICB5PWxpc3QodGJsY29yci5jb2x1bW5zKSwKICAgICAgICBhbm5vdGF0aW9uX3RleHQ9el90ZXh0LAogICAgICAgIGNvbG9yc2NhbGU9ImFnc3Vuc2V0IiwKICAgICkKICAgIGZpZ1sibGF5b3V0Il1bInlheGlzIl1bImF1dG9yYW5nZSJdID0gInJldmVyc2VkIiAgIyBsIC0+IHIKICAgIGZpZy51cGRhdGVfbGF5b3V0KHRpdGxlX3RleHQ9IjxpPjxiPkNvcnJlbGF0aW9uIG1hdHJpeDwvYj48L2k+IikKICAgIGZpZ1siZGF0YSJdWzBdWyJzaG93c2NhbGUiXSA9IFRydWUKCiAgICBleHRyYV9kYXRhWyJjb3JyZWxhdGlvbiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJjb3JyZWxhdGlvbiIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vY29ycmVsYXRpb24uaHRtbCIsCiAgICApCg== - code_origin: '' - origin_filename: '' description: describe and visualizes dataset stats - disable_auto_mount: false default_handler: analyze -verbose: false -metadata: - tag: '' - name: describe - categories: - - data-analysis -kind: job diff --git a/functions/src/describe/test_describe.py b/functions/src/describe/test_describe.py index 9ffe39abb..4ea56c979 100644 --- a/functions/src/describe/test_describe.py +++ b/functions/src/describe/test_describe.py @@ -15,12 +15,10 @@ import os import shutil from pathlib import Path -from typing import Set -import mlrun import pandas as pd import pytest -from mlrun import code_to_function, import_function, new_function +from mlrun import import_function from mlrun.execution import MLClientCtx from sklearn.datasets import make_classification, make_regression @@ -29,7 +27,7 @@ ARTIFACTS_PATH = os.path.abspath("./artifacts") -def _validate_paths(paths: Set): +def _validate_paths(paths: set): """ Check if all the expected plot are saved """ diff --git a/functions/src/describe_dask/describe_dask.py b/functions/src/describe_dask/describe_dask.py index 3dc382820..a34535a3c 100644 --- a/functions/src/describe_dask/describe_dask.py +++ b/functions/src/describe_dask/describe_dask.py @@ -12,19 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import mlrun import warnings + +import mlrun + warnings.simplefilter(action="ignore", category=FutureWarning) -import pandas as pd import matplotlib.pyplot as plt +import numpy as np +import pandas as pd import seaborn as sns from mlrun.artifacts import PlotArtifact, TableArtifact from mlrun.mlutils.plots import gcf_clear -import numpy as np - pd.set_option("display.float_format", lambda x: "%.2f" % x) + def summarize( context, dask_key: str = "dask_key", @@ -35,7 +37,7 @@ def summarize( dask_client=None, ) -> None: """Summarize a table - + Connects to dask client through the function context, or through an optional user-supplied scheduler. @@ -51,15 +53,17 @@ def summarize( elif dask_client: client = dask_client else: - raise ValueError('dask client was not provided') - + raise ValueError("dask client was not provided") + if dask_key in client.datasets: table = client.get_dataset(dask_key) elif dataset: - #table = dataset.as_df(df_module=dd) + # table = dataset.as_df(df_module=dd) table = dataset.as_df() else: - context.logger.info(f"only these datasets are available {client.datasets} in client {client}") + context.logger.info( + f"only these datasets are available {client.datasets} in client {client}" + ) raise Exception("dataset not found on dask cluster") df = table header = df.columns.values diff --git a/functions/src/describe_dask/function.yaml b/functions/src/describe_dask/function.yaml index baf3ced1d..eaf9b2177 100644 --- a/functions/src/describe_dask/function.yaml +++ b/functions/src/describe_dask/function.yaml @@ -1,17 +1,22 @@ +metadata: + tag: '' + name: describe-dask + categories: + - data-analysis verbose: false +kind: job spec: - disable_auto_mount: false image: mlrun/ml-models - command: '' - default_handler: summarize + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IHdhcm5pbmdzCgppbXBvcnQgbWxydW4KCndhcm5pbmdzLnNpbXBsZWZpbHRlcihhY3Rpb249Imlnbm9yZSIsIGNhdGVnb3J5PUZ1dHVyZVdhcm5pbmcpCmltcG9ydCBtYXRwbG90bGliLnB5cGxvdCBhcyBwbHQKaW1wb3J0IG51bXB5IGFzIG5wCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IHNlYWJvcm4gYXMgc25zCmZyb20gbWxydW4uYXJ0aWZhY3RzIGltcG9ydCBQbG90QXJ0aWZhY3QsIFRhYmxlQXJ0aWZhY3QKZnJvbSBtbHJ1bi5tbHV0aWxzLnBsb3RzIGltcG9ydCBnY2ZfY2xlYXIKCnBkLnNldF9vcHRpb24oImRpc3BsYXkuZmxvYXRfZm9ybWF0IiwgbGFtYmRhIHg6ICIlLjJmIiAlIHgpCgoKZGVmIHN1bW1hcml6ZSgKICAgIGNvbnRleHQsCiAgICBkYXNrX2tleTogc3RyID0gImRhc2tfa2V5IiwKICAgIGRhdGFzZXQ6IG1scnVuLkRhdGFJdGVtID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gImxhYmVsIiwKICAgIHBsb3RzX2Rlc3Q6IHN0ciA9ICJwbG90cyIsCiAgICBkYXNrX2Z1bmN0aW9uOiBzdHIgPSBOb25lLAogICAgZGFza19jbGllbnQ9Tm9uZSwKKSAtPiBOb25lOgogICAgIiIiU3VtbWFyaXplIGEgdGFibGUKCiAgICBDb25uZWN0cyB0byBkYXNrIGNsaWVudCB0aHJvdWdoIHRoZSBmdW5jdGlvbiBjb250ZXh0LCBvciB0aHJvdWdoIGFuIG9wdGlvbmFsCiAgICB1c2VyLXN1cHBsaWVkIHNjaGVkdWxlci4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICB0aGUgZnVuY3Rpb24gY29udGV4dAogICAgOnBhcmFtIGRhc2tfa2V5OiAgICAgICAga2V5IG9mIGRhdGFmcmFtZSBpbiBkYXNrIGNsaWVudCAiZGF0YXNldHMiIGF0dHJpYnV0ZQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgZ3JvdW5kIHRydXRoIGNvbHVtbiBsYWJlbAogICAgOnBhcmFtIHBsb3RzX2Rlc3Q6ICAgICAgZGVzdGluYXRpb24gZm9sZGVyIG9mIHN1bW1hcnkgcGxvdHMgKHJlbGF0aXZlIHRvIGFydGlmYWN0X3BhdGgpCiAgICA6cGFyYW0gZGFza19mdW5jdGlvbjogICBkYXNrIGZ1bmN0aW9uIHVybCAoZGI6Ly8uLikKICAgIDpwYXJhbSBkYXNrX2NsaWVudDogICAgIGRhc2sgY2xpZW50IG9iamVjdAogICAgIiIiCiAgICBpZiBkYXNrX2Z1bmN0aW9uOgogICAgICAgIGNsaWVudCA9IG1scnVuLmltcG9ydF9mdW5jdGlvbihkYXNrX2Z1bmN0aW9uKS5jbGllbnQKICAgIGVsaWYgZGFza19jbGllbnQ6CiAgICAgICAgY2xpZW50ID0gZGFza19jbGllbnQKICAgIGVsc2U6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigiZGFzayBjbGllbnQgd2FzIG5vdCBwcm92aWRlZCIpCgogICAgaWYgZGFza19rZXkgaW4gY2xpZW50LmRhdGFzZXRzOgogICAgICAgIHRhYmxlID0gY2xpZW50LmdldF9kYXRhc2V0KGRhc2tfa2V5KQogICAgZWxpZiBkYXRhc2V0OgogICAgICAgICMgdGFibGUgPSBkYXRhc2V0LmFzX2RmKGRmX21vZHVsZT1kZCkKICAgICAgICB0YWJsZSA9IGRhdGFzZXQuYXNfZGYoKQogICAgZWxzZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIm9ubHkgdGhlc2UgZGF0YXNldHMgYXJlIGF2YWlsYWJsZSB7Y2xpZW50LmRhdGFzZXRzfSBpbiBjbGllbnQge2NsaWVudH0iCiAgICAgICAgKQogICAgICAgIHJhaXNlIEV4Y2VwdGlvbigiZGF0YXNldCBub3QgZm91bmQgb24gZGFzayBjbHVzdGVyIikKICAgIGRmID0gdGFibGUKICAgIGhlYWRlciA9IGRmLmNvbHVtbnMudmFsdWVzCiAgICBleHRyYV9kYXRhID0ge30KCiAgICB0cnk6CiAgICAgICAgZ2NmX2NsZWFyKHBsdCkKICAgICAgICBzbnNwbHQgPSBzbnMucGFpcnBsb3QoZGYsIGh1ZT1sYWJlbF9jb2x1bW4pICAjICwgZGlhZ19rd3M9eyJidyI6IDEuNX0pCiAgICAgICAgZXh0cmFfZGF0YVsiaGlzdG9ncmFtcyJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgICAgIFBsb3RBcnRpZmFjdCgiaGlzdG9ncmFtcyIsIGJvZHk9cGx0LmdjZigpKSwKICAgICAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9oaXN0Lmh0bWwiLAogICAgICAgICAgICBkYl9rZXk9RmFsc2UsCiAgICAgICAgKQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKGYiRmFpbGVkIHRvIGNyZWF0ZSBwYWlycGxvdCBoaXN0b2dyYW1zIGR1ZSB0bzoge2V9IikKCiAgICB0cnk6CiAgICAgICAgZ2NmX2NsZWFyKHBsdCkKICAgICAgICBwbG90X2NvbHMgPSAzCiAgICAgICAgcGxvdF9yb3dzID0gaW50KChsZW4oaGVhZGVyKSAtIDEpIC8gcGxvdF9jb2xzKSArIDEKICAgICAgICBmaWcsIGF4ID0gcGx0LnN1YnBsb3RzKHBsb3Rfcm93cywgcGxvdF9jb2xzLCBmaWdzaXplPSgxNSwgNCkpCiAgICAgICAgZmlnLnRpZ2h0X2xheW91dChwYWQ9Mi4wKQogICAgICAgIGZvciBpIGluIHJhbmdlKHBsb3Rfcm93cyAqIHBsb3RfY29scyk6CiAgICAgICAgICAgIGlmIGkgPCBsZW4oaGVhZGVyKToKICAgICAgICAgICAgICAgIHNucy52aW9saW5wbG90KAogICAgICAgICAgICAgICAgICAgIHg9ZGZbaGVhZGVyW2ldXSwKICAgICAgICAgICAgICAgICAgICBheD1heFtpbnQoaSAvIHBsb3RfY29scyldW2kgJSBwbG90X2NvbHNdLAogICAgICAgICAgICAgICAgICAgIG9yaWVudD0iaCIsCiAgICAgICAgICAgICAgICAgICAgd2lkdGg9MC43LAogICAgICAgICAgICAgICAgICAgIGlubmVyPSJxdWFydGlsZSIsCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICBmaWcuZGVsYXhlcyhheFtpbnQoaSAvIHBsb3RfY29scyldW2kgJSBwbG90X2NvbHNdKQogICAgICAgICAgICBpICs9IDEKICAgICAgICBleHRyYV9kYXRhWyJ2aW9saW4iXSA9IGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgICAgICBQbG90QXJ0aWZhY3QoInZpb2xpbiIsIGJvZHk9cGx0LmdjZigpLCB0aXRsZT0iVmlvbGluIFBsb3QiKSwKICAgICAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS92aW9saW4uaHRtbCIsCiAgICAgICAgICAgIGRiX2tleT1GYWxzZSwKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgdmlvbGluIGRpc3RyaWJ1dGlvbiBwbG90cyBkdWUgdG86IHtlfSIpCgogICAgaWYgbGFiZWxfY29sdW1uOgogICAgICAgIGxhYmVscyA9IGRmLnBvcChsYWJlbF9jb2x1bW4pCiAgICAgICAgaW1idGFibGUgPSBsYWJlbHMudmFsdWVfY291bnRzKG5vcm1hbGl6ZT1UcnVlKS5zb3J0X2luZGV4KCkKICAgICAgICB0cnk6CiAgICAgICAgICAgIGdjZl9jbGVhcihwbHQpCiAgICAgICAgICAgIGJhbGFuY2ViYXIgPSBpbWJ0YWJsZS5wbG90KGtpbmQ9ImJhciIsIHRpdGxlPSJjbGFzcyBpbWJhbGFuY2UgLSBsYWJlbHMiKQogICAgICAgICAgICBiYWxhbmNlYmFyLnNldF94bGFiZWwoImNsYXNzIikKICAgICAgICAgICAgYmFsYW5jZWJhci5zZXRfeWxhYmVsKCJwcm9wb3J0aW9uIG9mIHRvdGFsIikKICAgICAgICAgICAgZXh0cmFfZGF0YVsiaW1iYWxhbmNlIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICAgICAgICAgIFBsb3RBcnRpZmFjdCgiaW1iYWxhbmNlIiwgYm9keT1wbHQuZ2NmKCkpLAogICAgICAgICAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9pbWJhbGFuY2UuaHRtbCIsCiAgICAgICAgICAgICkKICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIGNsYXNzIGltYmFsYW5jZSBwbG90IGR1ZSB0bzoge2V9IikKICAgICAgICBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICAgICAgVGFibGVBcnRpZmFjdCgKICAgICAgICAgICAgICAgICJpbWJhbGFuY2Utd2VpZ2h0cy12ZWMiLCBkZj1wZC5EYXRhRnJhbWUoeyJ3ZWlnaHRzIjogaW1idGFibGV9KQogICAgICAgICAgICApLAogICAgICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2ltYmFsYW5jZS13ZWlnaHRzLXZlYy5jc3YiLAogICAgICAgICAgICBkYl9rZXk9RmFsc2UsCiAgICAgICAgKQoKICAgIHRibGNvcnIgPSBkZi5jb3JyKCkKICAgIG1hc2sgPSBucC56ZXJvc19saWtlKHRibGNvcnIsIGR0eXBlPW5wLmJvb2wpCiAgICBtYXNrW25wLnRyaXVfaW5kaWNlc19mcm9tKG1hc2spXSA9IFRydWUKCiAgICBkZmNvcnIgPSBwZC5EYXRhRnJhbWUoZGF0YT10Ymxjb3JyLCBjb2x1bW5zPWhlYWRlciwgaW5kZXg9aGVhZGVyKQogICAgZGZjb3JyID0gZGZjb3JyW25wLmFyYW5nZShkZmNvcnIuc2hhcGVbMF0pWzosIE5vbmVdID4gbnAuYXJhbmdlKGRmY29yci5zaGFwZVsxXSldCiAgICBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBUYWJsZUFydGlmYWN0KCJjb3JyZWxhdGlvbi1tYXRyaXgiLCBkZj10Ymxjb3JyLCB2aXNpYmxlPVRydWUpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vY29ycmVsYXRpb24tbWF0cml4LmNzdiIsCiAgICAgICAgZGJfa2V5PUZhbHNlLAogICAgKQoKICAgIHRyeToKICAgICAgICBnY2ZfY2xlYXIocGx0KQogICAgICAgIGF4ID0gcGx0LmF4ZXMoKQogICAgICAgIHNucy5oZWF0bWFwKHRibGNvcnIsIGF4PWF4LCBtYXNrPW1hc2ssIGFubm90PUZhbHNlLCBjbWFwPXBsdC5jbS5SZWRzKQogICAgICAgIGF4LnNldF90aXRsZSgiZmVhdHVyZXMgY29ycmVsYXRpb24iKQogICAgICAgIGV4dHJhX2RhdGFbImNvcnJlbGF0aW9uIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICAgICAgUGxvdEFydGlmYWN0KCJjb3JyZWxhdGlvbiIsIGJvZHk9cGx0LmdjZigpLCB0aXRsZT0iQ29ycmVsYXRpb24gTWF0cml4IiksCiAgICAgICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vY29yci5odG1sIiwKICAgICAgICAgICAgZGJfa2V5PUZhbHNlLAogICAgICAgICkKICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICBjb250ZXh0LmxvZ2dlci53YXJuKGYiRmFpbGVkIHRvIGNyZWF0ZSBmZWF0dXJlcyBjb3JyZWxhdGlvbiBwbG90IGR1ZSB0bzoge2V9IikKCiAgICBnY2ZfY2xlYXIocGx0KQo= + code_origin: '' + filename: describe_dask.py entry_points: summarize: outputs: - type: None - has_kwargs: false - name: summarize - has_varargs: false - lineno: 28 parameters: - name: context doc: the function context @@ -37,20 +42,16 @@ spec: - name: dask_client doc: dask client object default: null + name: summarize doc: 'Summarize a table Connects to dask client through the function context, or through an optional user-supplied scheduler.' - build: - code_origin: '' - origin_filename: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IG1scnVuCmltcG9ydCB3YXJuaW5ncwp3YXJuaW5ncy5zaW1wbGVmaWx0ZXIoYWN0aW9uPSJpZ25vcmUiLCBjYXRlZ29yeT1GdXR1cmVXYXJuaW5nKQppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCBtYXRwbG90bGliLnB5cGxvdCBhcyBwbHQKaW1wb3J0IHNlYWJvcm4gYXMgc25zCmZyb20gbWxydW4uYXJ0aWZhY3RzIGltcG9ydCBQbG90QXJ0aWZhY3QsIFRhYmxlQXJ0aWZhY3QKZnJvbSBtbHJ1bi5tbHV0aWxzLnBsb3RzIGltcG9ydCBnY2ZfY2xlYXIKaW1wb3J0IG51bXB5IGFzIG5wCgoKcGQuc2V0X29wdGlvbigiZGlzcGxheS5mbG9hdF9mb3JtYXQiLCBsYW1iZGEgeDogIiUuMmYiICUgeCkKCmRlZiBzdW1tYXJpemUoCiAgICBjb250ZXh0LAogICAgZGFza19rZXk6IHN0ciA9ICJkYXNrX2tleSIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW46IHN0ciA9ICJsYWJlbCIsCiAgICBwbG90c19kZXN0OiBzdHIgPSAicGxvdHMiLAogICAgZGFza19mdW5jdGlvbjogc3RyID0gTm9uZSwKICAgIGRhc2tfY2xpZW50PU5vbmUsCikgLT4gTm9uZToKICAgICIiIlN1bW1hcml6ZSBhIHRhYmxlCiAgICAKICAgIENvbm5lY3RzIHRvIGRhc2sgY2xpZW50IHRocm91Z2ggdGhlIGZ1bmN0aW9uIGNvbnRleHQsIG9yIHRocm91Z2ggYW4gb3B0aW9uYWwKICAgIHVzZXItc3VwcGxpZWQgc2NoZWR1bGVyLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIHRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gZGFza19rZXk6ICAgICAgICBrZXkgb2YgZGF0YWZyYW1lIGluIGRhc2sgY2xpZW50ICJkYXRhc2V0cyIgYXR0cmlidXRlCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uOiAgICBncm91bmQgdHJ1dGggY29sdW1uIGxhYmVsCiAgICA6cGFyYW0gcGxvdHNfZGVzdDogICAgICBkZXN0aW5hdGlvbiBmb2xkZXIgb2Ygc3VtbWFyeSBwbG90cyAocmVsYXRpdmUgdG8gYXJ0aWZhY3RfcGF0aCkKICAgIDpwYXJhbSBkYXNrX2Z1bmN0aW9uOiAgIGRhc2sgZnVuY3Rpb24gdXJsIChkYjovLy4uKQogICAgOnBhcmFtIGRhc2tfY2xpZW50OiAgICAgZGFzayBjbGllbnQgb2JqZWN0CiAgICAiIiIKICAgIGlmIGRhc2tfZnVuY3Rpb246CiAgICAgICAgY2xpZW50ID0gbWxydW4uaW1wb3J0X2Z1bmN0aW9uKGRhc2tfZnVuY3Rpb24pLmNsaWVudAogICAgZWxpZiBkYXNrX2NsaWVudDoKICAgICAgICBjbGllbnQgPSBkYXNrX2NsaWVudAogICAgZWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCdkYXNrIGNsaWVudCB3YXMgbm90IHByb3ZpZGVkJykKICAgICAgICAKICAgIGlmIGRhc2tfa2V5IGluIGNsaWVudC5kYXRhc2V0czoKICAgICAgICB0YWJsZSA9IGNsaWVudC5nZXRfZGF0YXNldChkYXNrX2tleSkKICAgIGVsaWYgZGF0YXNldDoKICAgICAgICAjdGFibGUgPSBkYXRhc2V0LmFzX2RmKGRmX21vZHVsZT1kZCkKICAgICAgICB0YWJsZSA9IGRhdGFzZXQuYXNfZGYoKQogICAgZWxzZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYib25seSB0aGVzZSBkYXRhc2V0cyBhcmUgYXZhaWxhYmxlIHtjbGllbnQuZGF0YXNldHN9IGluIGNsaWVudCB7Y2xpZW50fSIpCiAgICAgICAgcmFpc2UgRXhjZXB0aW9uKCJkYXRhc2V0IG5vdCBmb3VuZCBvbiBkYXNrIGNsdXN0ZXIiKQogICAgZGYgPSB0YWJsZQogICAgaGVhZGVyID0gZGYuY29sdW1ucy52YWx1ZXMKICAgIGV4dHJhX2RhdGEgPSB7fQoKICAgIHRyeToKICAgICAgICBnY2ZfY2xlYXIocGx0KQogICAgICAgIHNuc3BsdCA9IHNucy5wYWlycGxvdChkZiwgaHVlPWxhYmVsX2NvbHVtbikgICMgLCBkaWFnX2t3cz17ImJ3IjogMS41fSkKICAgICAgICBleHRyYV9kYXRhWyJoaXN0b2dyYW1zIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICAgICAgUGxvdEFydGlmYWN0KCJoaXN0b2dyYW1zIiwgYm9keT1wbHQuZ2NmKCkpLAogICAgICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2hpc3QuaHRtbCIsCiAgICAgICAgICAgIGRiX2tleT1GYWxzZSwKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoZiJGYWlsZWQgdG8gY3JlYXRlIHBhaXJwbG90IGhpc3RvZ3JhbXMgZHVlIHRvOiB7ZX0iKQoKICAgIHRyeToKICAgICAgICBnY2ZfY2xlYXIocGx0KQogICAgICAgIHBsb3RfY29scyA9IDMKICAgICAgICBwbG90X3Jvd3MgPSBpbnQoKGxlbihoZWFkZXIpIC0gMSkgLyBwbG90X2NvbHMpICsgMQogICAgICAgIGZpZywgYXggPSBwbHQuc3VicGxvdHMocGxvdF9yb3dzLCBwbG90X2NvbHMsIGZpZ3NpemU9KDE1LCA0KSkKICAgICAgICBmaWcudGlnaHRfbGF5b3V0KHBhZD0yLjApCiAgICAgICAgZm9yIGkgaW4gcmFuZ2UocGxvdF9yb3dzICogcGxvdF9jb2xzKToKICAgICAgICAgICAgaWYgaSA8IGxlbihoZWFkZXIpOgogICAgICAgICAgICAgICAgc25zLnZpb2xpbnBsb3QoCiAgICAgICAgICAgICAgICAgICAgeD1kZltoZWFkZXJbaV1dLAogICAgICAgICAgICAgICAgICAgIGF4PWF4W2ludChpIC8gcGxvdF9jb2xzKV1baSAlIHBsb3RfY29sc10sCiAgICAgICAgICAgICAgICAgICAgb3JpZW50PSJoIiwKICAgICAgICAgICAgICAgICAgICB3aWR0aD0wLjcsCiAgICAgICAgICAgICAgICAgICAgaW5uZXI9InF1YXJ0aWxlIiwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIGZpZy5kZWxheGVzKGF4W2ludChpIC8gcGxvdF9jb2xzKV1baSAlIHBsb3RfY29sc10pCiAgICAgICAgICAgIGkgKz0gMQogICAgICAgIGV4dHJhX2RhdGFbInZpb2xpbiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgICAgIFBsb3RBcnRpZmFjdCgidmlvbGluIiwgYm9keT1wbHQuZ2NmKCksIHRpdGxlPSJWaW9saW4gUGxvdCIpLAogICAgICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L3Zpb2xpbi5odG1sIiwKICAgICAgICAgICAgZGJfa2V5PUZhbHNlLAogICAgICAgICkKICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICBjb250ZXh0LmxvZ2dlci53YXJuKGYiRmFpbGVkIHRvIGNyZWF0ZSB2aW9saW4gZGlzdHJpYnV0aW9uIHBsb3RzIGR1ZSB0bzoge2V9IikKCiAgICBpZiBsYWJlbF9jb2x1bW46CiAgICAgICAgbGFiZWxzID0gZGYucG9wKGxhYmVsX2NvbHVtbikKICAgICAgICBpbWJ0YWJsZSA9IGxhYmVscy52YWx1ZV9jb3VudHMobm9ybWFsaXplPVRydWUpLnNvcnRfaW5kZXgoKQogICAgICAgIHRyeToKICAgICAgICAgICAgZ2NmX2NsZWFyKHBsdCkKICAgICAgICAgICAgYmFsYW5jZWJhciA9IGltYnRhYmxlLnBsb3Qoa2luZD0iYmFyIiwgdGl0bGU9ImNsYXNzIGltYmFsYW5jZSAtIGxhYmVscyIpCiAgICAgICAgICAgIGJhbGFuY2ViYXIuc2V0X3hsYWJlbCgiY2xhc3MiKQogICAgICAgICAgICBiYWxhbmNlYmFyLnNldF95bGFiZWwoInByb3BvcnRpb24gb2YgdG90YWwiKQogICAgICAgICAgICBleHRyYV9kYXRhWyJpbWJhbGFuY2UiXSA9IGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgICAgICAgICAgUGxvdEFydGlmYWN0KCJpbWJhbGFuY2UiLCBib2R5PXBsdC5nY2YoKSksCiAgICAgICAgICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2ltYmFsYW5jZS5odG1sIiwKICAgICAgICAgICAgKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgY2xhc3MgaW1iYWxhbmNlIHBsb3QgZHVlIHRvOiB7ZX0iKQogICAgICAgIGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgICAgICBUYWJsZUFydGlmYWN0KAogICAgICAgICAgICAgICAgImltYmFsYW5jZS13ZWlnaHRzLXZlYyIsIGRmPXBkLkRhdGFGcmFtZSh7IndlaWdodHMiOiBpbWJ0YWJsZX0pCiAgICAgICAgICAgICksCiAgICAgICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vaW1iYWxhbmNlLXdlaWdodHMtdmVjLmNzdiIsCiAgICAgICAgICAgIGRiX2tleT1GYWxzZSwKICAgICAgICApCgogICAgdGJsY29yciA9IGRmLmNvcnIoKQogICAgbWFzayA9IG5wLnplcm9zX2xpa2UodGJsY29yciwgZHR5cGU9bnAuYm9vbCkKICAgIG1hc2tbbnAudHJpdV9pbmRpY2VzX2Zyb20obWFzayldID0gVHJ1ZQoKICAgIGRmY29yciA9IHBkLkRhdGFGcmFtZShkYXRhPXRibGNvcnIsIGNvbHVtbnM9aGVhZGVyLCBpbmRleD1oZWFkZXIpCiAgICBkZmNvcnIgPSBkZmNvcnJbbnAuYXJhbmdlKGRmY29yci5zaGFwZVswXSlbOiwgTm9uZV0gPiBucC5hcmFuZ2UoZGZjb3JyLnNoYXBlWzFdKV0KICAgIGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgIFRhYmxlQXJ0aWZhY3QoImNvcnJlbGF0aW9uLW1hdHJpeCIsIGRmPXRibGNvcnIsIHZpc2libGU9VHJ1ZSksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9jb3JyZWxhdGlvbi1tYXRyaXguY3N2IiwKICAgICAgICBkYl9rZXk9RmFsc2UsCiAgICApCgogICAgdHJ5OgogICAgICAgIGdjZl9jbGVhcihwbHQpCiAgICAgICAgYXggPSBwbHQuYXhlcygpCiAgICAgICAgc25zLmhlYXRtYXAodGJsY29yciwgYXg9YXgsIG1hc2s9bWFzaywgYW5ub3Q9RmFsc2UsIGNtYXA9cGx0LmNtLlJlZHMpCiAgICAgICAgYXguc2V0X3RpdGxlKCJmZWF0dXJlcyBjb3JyZWxhdGlvbiIpCiAgICAgICAgZXh0cmFfZGF0YVsiY29ycmVsYXRpb24iXSA9IGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgICAgICBQbG90QXJ0aWZhY3QoImNvcnJlbGF0aW9uIiwgYm9keT1wbHQuZ2NmKCksIHRpdGxlPSJDb3JyZWxhdGlvbiBNYXRyaXgiKSwKICAgICAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9jb3JyLmh0bWwiLAogICAgICAgICAgICBkYl9rZXk9RmFsc2UsCiAgICAgICAgKQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIGZlYXR1cmVzIGNvcnJlbGF0aW9uIHBsb3QgZHVlIHRvOiB7ZX0iKQoKICAgIGdjZl9jbGVhcihwbHQpCg== + has_kwargs: false + has_varargs: false + lineno: 30 + command: '' description: describe and visualizes dataset stats -metadata: - categories: - - data-analysis - tag: '' - name: describe-dask -kind: job + default_handler: summarize diff --git a/functions/src/describe_dask/test_describe_dask.py b/functions/src/describe_dask/test_describe_dask.py index d5c38b71c..c478ac2b7 100644 --- a/functions/src/describe_dask/test_describe_dask.py +++ b/functions/src/describe_dask/test_describe_dask.py @@ -12,21 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from mlrun import code_to_function, new_function, import_function -from pathlib import Path import os -DATA_URL = 'https://s3.wasabisys.com/iguazio/data/iris/iris_dataset.csv' -ARTIFACTS_PATH = 'artifacts' -PLOTS_PATH = ARTIFACTS_PATH + '/plots' +from mlrun import code_to_function, import_function, new_function + +DATA_URL = "https://s3.wasabisys.com/iguazio/data/iris/iris_dataset.csv" +ARTIFACTS_PATH = "artifacts" +PLOTS_PATH = ARTIFACTS_PATH + "/plots" GENERATED_ARTIFACTS = [ - 'correlation', 'correlation-matrix', 'histograms', 'imbalance', 'imbalance-weights-vec', 'violin' + "correlation", + "correlation-matrix", + "histograms", + "imbalance", + "imbalance-weights-vec", + "violin", ] def _create_dask_func(uri): dask_cluster_name = "dask-cluster" - dask_cluster = new_function(dask_cluster_name, kind='dask', image='mlrun/ml-models') + dask_cluster = new_function(dask_cluster_name, kind="dask", image="mlrun/ml-models") dask_cluster.spec.remote = False dask_uri = uri dask_cluster.export(dask_uri) @@ -35,15 +40,15 @@ def _create_dask_func(uri): def test_code_to_function_describe_dask(): dask_uri = "dask_func.yaml" _create_dask_func(dask_uri) - fn = code_to_function(filename="describe_dask.py", kind='local') + fn = code_to_function(filename="describe_dask.py", kind="local") fn.spec.command = "describe_dask.py" run = fn.run( inputs={"dataset": DATA_URL}, params={ - 'update_dataset': True, - 'label_column': 'label', - 'dask_function': dask_uri, + "update_dataset": True, + "label_column": "label", + "dask_function": dask_uri, }, handler="summarize", ) @@ -54,18 +59,17 @@ def test_code_to_function_describe_dask(): def test_import_function_describe_dask(): dask_uri = "dask_func.yaml" _create_dask_func(dask_uri) - fn = import_function('function.yaml') + fn = import_function("function.yaml") run = fn.run( - inputs={ - "dataset": DATA_URL}, + inputs={"dataset": DATA_URL}, params={ - 'update_dataset': True, - 'label_column': 'label', - 'dask_function': dask_uri, + "update_dataset": True, + "label_column": "label", + "dask_function": dask_uri, }, handler="summarize", - artifact_path=os.getcwd() + '/artifacts', + artifact_path=os.getcwd() + "/artifacts", local=True, ) diff --git a/functions/src/describe_spark/describe_spark.py b/functions/src/describe_spark/describe_spark.py index 856b2505c..5e5902781 100644 --- a/functions/src/describe_spark/describe_spark.py +++ b/functions/src/describe_spark/describe_spark.py @@ -14,39 +14,45 @@ # # Generated by nuclio.export.NuclioExporter -import mlrun -from mlrun.platforms.iguazio import mount_v3io, mount_v3iod -from mlrun.datastore import DataItem -from mlrun.execution import MLClientCtx - -import os +import warnings from subprocess import run -import pandas as pd -import numpy as np -from pyspark.sql.types import LongType +import numpy as np +import pandas as pd +from mlrun.datastore import DataItem +from mlrun.execution import MLClientCtx from pyspark.sql import SparkSession -import sys -import base64 as b64 -import warnings warnings.filterwarnings("ignore") +import json from itertools import product -import matplotlib -import numpy as np -import json -import pandas as pd -from matplotlib import pyplot as plt -from pkg_resources import resource_filename -import six +import matplotlib from pyspark.sql import DataFrame as SparkDataFrame -from pyspark.sql.functions import (abs as df_abs, col, count, countDistinct, - max as df_max, mean, min as df_min, - sum as df_sum, when - ) -from pyspark.sql.functions import variance, stddev, kurtosis, skewness +from pyspark.sql.functions import ( + abs as df_abs, +) +from pyspark.sql.functions import ( + col, + count, + countDistinct, + kurtosis, + mean, + skewness, + stddev, + variance, + when, +) +from pyspark.sql.functions import ( + max as df_max, +) +from pyspark.sql.functions import ( + min as df_min, +) +from pyspark.sql.functions import ( + sum as df_sum, +) def describe(df, bins, corr_reject, config, **kwargs): @@ -65,20 +71,20 @@ def describe(df, bins, corr_reject, config, **kwargs): def pretty_name(x): x *= 100 if x == int(x): - return '%.0f%%' % x + return "%.0f%%" % x else: - return '%.1f%%' % x + return "%.1f%%" % x def corr_matrix(df, columns=None): if columns is None: columns = df.columns - combinations = list(product(columns,columns)) + combinations = list(product(columns, columns)) def separate(l, n): for i in range(0, len(l), n): - yield l[i:i+n] + yield l[i : i + n] - grouped = list(separate(combinations,len(columns))) + grouped = list(separate(combinations, len(columns))) df_cleaned = df.select(*columns).na.drop(how="any") for i in grouped: @@ -88,11 +94,10 @@ def separate(l, n): df_pandas = pd.DataFrame(grouped).applymap(lambda x: x[2]) df_pandas.columns = columns df_pandas.index = columns - + return df_pandas def create_hist_data(df, column, minim, maxim, bins=10): - def create_all_conditions(current_col, column, left_edges, count=1): """ Recursive function that exploits the @@ -105,11 +110,14 @@ def create_all_conditions(current_col, column, left_edges, count=1): if len(left_edges) == 1: next_col = current_col.when(col(column) >= float(left_edges[0]), count) left_edges.pop(0) - return create_all_conditions(next_col, column, left_edges[:], count+1) - next_col = current_col.when((float(left_edges[0]) <= col(column)) - & (col(column) < float(left_edges[1])), count) + return create_all_conditions(next_col, column, left_edges[:], count + 1) + next_col = current_col.when( + (float(left_edges[0]) <= col(column)) + & (col(column) < float(left_edges[1])), + count, + ) left_edges.pop(0) - return create_all_conditions(next_col, column, left_edges[:], count+1) + return create_all_conditions(next_col, column, left_edges[:], count + 1) num_range = maxim - minim bin_width = num_range / float(bins) @@ -117,20 +125,25 @@ def create_all_conditions(current_col, column, left_edges, count=1): for _bin in range(bins): left_edges = left_edges + [left_edges[-1] + bin_width] left_edges.pop() - expression_col = when((float(left_edges[0]) <= col(column)) - & (col(column) < float(left_edges[1])), 0) + expression_col = when( + (float(left_edges[0]) <= col(column)) + & (col(column) < float(left_edges[1])), + 0, + ) left_edges_copy = left_edges[:] left_edges_copy.pop(0) - bin_data = (df.select(col(column)) - .na.drop() - .select(col(column), - create_all_conditions(expression_col, - column, - left_edges_copy - ).alias("bin_id") - ) - .groupBy("bin_id").count() - ).toPandas() + bin_data = ( + df.select(col(column)) + .na.drop() + .select( + col(column), + create_all_conditions(expression_col, column, left_edges_copy).alias( + "bin_id" + ), + ) + .groupBy("bin_id") + .count() + ).toPandas() bin_data.index = bin_data["bin_id"] new_index = list(range(bins)) @@ -140,85 +153,102 @@ def create_all_conditions(current_col, column, left_edges, count=1): bin_data["left_edge"] = left_edges bin_data["width"] = bin_width - return bin_data - def describe_integer_1d(df, column, current_result, nrows): - - stats_df = df.select(column).na.drop().agg(mean(col(column)).alias("mean"), - df_min(col(column)).alias("min"), - df_max(col(column)).alias("max"), - variance(col(column)).alias("variance"), - kurtosis(col(column)).alias("kurtosis"), - stddev(col(column)).alias("std"), - skewness(col(column)).alias("skewness"), - df_sum(col(column)).alias("sum") - ).toPandas() - + stats_df = ( + df.select(column) + .na.drop() + .agg( + mean(col(column)).alias("mean"), + df_min(col(column)).alias("min"), + df_max(col(column)).alias("max"), + variance(col(column)).alias("variance"), + kurtosis(col(column)).alias("kurtosis"), + stddev(col(column)).alias("std"), + skewness(col(column)).alias("skewness"), + df_sum(col(column)).alias("sum"), + ) + .toPandas() + ) for x in np.array([0.05, 0.25, 0.5, 0.75, 0.95]): - stats_df[pretty_name(x)] = (df.select(column) - .na.drop() - .selectExpr("percentile(`{col}`,CAST({n} AS DOUBLE))" - .format(col=column, n=x)).toPandas().iloc[:,0] - ) + stats_df[pretty_name(x)] = ( + df.select(column) + .na.drop() + .selectExpr(f"percentile(`{column}`,CAST({x} AS DOUBLE))") + .toPandas() + .iloc[:, 0] + ) stats = stats_df.iloc[0].copy() stats.name = column stats["range"] = stats["max"] - stats["min"] stats["iqr"] = stats[pretty_name(0.75)] - stats[pretty_name(0.25)] stats["cv"] = stats["std"] / float(stats["mean"]) - stats["mad"] = (df.select(column) - .na.drop() - .select(df_abs(col(column)-stats["mean"]).alias("delta")) - .agg(df_sum(col("delta"))).toPandas().iloc[0,0] / float(current_result["count"])) + stats["mad"] = df.select(column).na.drop().select( + df_abs(col(column) - stats["mean"]).alias("delta") + ).agg(df_sum(col("delta"))).toPandas().iloc[0, 0] / float( + current_result["count"] + ) stats["type"] = "NUM" - stats['n_zeros'] = df.select(column).where(col(column)==0.0).count() - stats['p_zeros'] = stats['n_zeros'] / float(nrows) + stats["n_zeros"] = df.select(column).where(col(column) == 0.0).count() + stats["p_zeros"] = stats["n_zeros"] / float(nrows) hist_data = create_hist_data(df, column, stats["min"], stats["max"], bins) return stats def describe_float_1d(df, column, current_result, nrows): - stats_df = df.select(column).na.drop().agg(mean(col(column)).alias("mean"), - df_min(col(column)).alias("min"), - df_max(col(column)).alias("max"), - variance(col(column)).alias("variance"), - kurtosis(col(column)).alias("kurtosis"), - stddev(col(column)).alias("std"), - skewness(col(column)).alias("skewness"), - df_sum(col(column)).alias("sum") - ).toPandas() + stats_df = ( + df.select(column) + .na.drop() + .agg( + mean(col(column)).alias("mean"), + df_min(col(column)).alias("min"), + df_max(col(column)).alias("max"), + variance(col(column)).alias("variance"), + kurtosis(col(column)).alias("kurtosis"), + stddev(col(column)).alias("std"), + skewness(col(column)).alias("skewness"), + df_sum(col(column)).alias("sum"), + ) + .toPandas() + ) for x in np.array([0.05, 0.25, 0.5, 0.75, 0.95]): - stats_df[pretty_name(x)] = (df.select(column) - .na.drop() - .selectExpr("percentile_approx(`{col}`,CAST({n} AS DOUBLE))" - .format(col=column, n=x)).toPandas().iloc[:,0] - ) + stats_df[pretty_name(x)] = ( + df.select(column) + .na.drop() + .selectExpr(f"percentile_approx(`{column}`,CAST({x} AS DOUBLE))") + .toPandas() + .iloc[:, 0] + ) stats = stats_df.iloc[0].copy() stats.name = column stats["range"] = stats["max"] - stats["min"] stats["iqr"] = stats[pretty_name(0.75)] - stats[pretty_name(0.25)] stats["cv"] = stats["std"] / float(stats["mean"]) - stats["mad"] = (df.select(column) - .na.drop() - .select(df_abs(col(column)-stats["mean"]).alias("delta")) - .agg(df_sum(col("delta"))).toPandas().iloc[0,0] / float(current_result["count"])) + stats["mad"] = df.select(column).na.drop().select( + df_abs(col(column) - stats["mean"]).alias("delta") + ).agg(df_sum(col("delta"))).toPandas().iloc[0, 0] / float( + current_result["count"] + ) stats["type"] = "NUM" - stats['n_zeros'] = df.select(column).where(col(column)==0.0).count() - stats['p_zeros'] = stats['n_zeros'] / float(nrows) + stats["n_zeros"] = df.select(column).where(col(column) == 0.0).count() + stats["p_zeros"] = stats["n_zeros"] / float(nrows) hist_data = create_hist_data(df, column, stats["min"], stats["max"], bins) return stats def describe_date_1d(df, column): - stats_df = df.select(column).na.drop().agg(df_min(col(column)).alias("min"), - df_max(col(column)).alias("max") - ).toPandas() + stats_df = ( + df.select(column) + .na.drop() + .agg(df_min(col(column)).alias("min"), df_max(col(column)).alias("max")) + .toPandas() + ) stats = stats_df.iloc[0].copy() stats.name = column @@ -241,66 +271,102 @@ def guess_json_type(string_value): return type(obj) def describe_categorical_1d(df, column): - value_counts = (df.select(column).na.drop() - .groupBy(column) - .agg(count(col(column))) - .orderBy("count({c})".format(c=column),ascending=False) - ).cache() - - stats = (value_counts - .limit(1) - .withColumnRenamed(column, "top") - .withColumnRenamed("count({c})".format(c=column), "freq") - ).toPandas().iloc[0] - - top_50 = value_counts.limit(50).toPandas().sort_values("count({c})".format(c=column), - ascending=False) + value_counts = ( + df.select(column) + .na.drop() + .groupBy(column) + .agg(count(col(column))) + .orderBy(f"count({column})", ascending=False) + ).cache() + + stats = ( + ( + value_counts.limit(1) + .withColumnRenamed(column, "top") + .withColumnRenamed(f"count({column})", "freq") + ) + .toPandas() + .iloc[0] + ) + + top_50 = ( + value_counts.limit(50) + .toPandas() + .sort_values(f"count({column})", ascending=False) + ) top_50_categories = top_50[column].values.tolist() - others_count = pd.Series([df.select(column).na.drop() - .where(~(col(column).isin(*top_50_categories))) - .count() - ], index=["***Other Values***"]) - others_distinct_count = pd.Series([value_counts - .where(~(col(column).isin(*top_50_categories))) - .count() - ], index=["***Other Values Distinct Count***"]) - - top = top_50.set_index(column)["count({c})".format(c=column)] + others_count = pd.Series( + [ + df.select(column) + .na.drop() + .where(~(col(column).isin(*top_50_categories))) + .count() + ], + index=["***Other Values***"], + ) + others_distinct_count = pd.Series( + [value_counts.where(~(col(column).isin(*top_50_categories))).count()], + index=["***Other Values Distinct Count***"], + ) + + top = top_50.set_index(column)[f"count({column})"] top = top.append(others_count) top = top.append(others_distinct_count) stats["value_counts"] = top stats["type"] = "CAT" value_counts.unpersist() - unparsed_valid_jsons = df.select(column).na.drop().rdd.map( - lambda x: guess_json_type(x[column])).filter( - lambda x: x).distinct().collect() + unparsed_valid_jsons = ( + df.select(column) + .na.drop() + .rdd.map(lambda x: guess_json_type(x[column])) + .filter(lambda x: x) + .distinct() + .collect() + ) stats["unparsed_json_types"] = unparsed_valid_jsons return stats def describe_constant_1d(df, column): - stats = pd.Series(['CONST'], index=['type'], name=column) - stats["value_counts"] = (df.select(column) - .na.drop() - .limit(1)).toPandas().iloc[:,0].value_counts() + stats = pd.Series(["CONST"], index=["type"], name=column) + stats["value_counts"] = ( + (df.select(column).na.drop().limit(1)).toPandas().iloc[:, 0].value_counts() + ) return stats def describe_unique_1d(df, column): - stats = pd.Series(['UNIQUE'], index=['type'], name=column) - stats["value_counts"] = (df.select(column) - .na.drop() - .limit(50)).toPandas().iloc[:,0].value_counts() + stats = pd.Series(["UNIQUE"], index=["type"], name=column) + stats["value_counts"] = ( + (df.select(column).na.drop().limit(50)).toPandas().iloc[:, 0].value_counts() + ) return stats def describe_1d(df, column, nrows, lookup_config=None): column_type = df.select(column).dtypes[0][1] - if ("array" in column_type) or ("stuct" in column_type) or ("map" in column_type): - raise NotImplementedError("Column {c} is of type {t} and cannot be analyzed".format(c=column, t=column_type)) - - distinct_count = df.select(column).agg(countDistinct(col(column)).alias("distinct_count")).toPandas() - non_nan_count = df.select(column).na.drop().select(count(col(column)).alias("count")).toPandas() - results_data = pd.concat([distinct_count, non_nan_count],axis=1) - results_data["p_unique"] = results_data["distinct_count"] / float(results_data["count"]) + if ( + ("array" in column_type) + or ("stuct" in column_type) + or ("map" in column_type) + ): + raise NotImplementedError( + f"Column {column} is of type {column_type} and cannot be analyzed" + ) + + distinct_count = ( + df.select(column) + .agg(countDistinct(col(column)).alias("distinct_count")) + .toPandas() + ) + non_nan_count = ( + df.select(column) + .na.drop() + .select(count(col(column)).alias("count")) + .toPandas() + ) + results_data = pd.concat([distinct_count, non_nan_count], axis=1) + results_data["p_unique"] = results_data["distinct_count"] / float( + results_data["count"] + ) results_data["is_unique"] = results_data["distinct_count"] == nrows results_data["n_missing"] = nrows - results_data["count"] results_data["p_missing"] = results_data["n_missing"] / float(nrows) @@ -325,7 +391,7 @@ def describe_1d(df, column, nrows, lookup_config=None): if result["n_missing"] > 0: result["distinct_count"] = result["distinct_count"] + 1 - if (result["count"] > result["distinct_count"] > 1): + if result["count"] > result["distinct_count"] > 1: try: result["mode"] = result["top"] except KeyError: @@ -339,25 +405,34 @@ def describe_1d(df, column, nrows, lookup_config=None): result["mode"] = "MISSING" if lookup_config: - lookup_object = lookup_config['object'] - col_name_in_db = lookup_config['col_name_in_db'] if 'col_name_in_db' in lookup_config else None + lookup_object = lookup_config["object"] + col_name_in_db = ( + lookup_config["col_name_in_db"] + if "col_name_in_db" in lookup_config + else None + ) try: - matched, unmatched = lookup_object.lookup(df.select(column), col_name_in_db) - result['lookedup_values'] = str(matched.count()) + "/" + str(df.select(column).count()) + matched, unmatched = lookup_object.lookup( + df.select(column), col_name_in_db + ) + result["lookedup_values"] = ( + str(matched.count()) + "/" + str(df.select(column).count()) + ) except: - result['lookedup_values'] = 'FAILED' + result["lookedup_values"] = "FAILED" else: - result['lookedup_values'] = '' + result["lookedup_values"] = "" return result - ldesc = {} for colum in df.columns: if colum in config: - if 'lookup' in config[colum]: - lookup_config = config[colum]['lookup'] - desc = describe_1d(df, colum, table_stats["n"], lookup_config=lookup_config) + if "lookup" in config[colum]: + lookup_config = config[colum]["lookup"] + desc = describe_1d( + df, colum, table_stats["n"], lookup_config=lookup_config + ) else: desc = describe_1d(df, colum, table_stats["n"]) else: @@ -377,19 +452,23 @@ def describe_1d(df, column, nrows, lookup_config=None): variable_stats = pd.DataFrame(ldesc) table_stats["nvar"] = len(df.columns) - table_stats["total_missing"] = float(variable_stats.loc["n_missing"].sum()) / (table_stats["n"] * table_stats["nvar"]) + table_stats["total_missing"] = float(variable_stats.loc["n_missing"].sum()) / ( + table_stats["n"] * table_stats["nvar"] + ) memsize = 0 - table_stats['memsize'] = fmt_bytesize(memsize) - table_stats['recordsize'] = fmt_bytesize(memsize / table_stats['n']) - table_stats.update({k: 0 for k in ("NUM", "DATE", "CONST", "CAT", "UNIQUE", "CORR")}) - table_stats.update(dict(variable_stats.loc['type'].value_counts())) - table_stats['REJECTED'] = table_stats['CONST'] + table_stats['CORR'] + table_stats["memsize"] = fmt_bytesize(memsize) + table_stats["recordsize"] = fmt_bytesize(memsize / table_stats["n"]) + table_stats.update( + {k: 0 for k in ("NUM", "DATE", "CONST", "CAT", "UNIQUE", "CORR")} + ) + table_stats.update(dict(variable_stats.loc["type"].value_counts())) + table_stats["REJECTED"] = table_stats["CONST"] + table_stats["CORR"] freq_dict = {} for var in variable_stats: if "value_counts" not in variable_stats[var]: pass - elif not(variable_stats[var]["value_counts"] is np.nan): + elif variable_stats[var]["value_counts"] is not np.nan: freq_dict[var] = variable_stats[var]["value_counts"] else: pass @@ -400,129 +479,155 @@ def describe_1d(df, column, nrows, lookup_config=None): return table_stats, variable_stats.T, freq_dict -import numpy as np -from pyspark.sql.functions import abs as absou SKEWNESS_CUTOFF = 20 -DEFAULT_FLOAT_FORMATTER = u'spark_df_profiling.__default_float_formatter' +DEFAULT_FLOAT_FORMATTER = "spark_df_profiling.__default_float_formatter" def gradient_format(value, limit1, limit2, c1, c2): - def LerpColour(c1,c2,t): - return (int(c1[0]+(c2[0]-c1[0])*t),int(c1[1]+(c2[1]-c1[1])*t),int(c1[2]+(c2[2]-c1[2])*t)) - c = LerpColour(c1, c2, (value-limit1)/(limit2-limit1)) - return fmt_color(value,"rgb{}".format(str(c))) + def LerpColour(c1, c2, t): + return ( + int(c1[0] + (c2[0] - c1[0]) * t), + int(c1[1] + (c2[1] - c1[1]) * t), + int(c1[2] + (c2[2] - c1[2]) * t), + ) + + c = LerpColour(c1, c2, (value - limit1) / (limit2 - limit1)) + return fmt_color(value, f"rgb{str(c)}") def fmt_color(text, color): - return(u'{text}'.format(color=color,text=str(text))) + return f'{str(text)}' def fmt_class(text, cls): - return(u'{text}'.format(cls=cls,text=str(text))) + return f'{str(text)}' -def fmt_bytesize(num, suffix='B'): - for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']: +def fmt_bytesize(num, suffix="B"): + for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: if num < 0: - num = num*-1 + num = num * -1 if num < 1024.0: return "%3.1f %s%s" % (num, unit, suffix) num /= 1024.0 - return "%.1f %s%s" % (num, 'Yi', suffix) + return "%.1f %s%s" % (num, "Yi", suffix) def fmt_percent(v): - return "{:2.1f}%".format(v*100) + return f"{v * 100:2.1f}%" + def fmt_varname(v): - return u'{0}'.format(v) - - -value_formatters={ - u'freq': (lambda v: gradient_format(v, 0, 62000, (30, 198, 244), (99, 200, 72))), - u'p_missing': fmt_percent, - u'p_infinite': fmt_percent, - u'p_unique': fmt_percent, - u'p_zeros': fmt_percent, - u'memorysize': fmt_bytesize, - u'total_missing': fmt_percent, - DEFAULT_FLOAT_FORMATTER: lambda v: str(float('{:.5g}'.format(v))).rstrip('0').rstrip('.'), - u'correlation_var': lambda v: fmt_varname(v), - u'unparsed_json_types': lambda v: ', '.join([s.__name__ for s in v]) - } + return f"{v}" + + +value_formatters = { + "freq": (lambda v: gradient_format(v, 0, 62000, (30, 198, 244), (99, 200, 72))), + "p_missing": fmt_percent, + "p_infinite": fmt_percent, + "p_unique": fmt_percent, + "p_zeros": fmt_percent, + "memorysize": fmt_bytesize, + "total_missing": fmt_percent, + DEFAULT_FLOAT_FORMATTER: lambda v: str(float(f"{v:.5g}")).rstrip("0").rstrip("."), + "correlation_var": lambda v: fmt_varname(v), + "unparsed_json_types": lambda v: ", ".join([s.__name__ for s in v]), +} + def fmt_row_severity(v): - if np.isnan(v) or v<= 0.01: + if np.isnan(v) or v <= 0.01: return "ignore" else: return "alert" + def fmt_skewness(v): - if not np.isnan(v) and (v<-SKEWNESS_CUTOFF or v> SKEWNESS_CUTOFF): + if not np.isnan(v) and (v < -SKEWNESS_CUTOFF or v > SKEWNESS_CUTOFF): return "alert" else: return "" -row_formatters={ - u'p_zeros': fmt_row_severity, - u'p_missing': fmt_row_severity, - u'p_infinite': fmt_row_severity, - u'n_duplicates': fmt_row_severity, - u'skewness': fmt_skewness, + +row_formatters = { + "p_zeros": fmt_row_severity, + "p_missing": fmt_row_severity, + "p_infinite": fmt_row_severity, + "n_duplicates": fmt_row_severity, + "skewness": fmt_skewness, } run(["/bin/bash", "/etc/config/v3io/v3io-spark-operator.sh"]) -def describe_spark(context: MLClientCtx, - dataset: DataItem, - artifact_path, - bins: int=30, - describe_extended: bool=True): - + +def describe_spark( + context: MLClientCtx, + dataset: DataItem, + artifact_path, + bins: int = 30, + describe_extended: bool = True, +): location = dataset.local() - + spark = SparkSession.builder.appName("Spark job").getOrCreate() - - df = spark.read.csv(location, header=True, inferSchema= True) + + df = spark.read.csv(location, header=True, inferSchema=True) kwargs = [] - - float_cols = [item[0] for item in df.dtypes if item[1].startswith('float') or item[1].startswith('double')] - + + float_cols = [ + item[0] + for item in df.dtypes + if item[1].startswith("float") or item[1].startswith("double") + ] + if describe_extended == True: - table, variables, freq = describe(df, bins, float_cols, kwargs) tbl_1 = variables.reset_index() if len(freq) != 0: - tbl_2 = pd.DataFrame.from_dict(freq, orient = "index").sort_index().stack().reset_index() - tbl_2.columns = ['col', 'key', 'val'] - tbl_2['Merged'] = [{key: val} for key, val in zip(tbl_2.key, tbl_2.val)] - tbl_2 = tbl_2.groupby('col', as_index=False).agg(lambda x: tuple(x))[['col','Merged']] - - summary = pd.merge(tbl_1, tbl_2, how='left', left_on='index', right_on='col') + tbl_2 = ( + pd.DataFrame.from_dict(freq, orient="index") + .sort_index() + .stack() + .reset_index() + ) + tbl_2.columns = ["col", "key", "val"] + tbl_2["Merged"] = [{key: val} for key, val in zip(tbl_2.key, tbl_2.val)] + tbl_2 = tbl_2.groupby("col", as_index=False).agg(lambda x: tuple(x))[ + ["col", "Merged"] + ] + + summary = pd.merge( + tbl_1, tbl_2, how="left", left_on="index", right_on="col" + ) else: summary = tbl_1 - context.log_dataset("summary_stats", - df=summary, - format="csv", index=False, - artifact_path=context.artifact_subpath('data')) + context.log_dataset( + "summary_stats", + df=summary, + format="csv", + index=False, + artifact_path=context.artifact_subpath("data"), + ) context.log_results(table) - + else: tbl_1 = df.describe().toPandas() - + summary = tbl_1.T - - context.log_dataset("summary_stats", - df=summary, - format="csv", index=False, - artifact_path=context.artifact_subpath('data')) - - spark.stop() + context.log_dataset( + "summary_stats", + df=summary, + format="csv", + index=False, + artifact_path=context.artifact_subpath("data"), + ) + + spark.stop() diff --git a/functions/src/describe_spark/function.yaml b/functions/src/describe_spark/function.yaml index 688f4260b..12223e77c 100644 --- a/functions/src/describe_spark/function.yaml +++ b/functions/src/describe_spark/function.yaml @@ -1,322 +1,264 @@ -kind: job metadata: - name: describe-spark tag: '' - hash: bd54bbf6350fb0dc392ff7f91b4aa6ea3c742e93 - project: '' + name: describe-spark categories: - data-analysis +verbose: false +kind: job spec: - command: '' - args: [] image: iguazio/shell:3.0_b5565_20201026062233_wsdf - env: [] - default_handler: describe_spark + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCmZyb20gc3VicHJvY2VzcyBpbXBvcnQgcnVuCgppbXBvcnQgbnVtcHkgYXMgbnAKaW1wb3J0IHBhbmRhcyBhcyBwZApmcm9tIG1scnVuLmRhdGFzdG9yZSBpbXBvcnQgRGF0YUl0ZW0KZnJvbSBtbHJ1bi5leGVjdXRpb24gaW1wb3J0IE1MQ2xpZW50Q3R4CmZyb20gcHlzcGFyay5zcWwgaW1wb3J0IFNwYXJrU2Vzc2lvbgoKd2FybmluZ3MuZmlsdGVyd2FybmluZ3MoImlnbm9yZSIpCgppbXBvcnQganNvbgpmcm9tIGl0ZXJ0b29scyBpbXBvcnQgcHJvZHVjdAoKaW1wb3J0IG1hdHBsb3RsaWIKZnJvbSBweXNwYXJrLnNxbCBpbXBvcnQgRGF0YUZyYW1lIGFzIFNwYXJrRGF0YUZyYW1lCmZyb20gcHlzcGFyay5zcWwuZnVuY3Rpb25zIGltcG9ydCAoCiAgICBhYnMgYXMgZGZfYWJzLAopCmZyb20gcHlzcGFyay5zcWwuZnVuY3Rpb25zIGltcG9ydCAoCiAgICBjb2wsCiAgICBjb3VudCwKICAgIGNvdW50RGlzdGluY3QsCiAgICBrdXJ0b3NpcywKICAgIG1lYW4sCiAgICBza2V3bmVzcywKICAgIHN0ZGRldiwKICAgIHZhcmlhbmNlLAogICAgd2hlbiwKKQpmcm9tIHB5c3Bhcmsuc3FsLmZ1bmN0aW9ucyBpbXBvcnQgKAogICAgbWF4IGFzIGRmX21heCwKKQpmcm9tIHB5c3Bhcmsuc3FsLmZ1bmN0aW9ucyBpbXBvcnQgKAogICAgbWluIGFzIGRmX21pbiwKKQpmcm9tIHB5c3Bhcmsuc3FsLmZ1bmN0aW9ucyBpbXBvcnQgKAogICAgc3VtIGFzIGRmX3N1bSwKKQoKCmRlZiBkZXNjcmliZShkZiwgYmlucywgY29ycl9yZWplY3QsIGNvbmZpZywgKiprd2FyZ3MpOgogICAgaWYgbm90IGlzaW5zdGFuY2UoZGYsIFNwYXJrRGF0YUZyYW1lKToKICAgICAgICByYWlzZSBUeXBlRXJyb3IoImRmIG11c3QgYmUgb2YgdHlwZSBweXNwYXJrLnNxbC5EYXRhRnJhbWUiKQoKICAgIHRhYmxlX3N0YXRzID0geyJuIjogZGYuY291bnQoKX0KICAgIGlmIHRhYmxlX3N0YXRzWyJuIl0gPT0gMDoKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJkZiBjYW5ub3QgYmUgZW1wdHkiKQoKICAgIHRyeToKICAgICAgICBtYXRwbG90bGliLnN0eWxlLnVzZSgiZGVmYXVsdCIpCiAgICBleGNlcHQ6CiAgICAgICAgcGFzcwoKICAgIGRlZiBwcmV0dHlfbmFtZSh4KToKICAgICAgICB4ICo9IDEwMAogICAgICAgIGlmIHggPT0gaW50KHgpOgogICAgICAgICAgICByZXR1cm4gIiUuMGYlJSIgJSB4CiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmV0dXJuICIlLjFmJSUiICUgeAoKICAgIGRlZiBjb3JyX21hdHJpeChkZiwgY29sdW1ucz1Ob25lKToKICAgICAgICBpZiBjb2x1bW5zIGlzIE5vbmU6CiAgICAgICAgICAgIGNvbHVtbnMgPSBkZi5jb2x1bW5zCiAgICAgICAgY29tYmluYXRpb25zID0gbGlzdChwcm9kdWN0KGNvbHVtbnMsIGNvbHVtbnMpKQoKICAgICAgICBkZWYgc2VwYXJhdGUobCwgbik6CiAgICAgICAgICAgIGZvciBpIGluIHJhbmdlKDAsIGxlbihsKSwgbik6CiAgICAgICAgICAgICAgICB5aWVsZCBsW2kgOiBpICsgbl0KCiAgICAgICAgZ3JvdXBlZCA9IGxpc3Qoc2VwYXJhdGUoY29tYmluYXRpb25zLCBsZW4oY29sdW1ucykpKQogICAgICAgIGRmX2NsZWFuZWQgPSBkZi5zZWxlY3QoKmNvbHVtbnMpLm5hLmRyb3AoaG93PSJhbnkiKQoKICAgICAgICBmb3IgaSBpbiBncm91cGVkOgogICAgICAgICAgICBmb3IgaiBpbiBlbnVtZXJhdGUoaSk6CiAgICAgICAgICAgICAgICBpW2pbMF1dID0gaVtqWzBdXSArIChkZl9jbGVhbmVkLmNvcnIoc3RyKGpbMV1bMF0pLCBzdHIoalsxXVsxXSkpLCkKCiAgICAgICAgZGZfcGFuZGFzID0gcGQuRGF0YUZyYW1lKGdyb3VwZWQpLmFwcGx5bWFwKGxhbWJkYSB4OiB4WzJdKQogICAgICAgIGRmX3BhbmRhcy5jb2x1bW5zID0gY29sdW1ucwogICAgICAgIGRmX3BhbmRhcy5pbmRleCA9IGNvbHVtbnMKCiAgICAgICAgcmV0dXJuIGRmX3BhbmRhcwoKICAgIGRlZiBjcmVhdGVfaGlzdF9kYXRhKGRmLCBjb2x1bW4sIG1pbmltLCBtYXhpbSwgYmlucz0xMCk6CiAgICAgICAgZGVmIGNyZWF0ZV9hbGxfY29uZGl0aW9ucyhjdXJyZW50X2NvbCwgY29sdW1uLCBsZWZ0X2VkZ2VzLCBjb3VudD0xKToKICAgICAgICAgICAgIiIiCiAgICAgICAgICAgIFJlY3Vyc2l2ZSBmdW5jdGlvbiB0aGF0IGV4cGxvaXRzIHRoZQogICAgICAgICAgICBhYmlsaXR5IHRvIGNhbGwgdGhlIFNwYXJrIFNRTCBDb2x1bW4gbWV0aG9kCiAgICAgICAgICAgIC53aGVuKCkgaW4gYSByZWN1cnNpdmUgd2F5LgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgbGVmdF9lZGdlcyA9IGxlZnRfZWRnZXNbOl0KICAgICAgICAgICAgaWYgbGVuKGxlZnRfZWRnZXMpID09IDA6CiAgICAgICAgICAgICAgICByZXR1cm4gY3VycmVudF9jb2wKICAgICAgICAgICAgaWYgbGVuKGxlZnRfZWRnZXMpID09IDE6CiAgICAgICAgICAgICAgICBuZXh0X2NvbCA9IGN1cnJlbnRfY29sLndoZW4oY29sKGNvbHVtbikgPj0gZmxvYXQobGVmdF9lZGdlc1swXSksIGNvdW50KQogICAgICAgICAgICAgICAgbGVmdF9lZGdlcy5wb3AoMCkKICAgICAgICAgICAgICAgIHJldHVybiBjcmVhdGVfYWxsX2NvbmRpdGlvbnMobmV4dF9jb2wsIGNvbHVtbiwgbGVmdF9lZGdlc1s6XSwgY291bnQgKyAxKQogICAgICAgICAgICBuZXh0X2NvbCA9IGN1cnJlbnRfY29sLndoZW4oCiAgICAgICAgICAgICAgICAoZmxvYXQobGVmdF9lZGdlc1swXSkgPD0gY29sKGNvbHVtbikpCiAgICAgICAgICAgICAgICAmIChjb2woY29sdW1uKSA8IGZsb2F0KGxlZnRfZWRnZXNbMV0pKSwKICAgICAgICAgICAgICAgIGNvdW50LAogICAgICAgICAgICApCiAgICAgICAgICAgIGxlZnRfZWRnZXMucG9wKDApCiAgICAgICAgICAgIHJldHVybiBjcmVhdGVfYWxsX2NvbmRpdGlvbnMobmV4dF9jb2wsIGNvbHVtbiwgbGVmdF9lZGdlc1s6XSwgY291bnQgKyAxKQoKICAgICAgICBudW1fcmFuZ2UgPSBtYXhpbSAtIG1pbmltCiAgICAgICAgYmluX3dpZHRoID0gbnVtX3JhbmdlIC8gZmxvYXQoYmlucykKICAgICAgICBsZWZ0X2VkZ2VzID0gW21pbmltXQogICAgICAgIGZvciBfYmluIGluIHJhbmdlKGJpbnMpOgogICAgICAgICAgICBsZWZ0X2VkZ2VzID0gbGVmdF9lZGdlcyArIFtsZWZ0X2VkZ2VzWy0xXSArIGJpbl93aWR0aF0KICAgICAgICBsZWZ0X2VkZ2VzLnBvcCgpCiAgICAgICAgZXhwcmVzc2lvbl9jb2wgPSB3aGVuKAogICAgICAgICAgICAoZmxvYXQobGVmdF9lZGdlc1swXSkgPD0gY29sKGNvbHVtbikpCiAgICAgICAgICAgICYgKGNvbChjb2x1bW4pIDwgZmxvYXQobGVmdF9lZGdlc1sxXSkpLAogICAgICAgICAgICAwLAogICAgICAgICkKICAgICAgICBsZWZ0X2VkZ2VzX2NvcHkgPSBsZWZ0X2VkZ2VzWzpdCiAgICAgICAgbGVmdF9lZGdlc19jb3B5LnBvcCgwKQogICAgICAgIGJpbl9kYXRhID0gKAogICAgICAgICAgICBkZi5zZWxlY3QoY29sKGNvbHVtbikpCiAgICAgICAgICAgIC5uYS5kcm9wKCkKICAgICAgICAgICAgLnNlbGVjdCgKICAgICAgICAgICAgICAgIGNvbChjb2x1bW4pLAogICAgICAgICAgICAgICAgY3JlYXRlX2FsbF9jb25kaXRpb25zKGV4cHJlc3Npb25fY29sLCBjb2x1bW4sIGxlZnRfZWRnZXNfY29weSkuYWxpYXMoCiAgICAgICAgICAgICAgICAgICAgImJpbl9pZCIKICAgICAgICAgICAgICAgICksCiAgICAgICAgICAgICkKICAgICAgICAgICAgLmdyb3VwQnkoImJpbl9pZCIpCiAgICAgICAgICAgIC5jb3VudCgpCiAgICAgICAgKS50b1BhbmRhcygpCgogICAgICAgIGJpbl9kYXRhLmluZGV4ID0gYmluX2RhdGFbImJpbl9pZCJdCiAgICAgICAgbmV3X2luZGV4ID0gbGlzdChyYW5nZShiaW5zKSkKICAgICAgICBiaW5fZGF0YSA9IGJpbl9kYXRhLnJlaW5kZXgobmV3X2luZGV4KQogICAgICAgIGJpbl9kYXRhWyJiaW5faWQiXSA9IGJpbl9kYXRhLmluZGV4CiAgICAgICAgYmluX2RhdGEgPSBiaW5fZGF0YS5maWxsbmEoMCkKCiAgICAgICAgYmluX2RhdGFbImxlZnRfZWRnZSJdID0gbGVmdF9lZGdlcwogICAgICAgIGJpbl9kYXRhWyJ3aWR0aCJdID0gYmluX3dpZHRoCgogICAgICAgIHJldHVybiBiaW5fZGF0YQoKICAgIGRlZiBkZXNjcmliZV9pbnRlZ2VyXzFkKGRmLCBjb2x1bW4sIGN1cnJlbnRfcmVzdWx0LCBucm93cyk6CiAgICAgICAgc3RhdHNfZGYgPSAoCiAgICAgICAgICAgIGRmLnNlbGVjdChjb2x1bW4pCiAgICAgICAgICAgIC5uYS5kcm9wKCkKICAgICAgICAgICAgLmFnZygKICAgICAgICAgICAgICAgIG1lYW4oY29sKGNvbHVtbikpLmFsaWFzKCJtZWFuIiksCiAgICAgICAgICAgICAgICBkZl9taW4oY29sKGNvbHVtbikpLmFsaWFzKCJtaW4iKSwKICAgICAgICAgICAgICAgIGRmX21heChjb2woY29sdW1uKSkuYWxpYXMoIm1heCIpLAogICAgICAgICAgICAgICAgdmFyaWFuY2UoY29sKGNvbHVtbikpLmFsaWFzKCJ2YXJpYW5jZSIpLAogICAgICAgICAgICAgICAga3VydG9zaXMoY29sKGNvbHVtbikpLmFsaWFzKCJrdXJ0b3NpcyIpLAogICAgICAgICAgICAgICAgc3RkZGV2KGNvbChjb2x1bW4pKS5hbGlhcygic3RkIiksCiAgICAgICAgICAgICAgICBza2V3bmVzcyhjb2woY29sdW1uKSkuYWxpYXMoInNrZXduZXNzIiksCiAgICAgICAgICAgICAgICBkZl9zdW0oY29sKGNvbHVtbikpLmFsaWFzKCJzdW0iKSwKICAgICAgICAgICAgKQogICAgICAgICAgICAudG9QYW5kYXMoKQogICAgICAgICkKCiAgICAgICAgZm9yIHggaW4gbnAuYXJyYXkoWzAuMDUsIDAuMjUsIDAuNSwgMC43NSwgMC45NV0pOgogICAgICAgICAgICBzdGF0c19kZltwcmV0dHlfbmFtZSh4KV0gPSAoCiAgICAgICAgICAgICAgICBkZi5zZWxlY3QoY29sdW1uKQogICAgICAgICAgICAgICAgLm5hLmRyb3AoKQogICAgICAgICAgICAgICAgLnNlbGVjdEV4cHIoZiJwZXJjZW50aWxlKGB7Y29sdW1ufWAsQ0FTVCh7eH0gQVMgRE9VQkxFKSkiKQogICAgICAgICAgICAgICAgLnRvUGFuZGFzKCkKICAgICAgICAgICAgICAgIC5pbG9jWzosIDBdCiAgICAgICAgICAgICkKICAgICAgICBzdGF0cyA9IHN0YXRzX2RmLmlsb2NbMF0uY29weSgpCiAgICAgICAgc3RhdHMubmFtZSA9IGNvbHVtbgogICAgICAgIHN0YXRzWyJyYW5nZSJdID0gc3RhdHNbIm1heCJdIC0gc3RhdHNbIm1pbiJdCiAgICAgICAgc3RhdHNbImlxciJdID0gc3RhdHNbcHJldHR5X25hbWUoMC43NSldIC0gc3RhdHNbcHJldHR5X25hbWUoMC4yNSldCiAgICAgICAgc3RhdHNbImN2Il0gPSBzdGF0c1sic3RkIl0gLyBmbG9hdChzdGF0c1sibWVhbiJdKQogICAgICAgIHN0YXRzWyJtYWQiXSA9IGRmLnNlbGVjdChjb2x1bW4pLm5hLmRyb3AoKS5zZWxlY3QoCiAgICAgICAgICAgIGRmX2Ficyhjb2woY29sdW1uKSAtIHN0YXRzWyJtZWFuIl0pLmFsaWFzKCJkZWx0YSIpCiAgICAgICAgKS5hZ2coZGZfc3VtKGNvbCgiZGVsdGEiKSkpLnRvUGFuZGFzKCkuaWxvY1swLCAwXSAvIGZsb2F0KAogICAgICAgICAgICBjdXJyZW50X3Jlc3VsdFsiY291bnQiXQogICAgICAgICkKICAgICAgICBzdGF0c1sidHlwZSJdID0gIk5VTSIKICAgICAgICBzdGF0c1sibl96ZXJvcyJdID0gZGYuc2VsZWN0KGNvbHVtbikud2hlcmUoY29sKGNvbHVtbikgPT0gMC4wKS5jb3VudCgpCiAgICAgICAgc3RhdHNbInBfemVyb3MiXSA9IHN0YXRzWyJuX3plcm9zIl0gLyBmbG9hdChucm93cykKCiAgICAgICAgaGlzdF9kYXRhID0gY3JlYXRlX2hpc3RfZGF0YShkZiwgY29sdW1uLCBzdGF0c1sibWluIl0sIHN0YXRzWyJtYXgiXSwgYmlucykKCiAgICAgICAgcmV0dXJuIHN0YXRzCgogICAgZGVmIGRlc2NyaWJlX2Zsb2F0XzFkKGRmLCBjb2x1bW4sIGN1cnJlbnRfcmVzdWx0LCBucm93cyk6CiAgICAgICAgc3RhdHNfZGYgPSAoCiAgICAgICAgICAgIGRmLnNlbGVjdChjb2x1bW4pCiAgICAgICAgICAgIC5uYS5kcm9wKCkKICAgICAgICAgICAgLmFnZygKICAgICAgICAgICAgICAgIG1lYW4oY29sKGNvbHVtbikpLmFsaWFzKCJtZWFuIiksCiAgICAgICAgICAgICAgICBkZl9taW4oY29sKGNvbHVtbikpLmFsaWFzKCJtaW4iKSwKICAgICAgICAgICAgICAgIGRmX21heChjb2woY29sdW1uKSkuYWxpYXMoIm1heCIpLAogICAgICAgICAgICAgICAgdmFyaWFuY2UoY29sKGNvbHVtbikpLmFsaWFzKCJ2YXJpYW5jZSIpLAogICAgICAgICAgICAgICAga3VydG9zaXMoY29sKGNvbHVtbikpLmFsaWFzKCJrdXJ0b3NpcyIpLAogICAgICAgICAgICAgICAgc3RkZGV2KGNvbChjb2x1bW4pKS5hbGlhcygic3RkIiksCiAgICAgICAgICAgICAgICBza2V3bmVzcyhjb2woY29sdW1uKSkuYWxpYXMoInNrZXduZXNzIiksCiAgICAgICAgICAgICAgICBkZl9zdW0oY29sKGNvbHVtbikpLmFsaWFzKCJzdW0iKSwKICAgICAgICAgICAgKQogICAgICAgICAgICAudG9QYW5kYXMoKQogICAgICAgICkKCiAgICAgICAgZm9yIHggaW4gbnAuYXJyYXkoWzAuMDUsIDAuMjUsIDAuNSwgMC43NSwgMC45NV0pOgogICAgICAgICAgICBzdGF0c19kZltwcmV0dHlfbmFtZSh4KV0gPSAoCiAgICAgICAgICAgICAgICBkZi5zZWxlY3QoY29sdW1uKQogICAgICAgICAgICAgICAgLm5hLmRyb3AoKQogICAgICAgICAgICAgICAgLnNlbGVjdEV4cHIoZiJwZXJjZW50aWxlX2FwcHJveChge2NvbHVtbn1gLENBU1Qoe3h9IEFTIERPVUJMRSkpIikKICAgICAgICAgICAgICAgIC50b1BhbmRhcygpCiAgICAgICAgICAgICAgICAuaWxvY1s6LCAwXQogICAgICAgICAgICApCiAgICAgICAgc3RhdHMgPSBzdGF0c19kZi5pbG9jWzBdLmNvcHkoKQogICAgICAgIHN0YXRzLm5hbWUgPSBjb2x1bW4KICAgICAgICBzdGF0c1sicmFuZ2UiXSA9IHN0YXRzWyJtYXgiXSAtIHN0YXRzWyJtaW4iXQogICAgICAgIHN0YXRzWyJpcXIiXSA9IHN0YXRzW3ByZXR0eV9uYW1lKDAuNzUpXSAtIHN0YXRzW3ByZXR0eV9uYW1lKDAuMjUpXQogICAgICAgIHN0YXRzWyJjdiJdID0gc3RhdHNbInN0ZCJdIC8gZmxvYXQoc3RhdHNbIm1lYW4iXSkKICAgICAgICBzdGF0c1sibWFkIl0gPSBkZi5zZWxlY3QoY29sdW1uKS5uYS5kcm9wKCkuc2VsZWN0KAogICAgICAgICAgICBkZl9hYnMoY29sKGNvbHVtbikgLSBzdGF0c1sibWVhbiJdKS5hbGlhcygiZGVsdGEiKQogICAgICAgICkuYWdnKGRmX3N1bShjb2woImRlbHRhIikpKS50b1BhbmRhcygpLmlsb2NbMCwgMF0gLyBmbG9hdCgKICAgICAgICAgICAgY3VycmVudF9yZXN1bHRbImNvdW50Il0KICAgICAgICApCiAgICAgICAgc3RhdHNbInR5cGUiXSA9ICJOVU0iCiAgICAgICAgc3RhdHNbIm5femVyb3MiXSA9IGRmLnNlbGVjdChjb2x1bW4pLndoZXJlKGNvbChjb2x1bW4pID09IDAuMCkuY291bnQoKQogICAgICAgIHN0YXRzWyJwX3plcm9zIl0gPSBzdGF0c1sibl96ZXJvcyJdIC8gZmxvYXQobnJvd3MpCgogICAgICAgIGhpc3RfZGF0YSA9IGNyZWF0ZV9oaXN0X2RhdGEoZGYsIGNvbHVtbiwgc3RhdHNbIm1pbiJdLCBzdGF0c1sibWF4Il0sIGJpbnMpCgogICAgICAgIHJldHVybiBzdGF0cwoKICAgIGRlZiBkZXNjcmliZV9kYXRlXzFkKGRmLCBjb2x1bW4pOgogICAgICAgIHN0YXRzX2RmID0gKAogICAgICAgICAgICBkZi5zZWxlY3QoY29sdW1uKQogICAgICAgICAgICAubmEuZHJvcCgpCiAgICAgICAgICAgIC5hZ2coZGZfbWluKGNvbChjb2x1bW4pKS5hbGlhcygibWluIiksIGRmX21heChjb2woY29sdW1uKSkuYWxpYXMoIm1heCIpKQogICAgICAgICAgICAudG9QYW5kYXMoKQogICAgICAgICkKICAgICAgICBzdGF0cyA9IHN0YXRzX2RmLmlsb2NbMF0uY29weSgpCiAgICAgICAgc3RhdHMubmFtZSA9IGNvbHVtbgoKICAgICAgICBpZiBpc2luc3RhbmNlKHN0YXRzWyJtYXgiXSwgcGQuVGltZXN0YW1wKToKICAgICAgICAgICAgc3RhdHMgPSBzdGF0cy5hc3R5cGUob2JqZWN0KQogICAgICAgICAgICBzdGF0c1sibWF4Il0gPSBzdHIoc3RhdHNbIm1heCJdLnRvX3B5ZGF0ZXRpbWUoKSkKICAgICAgICAgICAgc3RhdHNbIm1pbiJdID0gc3RyKHN0YXRzWyJtaW4iXS50b19weWRhdGV0aW1lKCkpCgogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHN0YXRzWyJyYW5nZSJdID0gc3RhdHNbIm1heCJdIC0gc3RhdHNbIm1pbiJdCiAgICAgICAgc3RhdHNbInR5cGUiXSA9ICJEQVRFIgogICAgICAgIHJldHVybiBzdGF0cwoKICAgIGRlZiBndWVzc19qc29uX3R5cGUoc3RyaW5nX3ZhbHVlKToKICAgICAgICB0cnk6CiAgICAgICAgICAgIG9iaiA9IGpzb24ubG9hZHMoc3RyaW5nX3ZhbHVlKQogICAgICAgIGV4Y2VwdDoKICAgICAgICAgICAgcmV0dXJuIE5vbmUKCiAgICAgICAgcmV0dXJuIHR5cGUob2JqKQoKICAgIGRlZiBkZXNjcmliZV9jYXRlZ29yaWNhbF8xZChkZiwgY29sdW1uKToKICAgICAgICB2YWx1ZV9jb3VudHMgPSAoCiAgICAgICAgICAgIGRmLnNlbGVjdChjb2x1bW4pCiAgICAgICAgICAgIC5uYS5kcm9wKCkKICAgICAgICAgICAgLmdyb3VwQnkoY29sdW1uKQogICAgICAgICAgICAuYWdnKGNvdW50KGNvbChjb2x1bW4pKSkKICAgICAgICAgICAgLm9yZGVyQnkoZiJjb3VudCh7Y29sdW1ufSkiLCBhc2NlbmRpbmc9RmFsc2UpCiAgICAgICAgKS5jYWNoZSgpCgogICAgICAgIHN0YXRzID0gKAogICAgICAgICAgICAoCiAgICAgICAgICAgICAgICB2YWx1ZV9jb3VudHMubGltaXQoMSkKICAgICAgICAgICAgICAgIC53aXRoQ29sdW1uUmVuYW1lZChjb2x1bW4sICJ0b3AiKQogICAgICAgICAgICAgICAgLndpdGhDb2x1bW5SZW5hbWVkKGYiY291bnQoe2NvbHVtbn0pIiwgImZyZXEiKQogICAgICAgICAgICApCiAgICAgICAgICAgIC50b1BhbmRhcygpCiAgICAgICAgICAgIC5pbG9jWzBdCiAgICAgICAgKQoKICAgICAgICB0b3BfNTAgPSAoCiAgICAgICAgICAgIHZhbHVlX2NvdW50cy5saW1pdCg1MCkKICAgICAgICAgICAgLnRvUGFuZGFzKCkKICAgICAgICAgICAgLnNvcnRfdmFsdWVzKGYiY291bnQoe2NvbHVtbn0pIiwgYXNjZW5kaW5nPUZhbHNlKQogICAgICAgICkKICAgICAgICB0b3BfNTBfY2F0ZWdvcmllcyA9IHRvcF81MFtjb2x1bW5dLnZhbHVlcy50b2xpc3QoKQoKICAgICAgICBvdGhlcnNfY291bnQgPSBwZC5TZXJpZXMoCiAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgIGRmLnNlbGVjdChjb2x1bW4pCiAgICAgICAgICAgICAgICAubmEuZHJvcCgpCiAgICAgICAgICAgICAgICAud2hlcmUofihjb2woY29sdW1uKS5pc2luKCp0b3BfNTBfY2F0ZWdvcmllcykpKQogICAgICAgICAgICAgICAgLmNvdW50KCkKICAgICAgICAgICAgXSwKICAgICAgICAgICAgaW5kZXg9WyIqKipPdGhlciBWYWx1ZXMqKioiXSwKICAgICAgICApCiAgICAgICAgb3RoZXJzX2Rpc3RpbmN0X2NvdW50ID0gcGQuU2VyaWVzKAogICAgICAgICAgICBbdmFsdWVfY291bnRzLndoZXJlKH4oY29sKGNvbHVtbikuaXNpbigqdG9wXzUwX2NhdGVnb3JpZXMpKSkuY291bnQoKV0sCiAgICAgICAgICAgIGluZGV4PVsiKioqT3RoZXIgVmFsdWVzIERpc3RpbmN0IENvdW50KioqIl0sCiAgICAgICAgKQoKICAgICAgICB0b3AgPSB0b3BfNTAuc2V0X2luZGV4KGNvbHVtbilbZiJjb3VudCh7Y29sdW1ufSkiXQogICAgICAgIHRvcCA9IHRvcC5hcHBlbmQob3RoZXJzX2NvdW50KQogICAgICAgIHRvcCA9IHRvcC5hcHBlbmQob3RoZXJzX2Rpc3RpbmN0X2NvdW50KQogICAgICAgIHN0YXRzWyJ2YWx1ZV9jb3VudHMiXSA9IHRvcAogICAgICAgIHN0YXRzWyJ0eXBlIl0gPSAiQ0FUIgogICAgICAgIHZhbHVlX2NvdW50cy51bnBlcnNpc3QoKQogICAgICAgIHVucGFyc2VkX3ZhbGlkX2pzb25zID0gKAogICAgICAgICAgICBkZi5zZWxlY3QoY29sdW1uKQogICAgICAgICAgICAubmEuZHJvcCgpCiAgICAgICAgICAgIC5yZGQubWFwKGxhbWJkYSB4OiBndWVzc19qc29uX3R5cGUoeFtjb2x1bW5dKSkKICAgICAgICAgICAgLmZpbHRlcihsYW1iZGEgeDogeCkKICAgICAgICAgICAgLmRpc3RpbmN0KCkKICAgICAgICAgICAgLmNvbGxlY3QoKQogICAgICAgICkKICAgICAgICBzdGF0c1sidW5wYXJzZWRfanNvbl90eXBlcyJdID0gdW5wYXJzZWRfdmFsaWRfanNvbnMKICAgICAgICByZXR1cm4gc3RhdHMKCiAgICBkZWYgZGVzY3JpYmVfY29uc3RhbnRfMWQoZGYsIGNvbHVtbik6CiAgICAgICAgc3RhdHMgPSBwZC5TZXJpZXMoWyJDT05TVCJdLCBpbmRleD1bInR5cGUiXSwgbmFtZT1jb2x1bW4pCiAgICAgICAgc3RhdHNbInZhbHVlX2NvdW50cyJdID0gKAogICAgICAgICAgICAoZGYuc2VsZWN0KGNvbHVtbikubmEuZHJvcCgpLmxpbWl0KDEpKS50b1BhbmRhcygpLmlsb2NbOiwgMF0udmFsdWVfY291bnRzKCkKICAgICAgICApCiAgICAgICAgcmV0dXJuIHN0YXRzCgogICAgZGVmIGRlc2NyaWJlX3VuaXF1ZV8xZChkZiwgY29sdW1uKToKICAgICAgICBzdGF0cyA9IHBkLlNlcmllcyhbIlVOSVFVRSJdLCBpbmRleD1bInR5cGUiXSwgbmFtZT1jb2x1bW4pCiAgICAgICAgc3RhdHNbInZhbHVlX2NvdW50cyJdID0gKAogICAgICAgICAgICAoZGYuc2VsZWN0KGNvbHVtbikubmEuZHJvcCgpLmxpbWl0KDUwKSkudG9QYW5kYXMoKS5pbG9jWzosIDBdLnZhbHVlX2NvdW50cygpCiAgICAgICAgKQogICAgICAgIHJldHVybiBzdGF0cwoKICAgIGRlZiBkZXNjcmliZV8xZChkZiwgY29sdW1uLCBucm93cywgbG9va3VwX2NvbmZpZz1Ob25lKToKICAgICAgICBjb2x1bW5fdHlwZSA9IGRmLnNlbGVjdChjb2x1bW4pLmR0eXBlc1swXVsxXQogICAgICAgIGlmICgKICAgICAgICAgICAgKCJhcnJheSIgaW4gY29sdW1uX3R5cGUpCiAgICAgICAgICAgIG9yICgic3R1Y3QiIGluIGNvbHVtbl90eXBlKQogICAgICAgICAgICBvciAoIm1hcCIgaW4gY29sdW1uX3R5cGUpCiAgICAgICAgKToKICAgICAgICAgICAgcmFpc2UgTm90SW1wbGVtZW50ZWRFcnJvcigKICAgICAgICAgICAgICAgIGYiQ29sdW1uIHtjb2x1bW59IGlzIG9mIHR5cGUge2NvbHVtbl90eXBlfSBhbmQgY2Fubm90IGJlIGFuYWx5emVkIgogICAgICAgICAgICApCgogICAgICAgIGRpc3RpbmN0X2NvdW50ID0gKAogICAgICAgICAgICBkZi5zZWxlY3QoY29sdW1uKQogICAgICAgICAgICAuYWdnKGNvdW50RGlzdGluY3QoY29sKGNvbHVtbikpLmFsaWFzKCJkaXN0aW5jdF9jb3VudCIpKQogICAgICAgICAgICAudG9QYW5kYXMoKQogICAgICAgICkKICAgICAgICBub25fbmFuX2NvdW50ID0gKAogICAgICAgICAgICBkZi5zZWxlY3QoY29sdW1uKQogICAgICAgICAgICAubmEuZHJvcCgpCiAgICAgICAgICAgIC5zZWxlY3QoY291bnQoY29sKGNvbHVtbikpLmFsaWFzKCJjb3VudCIpKQogICAgICAgICAgICAudG9QYW5kYXMoKQogICAgICAgICkKICAgICAgICByZXN1bHRzX2RhdGEgPSBwZC5jb25jYXQoW2Rpc3RpbmN0X2NvdW50LCBub25fbmFuX2NvdW50XSwgYXhpcz0xKQogICAgICAgIHJlc3VsdHNfZGF0YVsicF91bmlxdWUiXSA9IHJlc3VsdHNfZGF0YVsiZGlzdGluY3RfY291bnQiXSAvIGZsb2F0KAogICAgICAgICAgICByZXN1bHRzX2RhdGFbImNvdW50Il0KICAgICAgICApCiAgICAgICAgcmVzdWx0c19kYXRhWyJpc191bmlxdWUiXSA9IHJlc3VsdHNfZGF0YVsiZGlzdGluY3RfY291bnQiXSA9PSBucm93cwogICAgICAgIHJlc3VsdHNfZGF0YVsibl9taXNzaW5nIl0gPSBucm93cyAtIHJlc3VsdHNfZGF0YVsiY291bnQiXQogICAgICAgIHJlc3VsdHNfZGF0YVsicF9taXNzaW5nIl0gPSByZXN1bHRzX2RhdGFbIm5fbWlzc2luZyJdIC8gZmxvYXQobnJvd3MpCiAgICAgICAgcmVzdWx0c19kYXRhWyJwX2luZmluaXRlIl0gPSAwCiAgICAgICAgcmVzdWx0c19kYXRhWyJuX2luZmluaXRlIl0gPSAwCiAgICAgICAgcmVzdWx0ID0gcmVzdWx0c19kYXRhLmlsb2NbMF0uY29weSgpCiAgICAgICAgcmVzdWx0WyJtZW1vcnlzaXplIl0gPSAwCiAgICAgICAgcmVzdWx0Lm5hbWUgPSBjb2x1bW4KCiAgICAgICAgaWYgcmVzdWx0WyJkaXN0aW5jdF9jb3VudCJdIDw9IDE6CiAgICAgICAgICAgIHJlc3VsdCA9IHJlc3VsdC5hcHBlbmQoZGVzY3JpYmVfY29uc3RhbnRfMWQoZGYsIGNvbHVtbikpCiAgICAgICAgZWxpZiBjb2x1bW5fdHlwZSBpbiB7InRpbnlpbnQiLCAic21hbGxpbnQiLCAiaW50IiwgImJpZ2ludCJ9OgogICAgICAgICAgICByZXN1bHQgPSByZXN1bHQuYXBwZW5kKGRlc2NyaWJlX2ludGVnZXJfMWQoZGYsIGNvbHVtbiwgcmVzdWx0LCBucm93cykpCiAgICAgICAgZWxpZiBjb2x1bW5fdHlwZSBpbiB7ImZsb2F0IiwgImRvdWJsZSIsICJkZWNpbWFsIn06CiAgICAgICAgICAgIHJlc3VsdCA9IHJlc3VsdC5hcHBlbmQoZGVzY3JpYmVfZmxvYXRfMWQoZGYsIGNvbHVtbiwgcmVzdWx0LCBucm93cykpCiAgICAgICAgZWxpZiBjb2x1bW5fdHlwZSBpbiB7ImRhdGUiLCAidGltZXN0YW1wIn06CiAgICAgICAgICAgIHJlc3VsdCA9IHJlc3VsdC5hcHBlbmQoZGVzY3JpYmVfZGF0ZV8xZChkZiwgY29sdW1uKSkKICAgICAgICBlbGlmIHJlc3VsdFsiaXNfdW5pcXVlIl0gPT0gVHJ1ZToKICAgICAgICAgICAgcmVzdWx0ID0gcmVzdWx0LmFwcGVuZChkZXNjcmliZV91bmlxdWVfMWQoZGYsIGNvbHVtbikpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmVzdWx0ID0gcmVzdWx0LmFwcGVuZChkZXNjcmliZV9jYXRlZ29yaWNhbF8xZChkZiwgY29sdW1uKSkKICAgICAgICAgICAgaWYgcmVzdWx0WyJuX21pc3NpbmciXSA+IDA6CiAgICAgICAgICAgICAgICByZXN1bHRbImRpc3RpbmN0X2NvdW50Il0gPSByZXN1bHRbImRpc3RpbmN0X2NvdW50Il0gKyAxCgogICAgICAgIGlmIHJlc3VsdFsiY291bnQiXSA+IHJlc3VsdFsiZGlzdGluY3RfY291bnQiXSA+IDE6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIHJlc3VsdFsibW9kZSJdID0gcmVzdWx0WyJ0b3AiXQogICAgICAgICAgICBleGNlcHQgS2V5RXJyb3I6CiAgICAgICAgICAgICAgICByZXN1bHRbIm1vZGUiXSA9IDAKICAgICAgICBlbHNlOgogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICByZXN1bHRbIm1vZGUiXSA9IHJlc3VsdFsidmFsdWVfY291bnRzIl0uaW5kZXhbMF0KICAgICAgICAgICAgZXhjZXB0IEtleUVycm9yOgogICAgICAgICAgICAgICAgcmVzdWx0WyJtb2RlIl0gPSAwCiAgICAgICAgICAgIGV4Y2VwdCBJbmRleEVycm9yOgogICAgICAgICAgICAgICAgcmVzdWx0WyJtb2RlIl0gPSAiTUlTU0lORyIKCiAgICAgICAgaWYgbG9va3VwX2NvbmZpZzoKICAgICAgICAgICAgbG9va3VwX29iamVjdCA9IGxvb2t1cF9jb25maWdbIm9iamVjdCJdCiAgICAgICAgICAgIGNvbF9uYW1lX2luX2RiID0gKAogICAgICAgICAgICAgICAgbG9va3VwX2NvbmZpZ1siY29sX25hbWVfaW5fZGIiXQogICAgICAgICAgICAgICAgaWYgImNvbF9uYW1lX2luX2RiIiBpbiBsb29rdXBfY29uZmlnCiAgICAgICAgICAgICAgICBlbHNlIE5vbmUKICAgICAgICAgICAgKQogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICBtYXRjaGVkLCB1bm1hdGNoZWQgPSBsb29rdXBfb2JqZWN0Lmxvb2t1cCgKICAgICAgICAgICAgICAgICAgICBkZi5zZWxlY3QoY29sdW1uKSwgY29sX25hbWVfaW5fZGIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJlc3VsdFsibG9va2VkdXBfdmFsdWVzIl0gPSAoCiAgICAgICAgICAgICAgICAgICAgc3RyKG1hdGNoZWQuY291bnQoKSkgKyAiLyIgKyBzdHIoZGYuc2VsZWN0KGNvbHVtbikuY291bnQoKSkKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgZXhjZXB0OgogICAgICAgICAgICAgICAgcmVzdWx0WyJsb29rZWR1cF92YWx1ZXMiXSA9ICJGQUlMRUQiCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmVzdWx0WyJsb29rZWR1cF92YWx1ZXMiXSA9ICIiCgogICAgICAgIHJldHVybiByZXN1bHQKCiAgICBsZGVzYyA9IHt9CiAgICBmb3IgY29sdW0gaW4gZGYuY29sdW1uczoKICAgICAgICBpZiBjb2x1bSBpbiBjb25maWc6CiAgICAgICAgICAgIGlmICJsb29rdXAiIGluIGNvbmZpZ1tjb2x1bV06CiAgICAgICAgICAgICAgICBsb29rdXBfY29uZmlnID0gY29uZmlnW2NvbHVtXVsibG9va3VwIl0KICAgICAgICAgICAgICAgIGRlc2MgPSBkZXNjcmliZV8xZCgKICAgICAgICAgICAgICAgICAgICBkZiwgY29sdW0sIHRhYmxlX3N0YXRzWyJuIl0sIGxvb2t1cF9jb25maWc9bG9va3VwX2NvbmZpZwogICAgICAgICAgICAgICAgKQogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgZGVzYyA9IGRlc2NyaWJlXzFkKGRmLCBjb2x1bSwgdGFibGVfc3RhdHNbIm4iXSkKICAgICAgICBlbHNlOgogICAgICAgICAgICBkZXNjID0gZGVzY3JpYmVfMWQoZGYsIGNvbHVtLCB0YWJsZV9zdGF0c1sibiJdKQogICAgICAgIGxkZXNjLnVwZGF0ZSh7Y29sdW06IGRlc2N9KQoKICAgIGlmIGNvcnJfcmVqZWN0IGlzIG5vdCBOb25lOgogICAgICAgIGNvbXB1dGFibGVfY29ycnMgPSBbY29sdW0gZm9yIGNvbHVtIGluIGxkZXNjIGlmIGxkZXNjW2NvbHVtXVsidHlwZSJdIGluIHsiTlVNIn1dCgogICAgICAgIGlmIGxlbihjb21wdXRhYmxlX2NvcnJzKSA+IDA6CiAgICAgICAgICAgIGNvcnIgPSBjb3JyX21hdHJpeChkZiwgY29sdW1ucz1jb21wdXRhYmxlX2NvcnJzKQogICAgICAgICAgICBmb3IgeCwgY29ycl94IGluIGNvcnIuaXRlcnJvd3MoKToKICAgICAgICAgICAgICAgIGZvciB5LCBjb3JyIGluIGNvcnJfeC5pdGVyaXRlbXMoKToKICAgICAgICAgICAgICAgICAgICBpZiB4ID09IHk6CiAgICAgICAgICAgICAgICAgICAgICAgIGJyZWFrCgogICAgdmFyaWFibGVfc3RhdHMgPSBwZC5EYXRhRnJhbWUobGRlc2MpCgogICAgdGFibGVfc3RhdHNbIm52YXIiXSA9IGxlbihkZi5jb2x1bW5zKQogICAgdGFibGVfc3RhdHNbInRvdGFsX21pc3NpbmciXSA9IGZsb2F0KHZhcmlhYmxlX3N0YXRzLmxvY1sibl9taXNzaW5nIl0uc3VtKCkpIC8gKAogICAgICAgIHRhYmxlX3N0YXRzWyJuIl0gKiB0YWJsZV9zdGF0c1sibnZhciJdCiAgICApCiAgICBtZW1zaXplID0gMAogICAgdGFibGVfc3RhdHNbIm1lbXNpemUiXSA9IGZtdF9ieXRlc2l6ZShtZW1zaXplKQogICAgdGFibGVfc3RhdHNbInJlY29yZHNpemUiXSA9IGZtdF9ieXRlc2l6ZShtZW1zaXplIC8gdGFibGVfc3RhdHNbIm4iXSkKICAgIHRhYmxlX3N0YXRzLnVwZGF0ZSgKICAgICAgICB7azogMCBmb3IgayBpbiAoIk5VTSIsICJEQVRFIiwgIkNPTlNUIiwgIkNBVCIsICJVTklRVUUiLCAiQ09SUiIpfQogICAgKQogICAgdGFibGVfc3RhdHMudXBkYXRlKGRpY3QodmFyaWFibGVfc3RhdHMubG9jWyJ0eXBlIl0udmFsdWVfY291bnRzKCkpKQogICAgdGFibGVfc3RhdHNbIlJFSkVDVEVEIl0gPSB0YWJsZV9zdGF0c1siQ09OU1QiXSArIHRhYmxlX3N0YXRzWyJDT1JSIl0KCiAgICBmcmVxX2RpY3QgPSB7fQogICAgZm9yIHZhciBpbiB2YXJpYWJsZV9zdGF0czoKICAgICAgICBpZiAidmFsdWVfY291bnRzIiBub3QgaW4gdmFyaWFibGVfc3RhdHNbdmFyXToKICAgICAgICAgICAgcGFzcwogICAgICAgIGVsaWYgdmFyaWFibGVfc3RhdHNbdmFyXVsidmFsdWVfY291bnRzIl0gaXMgbm90IG5wLm5hbjoKICAgICAgICAgICAgZnJlcV9kaWN0W3Zhcl0gPSB2YXJpYWJsZV9zdGF0c1t2YXJdWyJ2YWx1ZV9jb3VudHMiXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHBhc3MKICAgIHRyeToKICAgICAgICB2YXJpYWJsZV9zdGF0cyA9IHZhcmlhYmxlX3N0YXRzLmRyb3AoInZhbHVlX2NvdW50cyIpCiAgICBleGNlcHQgKFZhbHVlRXJyb3IsIEtleUVycm9yKToKICAgICAgICBwYXNzCgogICAgcmV0dXJuIHRhYmxlX3N0YXRzLCB2YXJpYWJsZV9zdGF0cy5ULCBmcmVxX2RpY3QKCgpTS0VXTkVTU19DVVRPRkYgPSAyMApERUZBVUxUX0ZMT0FUX0ZPUk1BVFRFUiA9ICJzcGFya19kZl9wcm9maWxpbmcuX19kZWZhdWx0X2Zsb2F0X2Zvcm1hdHRlciIKCgpkZWYgZ3JhZGllbnRfZm9ybWF0KHZhbHVlLCBsaW1pdDEsIGxpbWl0MiwgYzEsIGMyKToKICAgIGRlZiBMZXJwQ29sb3VyKGMxLCBjMiwgdCk6CiAgICAgICAgcmV0dXJuICgKICAgICAgICAgICAgaW50KGMxWzBdICsgKGMyWzBdIC0gYzFbMF0pICogdCksCiAgICAgICAgICAgIGludChjMVsxXSArIChjMlsxXSAtIGMxWzFdKSAqIHQpLAogICAgICAgICAgICBpbnQoYzFbMl0gKyAoYzJbMl0gLSBjMVsyXSkgKiB0KSwKICAgICAgICApCgogICAgYyA9IExlcnBDb2xvdXIoYzEsIGMyLCAodmFsdWUgLSBsaW1pdDEpIC8gKGxpbWl0MiAtIGxpbWl0MSkpCiAgICByZXR1cm4gZm10X2NvbG9yKHZhbHVlLCBmInJnYntzdHIoYyl9IikKCgpkZWYgZm10X2NvbG9yKHRleHQsIGNvbG9yKToKICAgIHJldHVybiBmJzxzcGFuIHN0eWxlPSJjb2xvcjp7Y29sb3J9Ij57c3RyKHRleHQpfTwvc3Bhbj4nCgoKZGVmIGZtdF9jbGFzcyh0ZXh0LCBjbHMpOgogICAgcmV0dXJuIGYnPHNwYW4gY2xhc3M9IntjbHN9Ij57c3RyKHRleHQpfTwvc3Bhbj4nCgoKZGVmIGZtdF9ieXRlc2l6ZShudW0sIHN1ZmZpeD0iQiIpOgogICAgZm9yIHVuaXQgaW4gWyIiLCAiS2kiLCAiTWkiLCAiR2kiLCAiVGkiLCAiUGkiLCAiRWkiLCAiWmkiXToKICAgICAgICBpZiBudW0gPCAwOgogICAgICAgICAgICBudW0gPSBudW0gKiAtMQogICAgICAgICAgICBpZiBudW0gPCAxMDI0LjA6CiAgICAgICAgICAgICAgICByZXR1cm4gIiUzLjFmICVzJXMiICUgKG51bSwgdW5pdCwgc3VmZml4KQogICAgICAgICAgICBudW0gLz0gMTAyNC4wCiAgICByZXR1cm4gIiUuMWYgJXMlcyIgJSAobnVtLCAiWWkiLCBzdWZmaXgpCgoKZGVmIGZtdF9wZXJjZW50KHYpOgogICAgcmV0dXJuIGYie3YgKiAxMDA6Mi4xZn0lIgoKCmRlZiBmbXRfdmFybmFtZSh2KToKICAgIHJldHVybiBmIjxjb2RlPnt2fTwvY29kZT4iCgoKdmFsdWVfZm9ybWF0dGVycyA9IHsKICAgICJmcmVxIjogKGxhbWJkYSB2OiBncmFkaWVudF9mb3JtYXQodiwgMCwgNjIwMDAsICgzMCwgMTk4LCAyNDQpLCAoOTksIDIwMCwgNzIpKSksCiAgICAicF9taXNzaW5nIjogZm10X3BlcmNlbnQsCiAgICAicF9pbmZpbml0ZSI6IGZtdF9wZXJjZW50LAogICAgInBfdW5pcXVlIjogZm10X3BlcmNlbnQsCiAgICAicF96ZXJvcyI6IGZtdF9wZXJjZW50LAogICAgIm1lbW9yeXNpemUiOiBmbXRfYnl0ZXNpemUsCiAgICAidG90YWxfbWlzc2luZyI6IGZtdF9wZXJjZW50LAogICAgREVGQVVMVF9GTE9BVF9GT1JNQVRURVI6IGxhbWJkYSB2OiBzdHIoZmxvYXQoZiJ7djouNWd9IikpLnJzdHJpcCgiMCIpLnJzdHJpcCgiLiIpLAogICAgImNvcnJlbGF0aW9uX3ZhciI6IGxhbWJkYSB2OiBmbXRfdmFybmFtZSh2KSwKICAgICJ1bnBhcnNlZF9qc29uX3R5cGVzIjogbGFtYmRhIHY6ICIsICIuam9pbihbcy5fX25hbWVfXyBmb3IgcyBpbiB2XSksCn0KCgpkZWYgZm10X3Jvd19zZXZlcml0eSh2KToKICAgIGlmIG5wLmlzbmFuKHYpIG9yIHYgPD0gMC4wMToKICAgICAgICByZXR1cm4gImlnbm9yZSIKICAgIGVsc2U6CiAgICAgICAgcmV0dXJuICJhbGVydCIKCgpkZWYgZm10X3NrZXduZXNzKHYpOgogICAgaWYgbm90IG5wLmlzbmFuKHYpIGFuZCAodiA8IC1TS0VXTkVTU19DVVRPRkYgb3IgdiA+IFNLRVdORVNTX0NVVE9GRik6CiAgICAgICAgcmV0dXJuICJhbGVydCIKICAgIGVsc2U6CiAgICAgICAgcmV0dXJuICIiCgoKcm93X2Zvcm1hdHRlcnMgPSB7CiAgICAicF96ZXJvcyI6IGZtdF9yb3dfc2V2ZXJpdHksCiAgICAicF9taXNzaW5nIjogZm10X3Jvd19zZXZlcml0eSwKICAgICJwX2luZmluaXRlIjogZm10X3Jvd19zZXZlcml0eSwKICAgICJuX2R1cGxpY2F0ZXMiOiBmbXRfcm93X3NldmVyaXR5LAogICAgInNrZXduZXNzIjogZm10X3NrZXduZXNzLAp9CgpydW4oWyIvYmluL2Jhc2giLCAiL2V0Yy9jb25maWcvdjNpby92M2lvLXNwYXJrLW9wZXJhdG9yLnNoIl0pCgoKZGVmIGRlc2NyaWJlX3NwYXJrKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIGFydGlmYWN0X3BhdGgsCiAgICBiaW5zOiBpbnQgPSAzMCwKICAgIGRlc2NyaWJlX2V4dGVuZGVkOiBib29sID0gVHJ1ZSwKKToKICAgIGxvY2F0aW9uID0gZGF0YXNldC5sb2NhbCgpCgogICAgc3BhcmsgPSBTcGFya1Nlc3Npb24uYnVpbGRlci5hcHBOYW1lKCJTcGFyayBqb2IiKS5nZXRPckNyZWF0ZSgpCgogICAgZGYgPSBzcGFyay5yZWFkLmNzdihsb2NhdGlvbiwgaGVhZGVyPVRydWUsIGluZmVyU2NoZW1hPVRydWUpCgogICAga3dhcmdzID0gW10KCiAgICBmbG9hdF9jb2xzID0gWwogICAgICAgIGl0ZW1bMF0KICAgICAgICBmb3IgaXRlbSBpbiBkZi5kdHlwZXMKICAgICAgICBpZiBpdGVtWzFdLnN0YXJ0c3dpdGgoImZsb2F0Iikgb3IgaXRlbVsxXS5zdGFydHN3aXRoKCJkb3VibGUiKQogICAgXQoKICAgIGlmIGRlc2NyaWJlX2V4dGVuZGVkID09IFRydWU6CiAgICAgICAgdGFibGUsIHZhcmlhYmxlcywgZnJlcSA9IGRlc2NyaWJlKGRmLCBiaW5zLCBmbG9hdF9jb2xzLCBrd2FyZ3MpCgogICAgICAgIHRibF8xID0gdmFyaWFibGVzLnJlc2V0X2luZGV4KCkKCiAgICAgICAgaWYgbGVuKGZyZXEpICE9IDA6CiAgICAgICAgICAgIHRibF8yID0gKAogICAgICAgICAgICAgICAgcGQuRGF0YUZyYW1lLmZyb21fZGljdChmcmVxLCBvcmllbnQ9ImluZGV4IikKICAgICAgICAgICAgICAgIC5zb3J0X2luZGV4KCkKICAgICAgICAgICAgICAgIC5zdGFjaygpCiAgICAgICAgICAgICAgICAucmVzZXRfaW5kZXgoKQogICAgICAgICAgICApCiAgICAgICAgICAgIHRibF8yLmNvbHVtbnMgPSBbImNvbCIsICJrZXkiLCAidmFsIl0KICAgICAgICAgICAgdGJsXzJbIk1lcmdlZCJdID0gW3trZXk6IHZhbH0gZm9yIGtleSwgdmFsIGluIHppcCh0YmxfMi5rZXksIHRibF8yLnZhbCldCiAgICAgICAgICAgIHRibF8yID0gdGJsXzIuZ3JvdXBieSgiY29sIiwgYXNfaW5kZXg9RmFsc2UpLmFnZyhsYW1iZGEgeDogdHVwbGUoeCkpWwogICAgICAgICAgICAgICAgWyJjb2wiLCAiTWVyZ2VkIl0KICAgICAgICAgICAgXQoKICAgICAgICAgICAgc3VtbWFyeSA9IHBkLm1lcmdlKAogICAgICAgICAgICAgICAgdGJsXzEsIHRibF8yLCBob3c9ImxlZnQiLCBsZWZ0X29uPSJpbmRleCIsIHJpZ2h0X29uPSJjb2wiCiAgICAgICAgICAgICkKCiAgICAgICAgZWxzZToKICAgICAgICAgICAgc3VtbWFyeSA9IHRibF8xCgogICAgICAgIGNvbnRleHQubG9nX2RhdGFzZXQoCiAgICAgICAgICAgICJzdW1tYXJ5X3N0YXRzIiwKICAgICAgICAgICAgZGY9c3VtbWFyeSwKICAgICAgICAgICAgZm9ybWF0PSJjc3YiLAogICAgICAgICAgICBpbmRleD1GYWxzZSwKICAgICAgICAgICAgYXJ0aWZhY3RfcGF0aD1jb250ZXh0LmFydGlmYWN0X3N1YnBhdGgoImRhdGEiKSwKICAgICAgICApCgogICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdHModGFibGUpCgogICAgZWxzZToKICAgICAgICB0YmxfMSA9IGRmLmRlc2NyaWJlKCkudG9QYW5kYXMoKQoKICAgICAgICBzdW1tYXJ5ID0gdGJsXzEuVAoKICAgICAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgICAgICAic3VtbWFyeV9zdGF0cyIsCiAgICAgICAgICAgIGRmPXN1bW1hcnksCiAgICAgICAgICAgIGZvcm1hdD0iY3N2IiwKICAgICAgICAgICAgaW5kZXg9RmFsc2UsCiAgICAgICAgICAgIGFydGlmYWN0X3BhdGg9Y29udGV4dC5hcnRpZmFjdF9zdWJwYXRoKCJkYXRhIiksCiAgICAgICAgKQoKICAgIHNwYXJrLnN0b3AoKQo= + code_origin: '' + filename: describe_spark.py entry_points: describe: - name: describe - doc: '' parameters: - name: df - default: '' - name: bins - default: '' - name: corr_reject - default: '' - name: config - default: '' - outputs: - - default: '' - lineno: 38 - pretty_name: - name: pretty_name + name: describe doc: '' + has_kwargs: true + has_varargs: false + lineno: 58 + pretty_name: parameters: - name: x - default: '' - outputs: - - default: '' - lineno: 51 - corr_matrix: - name: corr_matrix + name: pretty_name doc: '' + has_kwargs: false + has_varargs: false + lineno: 71 + corr_matrix: parameters: - name: df - default: '' - name: columns default: null - outputs: - - default: '' - lineno: 58 - separate: - name: separate + name: corr_matrix doc: '' + has_kwargs: false + has_varargs: false + lineno: 78 + separate: parameters: - name: l - default: '' - name: n - default: '' - outputs: - - default: '' - lineno: 63 - create_hist_data: - name: create_hist_data + name: separate doc: '' + has_kwargs: false + has_varargs: false + lineno: 83 + create_hist_data: parameters: - name: df - default: '' - name: column - default: '' - name: minim - default: '' - name: maxim - default: '' - name: bins default: 10 - outputs: - - default: '' - lineno: 80 + name: create_hist_data + doc: '' + has_kwargs: false + has_varargs: false + lineno: 100 create_all_conditions: - name: create_all_conditions - doc: 'Recursive function that exploits the - - ability to call the Spark SQL Column method - - .when() in a recursive way.' parameters: - name: current_col - default: '' - name: column - default: '' - name: left_edges - default: '' - name: count default: 1 - outputs: - - default: '' - lineno: 82 + name: create_all_conditions + doc: 'Recursive function that exploits the + + ability to call the Spark SQL Column method + + .when() in a recursive way.' + has_kwargs: false + has_varargs: false + lineno: 101 describe_integer_1d: - name: describe_integer_1d - doc: '' parameters: - name: df - default: '' - name: column - default: '' - name: current_result - default: '' - name: nrows - default: '' - outputs: - - default: '' - lineno: 134 - describe_float_1d: - name: describe_float_1d + name: describe_integer_1d doc: '' + has_kwargs: false + has_varargs: false + lineno: 159 + describe_float_1d: parameters: - name: df - default: '' - name: column - default: '' - name: current_result - default: '' - name: nrows - default: '' - outputs: - - default: '' - lineno: 170 - describe_date_1d: - name: describe_date_1d + name: describe_float_1d doc: '' + has_kwargs: false + has_varargs: false + lineno: 202 + describe_date_1d: parameters: - name: df - default: '' - name: column - default: '' - outputs: - - default: '' - lineno: 204 - guess_json_type: - name: guess_json_type + name: describe_date_1d doc: '' + has_kwargs: false + has_varargs: false + lineno: 245 + guess_json_type: parameters: - name: string_value - default: '' - outputs: - - default: '' - lineno: 221 + name: guess_json_type + doc: '' + has_kwargs: false + has_varargs: false + lineno: 265 describe_categorical_1d: + parameters: + - name: df + - name: column name: describe_categorical_1d doc: '' + has_kwargs: false + has_varargs: false + lineno: 273 + describe_constant_1d: parameters: - name: df - default: '' - name: column - default: '' - outputs: - - default: '' - lineno: 229 - describe_constant_1d: name: describe_constant_1d doc: '' + has_kwargs: false + has_varargs: false + lineno: 330 + describe_unique_1d: parameters: - name: df - default: '' - name: column - default: '' - outputs: - - default: '' - lineno: 267 - describe_unique_1d: name: describe_unique_1d doc: '' - parameters: - - name: df - default: '' - - name: column - default: '' - outputs: - - default: '' - lineno: 274 + has_kwargs: false + has_varargs: false + lineno: 337 describe_1d: - name: describe_1d - doc: '' parameters: - name: df - default: '' - name: column - default: '' - name: nrows - default: '' - name: lookup_config default: null - outputs: - - default: '' - lineno: 281 - gradient_format: - name: gradient_format + name: describe_1d doc: '' + has_kwargs: false + has_varargs: false + lineno: 344 + gradient_format: parameters: - name: value - default: '' - name: limit1 - default: '' - name: limit2 - default: '' - name: c1 - default: '' - name: c2 - default: '' - outputs: - - default: '' - lineno: 396 - LerpColour: - name: LerpColour + name: gradient_format doc: '' + has_kwargs: false + has_varargs: false + lineno: 487 + LerpColour: parameters: - name: c1 - default: '' - name: c2 - default: '' - name: t - default: '' - outputs: - - default: '' - lineno: 397 - fmt_color: - name: fmt_color + name: LerpColour doc: '' + has_kwargs: false + has_varargs: false + lineno: 488 + fmt_color: parameters: - name: text - default: '' - name: color - default: '' - outputs: - - default: '' - lineno: 403 - fmt_class: - name: fmt_class + name: fmt_color doc: '' + has_kwargs: false + has_varargs: false + lineno: 499 + fmt_class: parameters: - name: text - default: '' - name: cls - default: '' - outputs: - - default: '' - lineno: 407 - fmt_bytesize: - name: fmt_bytesize + name: fmt_class doc: '' + has_kwargs: false + has_varargs: false + lineno: 503 + fmt_bytesize: parameters: - name: num - default: '' - name: suffix default: B - outputs: - - default: '' - lineno: 411 + name: fmt_bytesize + doc: '' + has_kwargs: false + has_varargs: false + lineno: 507 fmt_percent: + parameters: + - name: v name: fmt_percent doc: '' + has_kwargs: false + has_varargs: false + lineno: 517 + fmt_varname: parameters: - name: v - default: '' - outputs: - - default: '' - lineno: 421 - fmt_varname: name: fmt_varname doc: '' + has_kwargs: false + has_varargs: false + lineno: 521 + fmt_row_severity: parameters: - name: v - default: '' - outputs: - - default: '' - lineno: 424 - fmt_row_severity: name: fmt_row_severity doc: '' + has_kwargs: false + has_varargs: false + lineno: 539 + fmt_skewness: parameters: - name: v - default: '' - outputs: - - default: '' - lineno: 441 - fmt_skewness: name: fmt_skewness doc: '' - parameters: - - name: v - default: '' - outputs: - - default: '' - lineno: 447 + has_kwargs: false + has_varargs: false + lineno: 546 describe_spark: - name: describe_spark - doc: '' parameters: - name: context type: MLClientCtx - default: '' - name: dataset type: DataItem - default: '' - name: artifact_path - default: '' - name: bins type: int default: 30 - name: describe_extended type: bool default: true - outputs: - - default: '' - lineno: 463 + name: describe_spark + doc: '' + has_kwargs: false + has_varargs: false + lineno: 564 + command: '' description: '' - build: - functionSourceCode: IyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IG1scnVuCmZyb20gbWxydW4ucGxhdGZvcm1zLmlndWF6aW8gaW1wb3J0IG1vdW50X3YzaW8sIG1vdW50X3YzaW9kCmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKCmltcG9ydCBvcwpmcm9tIHN1YnByb2Nlc3MgaW1wb3J0IHJ1bgppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCBudW1weSBhcyBucAoKZnJvbSBweXNwYXJrLnNxbC50eXBlcyBpbXBvcnQgTG9uZ1R5cGUKZnJvbSBweXNwYXJrLnNxbCBpbXBvcnQgU3BhcmtTZXNzaW9uCgppbXBvcnQgc3lzCmltcG9ydCBiYXNlNjQgYXMgYjY0CmltcG9ydCB3YXJuaW5ncwp3YXJuaW5ncy5maWx0ZXJ3YXJuaW5ncygiaWdub3JlIikKCmZyb20gaXRlcnRvb2xzIGltcG9ydCBwcm9kdWN0CmltcG9ydCBtYXRwbG90bGliCgppbXBvcnQgbnVtcHkgYXMgbnAKaW1wb3J0IGpzb24KaW1wb3J0IHBhbmRhcyBhcyBwZApmcm9tIG1hdHBsb3RsaWIgaW1wb3J0IHB5cGxvdCBhcyBwbHQKZnJvbSBwa2dfcmVzb3VyY2VzIGltcG9ydCByZXNvdXJjZV9maWxlbmFtZQppbXBvcnQgc2l4CmZyb20gcHlzcGFyay5zcWwgaW1wb3J0IERhdGFGcmFtZSBhcyBTcGFya0RhdGFGcmFtZQpmcm9tIHB5c3Bhcmsuc3FsLmZ1bmN0aW9ucyBpbXBvcnQgKGFicyBhcyBkZl9hYnMsIGNvbCwgY291bnQsIGNvdW50RGlzdGluY3QsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWF4IGFzIGRmX21heCwgbWVhbiwgbWluIGFzIGRmX21pbiwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdW0gYXMgZGZfc3VtLCB3aGVuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKQpmcm9tIHB5c3Bhcmsuc3FsLmZ1bmN0aW9ucyBpbXBvcnQgdmFyaWFuY2UsIHN0ZGRldiwga3VydG9zaXMsIHNrZXduZXNzCgoKZGVmIGRlc2NyaWJlKGRmLCBiaW5zLCBjb3JyX3JlamVjdCwgY29uZmlnLCAqKmt3YXJncyk6CiAgICBpZiBub3QgaXNpbnN0YW5jZShkZiwgU3BhcmtEYXRhRnJhbWUpOgogICAgICAgIHJhaXNlIFR5cGVFcnJvcigiZGYgbXVzdCBiZSBvZiB0eXBlIHB5c3Bhcmsuc3FsLkRhdGFGcmFtZSIpCgogICAgdGFibGVfc3RhdHMgPSB7Im4iOiBkZi5jb3VudCgpfQogICAgaWYgdGFibGVfc3RhdHNbIm4iXSA9PSAwOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoImRmIGNhbm5vdCBiZSBlbXB0eSIpCgogICAgdHJ5OgogICAgICAgIG1hdHBsb3RsaWIuc3R5bGUudXNlKCJkZWZhdWx0IikKICAgIGV4Y2VwdDoKICAgICAgICBwYXNzCgogICAgZGVmIHByZXR0eV9uYW1lKHgpOgogICAgICAgIHggKj0gMTAwCiAgICAgICAgaWYgeCA9PSBpbnQoeCk6CiAgICAgICAgICAgIHJldHVybiAnJS4wZiUlJyAlIHgKICAgICAgICBlbHNlOgogICAgICAgICAgICByZXR1cm4gJyUuMWYlJScgJSB4CgogICAgZGVmIGNvcnJfbWF0cml4KGRmLCBjb2x1bW5zPU5vbmUpOgogICAgICAgIGlmIGNvbHVtbnMgaXMgTm9uZToKICAgICAgICAgICAgY29sdW1ucyA9IGRmLmNvbHVtbnMKICAgICAgICBjb21iaW5hdGlvbnMgPSBsaXN0KHByb2R1Y3QoY29sdW1ucyxjb2x1bW5zKSkKCiAgICAgICAgZGVmIHNlcGFyYXRlKGwsIG4pOgogICAgICAgICAgICBmb3IgaSBpbiByYW5nZSgwLCBsZW4obCksIG4pOgogICAgICAgICAgICAgICAgeWllbGQgbFtpOmkrbl0KCiAgICAgICAgZ3JvdXBlZCA9IGxpc3Qoc2VwYXJhdGUoY29tYmluYXRpb25zLGxlbihjb2x1bW5zKSkpCiAgICAgICAgZGZfY2xlYW5lZCA9IGRmLnNlbGVjdCgqY29sdW1ucykubmEuZHJvcChob3c9ImFueSIpCgogICAgICAgIGZvciBpIGluIGdyb3VwZWQ6CiAgICAgICAgICAgIGZvciBqIGluIGVudW1lcmF0ZShpKToKICAgICAgICAgICAgICAgIGlbalswXV0gPSBpW2pbMF1dICsgKGRmX2NsZWFuZWQuY29ycihzdHIoalsxXVswXSksIHN0cihqWzFdWzFdKSksKQoKICAgICAgICBkZl9wYW5kYXMgPSBwZC5EYXRhRnJhbWUoZ3JvdXBlZCkuYXBwbHltYXAobGFtYmRhIHg6IHhbMl0pCiAgICAgICAgZGZfcGFuZGFzLmNvbHVtbnMgPSBjb2x1bW5zCiAgICAgICAgZGZfcGFuZGFzLmluZGV4ID0gY29sdW1ucwogICAgICAgIAogICAgICAgIHJldHVybiBkZl9wYW5kYXMKCiAgICBkZWYgY3JlYXRlX2hpc3RfZGF0YShkZiwgY29sdW1uLCBtaW5pbSwgbWF4aW0sIGJpbnM9MTApOgoKICAgICAgICBkZWYgY3JlYXRlX2FsbF9jb25kaXRpb25zKGN1cnJlbnRfY29sLCBjb2x1bW4sIGxlZnRfZWRnZXMsIGNvdW50PTEpOgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgUmVjdXJzaXZlIGZ1bmN0aW9uIHRoYXQgZXhwbG9pdHMgdGhlCiAgICAgICAgICAgIGFiaWxpdHkgdG8gY2FsbCB0aGUgU3BhcmsgU1FMIENvbHVtbiBtZXRob2QKICAgICAgICAgICAgLndoZW4oKSBpbiBhIHJlY3Vyc2l2ZSB3YXkuCiAgICAgICAgICAgICIiIgogICAgICAgICAgICBsZWZ0X2VkZ2VzID0gbGVmdF9lZGdlc1s6XQogICAgICAgICAgICBpZiBsZW4obGVmdF9lZGdlcykgPT0gMDoKICAgICAgICAgICAgICAgIHJldHVybiBjdXJyZW50X2NvbAogICAgICAgICAgICBpZiBsZW4obGVmdF9lZGdlcykgPT0gMToKICAgICAgICAgICAgICAgIG5leHRfY29sID0gY3VycmVudF9jb2wud2hlbihjb2woY29sdW1uKSA+PSBmbG9hdChsZWZ0X2VkZ2VzWzBdKSwgY291bnQpCiAgICAgICAgICAgICAgICBsZWZ0X2VkZ2VzLnBvcCgwKQogICAgICAgICAgICAgICAgcmV0dXJuIGNyZWF0ZV9hbGxfY29uZGl0aW9ucyhuZXh0X2NvbCwgY29sdW1uLCBsZWZ0X2VkZ2VzWzpdLCBjb3VudCsxKQogICAgICAgICAgICBuZXh0X2NvbCA9IGN1cnJlbnRfY29sLndoZW4oKGZsb2F0KGxlZnRfZWRnZXNbMF0pIDw9IGNvbChjb2x1bW4pKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJiAoY29sKGNvbHVtbikgPCBmbG9hdChsZWZ0X2VkZ2VzWzFdKSksIGNvdW50KQogICAgICAgICAgICBsZWZ0X2VkZ2VzLnBvcCgwKQogICAgICAgICAgICByZXR1cm4gY3JlYXRlX2FsbF9jb25kaXRpb25zKG5leHRfY29sLCBjb2x1bW4sIGxlZnRfZWRnZXNbOl0sIGNvdW50KzEpCgogICAgICAgIG51bV9yYW5nZSA9IG1heGltIC0gbWluaW0KICAgICAgICBiaW5fd2lkdGggPSBudW1fcmFuZ2UgLyBmbG9hdChiaW5zKQogICAgICAgIGxlZnRfZWRnZXMgPSBbbWluaW1dCiAgICAgICAgZm9yIF9iaW4gaW4gcmFuZ2UoYmlucyk6CiAgICAgICAgICAgIGxlZnRfZWRnZXMgPSBsZWZ0X2VkZ2VzICsgW2xlZnRfZWRnZXNbLTFdICsgYmluX3dpZHRoXQogICAgICAgIGxlZnRfZWRnZXMucG9wKCkKICAgICAgICBleHByZXNzaW9uX2NvbCA9IHdoZW4oKGZsb2F0KGxlZnRfZWRnZXNbMF0pIDw9IGNvbChjb2x1bW4pKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAmIChjb2woY29sdW1uKSA8IGZsb2F0KGxlZnRfZWRnZXNbMV0pKSwgMCkKICAgICAgICBsZWZ0X2VkZ2VzX2NvcHkgPSBsZWZ0X2VkZ2VzWzpdCiAgICAgICAgbGVmdF9lZGdlc19jb3B5LnBvcCgwKQogICAgICAgIGJpbl9kYXRhID0gKGRmLnNlbGVjdChjb2woY29sdW1uKSkKICAgICAgICAgICAgICAgICAgICAubmEuZHJvcCgpCiAgICAgICAgICAgICAgICAgICAgLnNlbGVjdChjb2woY29sdW1uKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIGNyZWF0ZV9hbGxfY29uZGl0aW9ucyhleHByZXNzaW9uX2NvbCwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb2x1bW4sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGVmdF9lZGdlc19jb3B5CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICApLmFsaWFzKCJiaW5faWQiKQogICAgICAgICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICAgICAgLmdyb3VwQnkoImJpbl9pZCIpLmNvdW50KCkKICAgICAgICAgICAgICAgICAgICkudG9QYW5kYXMoKQoKICAgICAgICBiaW5fZGF0YS5pbmRleCA9IGJpbl9kYXRhWyJiaW5faWQiXQogICAgICAgIG5ld19pbmRleCA9IGxpc3QocmFuZ2UoYmlucykpCiAgICAgICAgYmluX2RhdGEgPSBiaW5fZGF0YS5yZWluZGV4KG5ld19pbmRleCkKICAgICAgICBiaW5fZGF0YVsiYmluX2lkIl0gPSBiaW5fZGF0YS5pbmRleAogICAgICAgIGJpbl9kYXRhID0gYmluX2RhdGEuZmlsbG5hKDApCgogICAgICAgIGJpbl9kYXRhWyJsZWZ0X2VkZ2UiXSA9IGxlZnRfZWRnZXMKICAgICAgICBiaW5fZGF0YVsid2lkdGgiXSA9IGJpbl93aWR0aAogICAgICAgIAoKICAgICAgICByZXR1cm4gYmluX2RhdGEKCgogICAgZGVmIGRlc2NyaWJlX2ludGVnZXJfMWQoZGYsIGNvbHVtbiwgY3VycmVudF9yZXN1bHQsIG5yb3dzKToKICAgICAgICAKICAgICAgICBzdGF0c19kZiA9IGRmLnNlbGVjdChjb2x1bW4pLm5hLmRyb3AoKS5hZ2cobWVhbihjb2woY29sdW1uKSkuYWxpYXMoIm1lYW4iKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRmX21pbihjb2woY29sdW1uKSkuYWxpYXMoIm1pbiIpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGZfbWF4KGNvbChjb2x1bW4pKS5hbGlhcygibWF4IiksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB2YXJpYW5jZShjb2woY29sdW1uKSkuYWxpYXMoInZhcmlhbmNlIiksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBrdXJ0b3Npcyhjb2woY29sdW1uKSkuYWxpYXMoImt1cnRvc2lzIiksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzdGRkZXYoY29sKGNvbHVtbikpLmFsaWFzKCJzdGQiKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNrZXduZXNzKGNvbChjb2x1bW4pKS5hbGlhcygic2tld25lc3MiKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRmX3N1bShjb2woY29sdW1uKSkuYWxpYXMoInN1bSIpCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICApLnRvUGFuZGFzKCkKCgogICAgICAgIGZvciB4IGluIG5wLmFycmF5KFswLjA1LCAwLjI1LCAwLjUsIDAuNzUsIDAuOTVdKToKICAgICAgICAgICAgc3RhdHNfZGZbcHJldHR5X25hbWUoeCldID0gKGRmLnNlbGVjdChjb2x1bW4pCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAubmEuZHJvcCgpCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAuc2VsZWN0RXhwcigicGVyY2VudGlsZShge2NvbH1gLENBU1Qoe259IEFTIERPVUJMRSkpIgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLmZvcm1hdChjb2w9Y29sdW1uLCBuPXgpKS50b1BhbmRhcygpLmlsb2NbOiwwXQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKQogICAgICAgIHN0YXRzID0gc3RhdHNfZGYuaWxvY1swXS5jb3B5KCkKICAgICAgICBzdGF0cy5uYW1lID0gY29sdW1uCiAgICAgICAgc3RhdHNbInJhbmdlIl0gPSBzdGF0c1sibWF4Il0gLSBzdGF0c1sibWluIl0KICAgICAgICBzdGF0c1siaXFyIl0gPSBzdGF0c1twcmV0dHlfbmFtZSgwLjc1KV0gLSBzdGF0c1twcmV0dHlfbmFtZSgwLjI1KV0KICAgICAgICBzdGF0c1siY3YiXSA9IHN0YXRzWyJzdGQiXSAvIGZsb2F0KHN0YXRzWyJtZWFuIl0pCiAgICAgICAgc3RhdHNbIm1hZCJdID0gKGRmLnNlbGVjdChjb2x1bW4pCiAgICAgICAgICAgICAgICAgICAgICAgIC5uYS5kcm9wKCkKICAgICAgICAgICAgICAgICAgICAgICAgLnNlbGVjdChkZl9hYnMoY29sKGNvbHVtbiktc3RhdHNbIm1lYW4iXSkuYWxpYXMoImRlbHRhIikpCiAgICAgICAgICAgICAgICAgICAgICAgIC5hZ2coZGZfc3VtKGNvbCgiZGVsdGEiKSkpLnRvUGFuZGFzKCkuaWxvY1swLDBdIC8gZmxvYXQoY3VycmVudF9yZXN1bHRbImNvdW50Il0pKQogICAgICAgIHN0YXRzWyJ0eXBlIl0gPSAiTlVNIgogICAgICAgIHN0YXRzWyduX3plcm9zJ10gPSBkZi5zZWxlY3QoY29sdW1uKS53aGVyZShjb2woY29sdW1uKT09MC4wKS5jb3VudCgpCiAgICAgICAgc3RhdHNbJ3BfemVyb3MnXSA9IHN0YXRzWyduX3plcm9zJ10gLyBmbG9hdChucm93cykKCiAgICAgICAgaGlzdF9kYXRhID0gY3JlYXRlX2hpc3RfZGF0YShkZiwgY29sdW1uLCBzdGF0c1sibWluIl0sIHN0YXRzWyJtYXgiXSwgYmlucykKCiAgICAgICAgcmV0dXJuIHN0YXRzCgogICAgZGVmIGRlc2NyaWJlX2Zsb2F0XzFkKGRmLCBjb2x1bW4sIGN1cnJlbnRfcmVzdWx0LCBucm93cyk6CiAgICAgICAgc3RhdHNfZGYgPSBkZi5zZWxlY3QoY29sdW1uKS5uYS5kcm9wKCkuYWdnKG1lYW4oY29sKGNvbHVtbikpLmFsaWFzKCJtZWFuIiksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkZl9taW4oY29sKGNvbHVtbikpLmFsaWFzKCJtaW4iKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRmX21heChjb2woY29sdW1uKSkuYWxpYXMoIm1heCIpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFyaWFuY2UoY29sKGNvbHVtbikpLmFsaWFzKCJ2YXJpYW5jZSIpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAga3VydG9zaXMoY29sKGNvbHVtbikpLmFsaWFzKCJrdXJ0b3NpcyIpLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RkZGV2KGNvbChjb2x1bW4pKS5hbGlhcygic3RkIiksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBza2V3bmVzcyhjb2woY29sdW1uKSkuYWxpYXMoInNrZXduZXNzIiksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkZl9zdW0oY29sKGNvbHVtbikpLmFsaWFzKCJzdW0iKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKS50b1BhbmRhcygpCgogICAgICAgIGZvciB4IGluIG5wLmFycmF5KFswLjA1LCAwLjI1LCAwLjUsIDAuNzUsIDAuOTVdKToKICAgICAgICAgICAgc3RhdHNfZGZbcHJldHR5X25hbWUoeCldID0gKGRmLnNlbGVjdChjb2x1bW4pCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAubmEuZHJvcCgpCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAuc2VsZWN0RXhwcigicGVyY2VudGlsZV9hcHByb3goYHtjb2x9YCxDQVNUKHtufSBBUyBET1VCTEUpKSIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC5mb3JtYXQoY29sPWNvbHVtbiwgbj14KSkudG9QYW5kYXMoKS5pbG9jWzosMF0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICkKICAgICAgICBzdGF0cyA9IHN0YXRzX2RmLmlsb2NbMF0uY29weSgpCiAgICAgICAgc3RhdHMubmFtZSA9IGNvbHVtbgogICAgICAgIHN0YXRzWyJyYW5nZSJdID0gc3RhdHNbIm1heCJdIC0gc3RhdHNbIm1pbiJdCiAgICAgICAgc3RhdHNbImlxciJdID0gc3RhdHNbcHJldHR5X25hbWUoMC43NSldIC0gc3RhdHNbcHJldHR5X25hbWUoMC4yNSldCiAgICAgICAgc3RhdHNbImN2Il0gPSBzdGF0c1sic3RkIl0gLyBmbG9hdChzdGF0c1sibWVhbiJdKQogICAgICAgIHN0YXRzWyJtYWQiXSA9IChkZi5zZWxlY3QoY29sdW1uKQogICAgICAgICAgICAgICAgICAgICAgICAubmEuZHJvcCgpCiAgICAgICAgICAgICAgICAgICAgICAgIC5zZWxlY3QoZGZfYWJzKGNvbChjb2x1bW4pLXN0YXRzWyJtZWFuIl0pLmFsaWFzKCJkZWx0YSIpKQogICAgICAgICAgICAgICAgICAgICAgICAuYWdnKGRmX3N1bShjb2woImRlbHRhIikpKS50b1BhbmRhcygpLmlsb2NbMCwwXSAvIGZsb2F0KGN1cnJlbnRfcmVzdWx0WyJjb3VudCJdKSkKICAgICAgICBzdGF0c1sidHlwZSJdID0gIk5VTSIKICAgICAgICBzdGF0c1snbl96ZXJvcyddID0gZGYuc2VsZWN0KGNvbHVtbikud2hlcmUoY29sKGNvbHVtbik9PTAuMCkuY291bnQoKQogICAgICAgIHN0YXRzWydwX3plcm9zJ10gPSBzdGF0c1snbl96ZXJvcyddIC8gZmxvYXQobnJvd3MpCgogICAgICAgIGhpc3RfZGF0YSA9IGNyZWF0ZV9oaXN0X2RhdGEoZGYsIGNvbHVtbiwgc3RhdHNbIm1pbiJdLCBzdGF0c1sibWF4Il0sIGJpbnMpCgogICAgICAgIHJldHVybiBzdGF0cwoKICAgIGRlZiBkZXNjcmliZV9kYXRlXzFkKGRmLCBjb2x1bW4pOgogICAgICAgIHN0YXRzX2RmID0gZGYuc2VsZWN0KGNvbHVtbikubmEuZHJvcCgpLmFnZyhkZl9taW4oY29sKGNvbHVtbikpLmFsaWFzKCJtaW4iKSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZGZfbWF4KGNvbChjb2x1bW4pKS5hbGlhcygibWF4IikKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICApLnRvUGFuZGFzKCkKICAgICAgICBzdGF0cyA9IHN0YXRzX2RmLmlsb2NbMF0uY29weSgpCiAgICAgICAgc3RhdHMubmFtZSA9IGNvbHVtbgoKICAgICAgICBpZiBpc2luc3RhbmNlKHN0YXRzWyJtYXgiXSwgcGQuVGltZXN0YW1wKToKICAgICAgICAgICAgc3RhdHMgPSBzdGF0cy5hc3R5cGUob2JqZWN0KQogICAgICAgICAgICBzdGF0c1sibWF4Il0gPSBzdHIoc3RhdHNbIm1heCJdLnRvX3B5ZGF0ZXRpbWUoKSkKICAgICAgICAgICAgc3RhdHNbIm1pbiJdID0gc3RyKHN0YXRzWyJtaW4iXS50b19weWRhdGV0aW1lKCkpCgogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHN0YXRzWyJyYW5nZSJdID0gc3RhdHNbIm1heCJdIC0gc3RhdHNbIm1pbiJdCiAgICAgICAgc3RhdHNbInR5cGUiXSA9ICJEQVRFIgogICAgICAgIHJldHVybiBzdGF0cwoKICAgIGRlZiBndWVzc19qc29uX3R5cGUoc3RyaW5nX3ZhbHVlKToKICAgICAgICB0cnk6CiAgICAgICAgICAgIG9iaiA9IGpzb24ubG9hZHMoc3RyaW5nX3ZhbHVlKQogICAgICAgIGV4Y2VwdDoKICAgICAgICAgICAgcmV0dXJuIE5vbmUKCiAgICAgICAgcmV0dXJuIHR5cGUob2JqKQoKICAgIGRlZiBkZXNjcmliZV9jYXRlZ29yaWNhbF8xZChkZiwgY29sdW1uKToKICAgICAgICB2YWx1ZV9jb3VudHMgPSAoZGYuc2VsZWN0KGNvbHVtbikubmEuZHJvcCgpCiAgICAgICAgICAgICAgICAgICAgICAgIC5ncm91cEJ5KGNvbHVtbikKICAgICAgICAgICAgICAgICAgICAgICAgLmFnZyhjb3VudChjb2woY29sdW1uKSkpCiAgICAgICAgICAgICAgICAgICAgICAgIC5vcmRlckJ5KCJjb3VudCh7Y30pIi5mb3JtYXQoYz1jb2x1bW4pLGFzY2VuZGluZz1GYWxzZSkKICAgICAgICAgICAgICAgICAgICAgICApLmNhY2hlKCkKCiAgICAgICAgc3RhdHMgPSAodmFsdWVfY291bnRzCiAgICAgICAgICAgICAgICAgLmxpbWl0KDEpCiAgICAgICAgICAgICAgICAgLndpdGhDb2x1bW5SZW5hbWVkKGNvbHVtbiwgInRvcCIpCiAgICAgICAgICAgICAgICAgLndpdGhDb2x1bW5SZW5hbWVkKCJjb3VudCh7Y30pIi5mb3JtYXQoYz1jb2x1bW4pLCAiZnJlcSIpCiAgICAgICAgICAgICAgICApLnRvUGFuZGFzKCkuaWxvY1swXQoKICAgICAgICB0b3BfNTAgPSB2YWx1ZV9jb3VudHMubGltaXQoNTApLnRvUGFuZGFzKCkuc29ydF92YWx1ZXMoImNvdW50KHtjfSkiLmZvcm1hdChjPWNvbHVtbiksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGFzY2VuZGluZz1GYWxzZSkKICAgICAgICB0b3BfNTBfY2F0ZWdvcmllcyA9IHRvcF81MFtjb2x1bW5dLnZhbHVlcy50b2xpc3QoKQoKICAgICAgICBvdGhlcnNfY291bnQgPSBwZC5TZXJpZXMoW2RmLnNlbGVjdChjb2x1bW4pLm5hLmRyb3AoKQogICAgICAgICAgICAgICAgICAgICAgICAud2hlcmUofihjb2woY29sdW1uKS5pc2luKCp0b3BfNTBfY2F0ZWdvcmllcykpKQogICAgICAgICAgICAgICAgICAgICAgICAuY291bnQoKQogICAgICAgICAgICAgICAgICAgICAgICBdLCBpbmRleD1bIioqKk90aGVyIFZhbHVlcyoqKiJdKQogICAgICAgIG90aGVyc19kaXN0aW5jdF9jb3VudCA9IHBkLlNlcmllcyhbdmFsdWVfY291bnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLndoZXJlKH4oY29sKGNvbHVtbikuaXNpbigqdG9wXzUwX2NhdGVnb3JpZXMpKSkKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAuY291bnQoKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIF0sIGluZGV4PVsiKioqT3RoZXIgVmFsdWVzIERpc3RpbmN0IENvdW50KioqIl0pCgogICAgICAgIHRvcCA9IHRvcF81MC5zZXRfaW5kZXgoY29sdW1uKVsiY291bnQoe2N9KSIuZm9ybWF0KGM9Y29sdW1uKV0KICAgICAgICB0b3AgPSB0b3AuYXBwZW5kKG90aGVyc19jb3VudCkKICAgICAgICB0b3AgPSB0b3AuYXBwZW5kKG90aGVyc19kaXN0aW5jdF9jb3VudCkKICAgICAgICBzdGF0c1sidmFsdWVfY291bnRzIl0gPSB0b3AKICAgICAgICBzdGF0c1sidHlwZSJdID0gIkNBVCIKICAgICAgICB2YWx1ZV9jb3VudHMudW5wZXJzaXN0KCkKICAgICAgICB1bnBhcnNlZF92YWxpZF9qc29ucyA9IGRmLnNlbGVjdChjb2x1bW4pLm5hLmRyb3AoKS5yZGQubWFwKAogICAgICAgICAgICBsYW1iZGEgeDogZ3Vlc3NfanNvbl90eXBlKHhbY29sdW1uXSkpLmZpbHRlcigKICAgICAgICAgICAgbGFtYmRhIHg6IHgpLmRpc3RpbmN0KCkuY29sbGVjdCgpCiAgICAgICAgc3RhdHNbInVucGFyc2VkX2pzb25fdHlwZXMiXSA9IHVucGFyc2VkX3ZhbGlkX2pzb25zCiAgICAgICAgcmV0dXJuIHN0YXRzCgogICAgZGVmIGRlc2NyaWJlX2NvbnN0YW50XzFkKGRmLCBjb2x1bW4pOgogICAgICAgIHN0YXRzID0gcGQuU2VyaWVzKFsnQ09OU1QnXSwgaW5kZXg9Wyd0eXBlJ10sIG5hbWU9Y29sdW1uKQogICAgICAgIHN0YXRzWyJ2YWx1ZV9jb3VudHMiXSA9IChkZi5zZWxlY3QoY29sdW1uKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAubmEuZHJvcCgpCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC5saW1pdCgxKSkudG9QYW5kYXMoKS5pbG9jWzosMF0udmFsdWVfY291bnRzKCkKICAgICAgICByZXR1cm4gc3RhdHMKCiAgICBkZWYgZGVzY3JpYmVfdW5pcXVlXzFkKGRmLCBjb2x1bW4pOgogICAgICAgIHN0YXRzID0gcGQuU2VyaWVzKFsnVU5JUVVFJ10sIGluZGV4PVsndHlwZSddLCBuYW1lPWNvbHVtbikKICAgICAgICBzdGF0c1sidmFsdWVfY291bnRzIl0gPSAoZGYuc2VsZWN0KGNvbHVtbikKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLm5hLmRyb3AoKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAubGltaXQoNTApKS50b1BhbmRhcygpLmlsb2NbOiwwXS52YWx1ZV9jb3VudHMoKQogICAgICAgIHJldHVybiBzdGF0cwoKICAgIGRlZiBkZXNjcmliZV8xZChkZiwgY29sdW1uLCBucm93cywgbG9va3VwX2NvbmZpZz1Ob25lKToKICAgICAgICBjb2x1bW5fdHlwZSA9IGRmLnNlbGVjdChjb2x1bW4pLmR0eXBlc1swXVsxXQogICAgICAgIGlmICgiYXJyYXkiIGluIGNvbHVtbl90eXBlKSBvciAoInN0dWN0IiBpbiBjb2x1bW5fdHlwZSkgb3IgKCJtYXAiIGluIGNvbHVtbl90eXBlKToKICAgICAgICAgICAgcmFpc2UgTm90SW1wbGVtZW50ZWRFcnJvcigiQ29sdW1uIHtjfSBpcyBvZiB0eXBlIHt0fSBhbmQgY2Fubm90IGJlIGFuYWx5emVkIi5mb3JtYXQoYz1jb2x1bW4sIHQ9Y29sdW1uX3R5cGUpKQoKICAgICAgICBkaXN0aW5jdF9jb3VudCA9IGRmLnNlbGVjdChjb2x1bW4pLmFnZyhjb3VudERpc3RpbmN0KGNvbChjb2x1bW4pKS5hbGlhcygiZGlzdGluY3RfY291bnQiKSkudG9QYW5kYXMoKQogICAgICAgIG5vbl9uYW5fY291bnQgPSBkZi5zZWxlY3QoY29sdW1uKS5uYS5kcm9wKCkuc2VsZWN0KGNvdW50KGNvbChjb2x1bW4pKS5hbGlhcygiY291bnQiKSkudG9QYW5kYXMoKQogICAgICAgIHJlc3VsdHNfZGF0YSA9IHBkLmNvbmNhdChbZGlzdGluY3RfY291bnQsIG5vbl9uYW5fY291bnRdLGF4aXM9MSkKICAgICAgICByZXN1bHRzX2RhdGFbInBfdW5pcXVlIl0gPSByZXN1bHRzX2RhdGFbImRpc3RpbmN0X2NvdW50Il0gLyBmbG9hdChyZXN1bHRzX2RhdGFbImNvdW50Il0pCiAgICAgICAgcmVzdWx0c19kYXRhWyJpc191bmlxdWUiXSA9IHJlc3VsdHNfZGF0YVsiZGlzdGluY3RfY291bnQiXSA9PSBucm93cwogICAgICAgIHJlc3VsdHNfZGF0YVsibl9taXNzaW5nIl0gPSBucm93cyAtIHJlc3VsdHNfZGF0YVsiY291bnQiXQogICAgICAgIHJlc3VsdHNfZGF0YVsicF9taXNzaW5nIl0gPSByZXN1bHRzX2RhdGFbIm5fbWlzc2luZyJdIC8gZmxvYXQobnJvd3MpCiAgICAgICAgcmVzdWx0c19kYXRhWyJwX2luZmluaXRlIl0gPSAwCiAgICAgICAgcmVzdWx0c19kYXRhWyJuX2luZmluaXRlIl0gPSAwCiAgICAgICAgcmVzdWx0ID0gcmVzdWx0c19kYXRhLmlsb2NbMF0uY29weSgpCiAgICAgICAgcmVzdWx0WyJtZW1vcnlzaXplIl0gPSAwCiAgICAgICAgcmVzdWx0Lm5hbWUgPSBjb2x1bW4KCiAgICAgICAgaWYgcmVzdWx0WyJkaXN0aW5jdF9jb3VudCJdIDw9IDE6CiAgICAgICAgICAgIHJlc3VsdCA9IHJlc3VsdC5hcHBlbmQoZGVzY3JpYmVfY29uc3RhbnRfMWQoZGYsIGNvbHVtbikpCiAgICAgICAgZWxpZiBjb2x1bW5fdHlwZSBpbiB7InRpbnlpbnQiLCAic21hbGxpbnQiLCAiaW50IiwgImJpZ2ludCJ9OgogICAgICAgICAgICByZXN1bHQgPSByZXN1bHQuYXBwZW5kKGRlc2NyaWJlX2ludGVnZXJfMWQoZGYsIGNvbHVtbiwgcmVzdWx0LCBucm93cykpCiAgICAgICAgZWxpZiBjb2x1bW5fdHlwZSBpbiB7ImZsb2F0IiwgImRvdWJsZSIsICJkZWNpbWFsIn06CiAgICAgICAgICAgIHJlc3VsdCA9IHJlc3VsdC5hcHBlbmQoZGVzY3JpYmVfZmxvYXRfMWQoZGYsIGNvbHVtbiwgcmVzdWx0LCBucm93cykpCiAgICAgICAgZWxpZiBjb2x1bW5fdHlwZSBpbiB7ImRhdGUiLCAidGltZXN0YW1wIn06CiAgICAgICAgICAgIHJlc3VsdCA9IHJlc3VsdC5hcHBlbmQoZGVzY3JpYmVfZGF0ZV8xZChkZiwgY29sdW1uKSkKICAgICAgICBlbGlmIHJlc3VsdFsiaXNfdW5pcXVlIl0gPT0gVHJ1ZToKICAgICAgICAgICAgcmVzdWx0ID0gcmVzdWx0LmFwcGVuZChkZXNjcmliZV91bmlxdWVfMWQoZGYsIGNvbHVtbikpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmVzdWx0ID0gcmVzdWx0LmFwcGVuZChkZXNjcmliZV9jYXRlZ29yaWNhbF8xZChkZiwgY29sdW1uKSkKICAgICAgICAgICAgaWYgcmVzdWx0WyJuX21pc3NpbmciXSA+IDA6CiAgICAgICAgICAgICAgICByZXN1bHRbImRpc3RpbmN0X2NvdW50Il0gPSByZXN1bHRbImRpc3RpbmN0X2NvdW50Il0gKyAxCgogICAgICAgIGlmIChyZXN1bHRbImNvdW50Il0gPiByZXN1bHRbImRpc3RpbmN0X2NvdW50Il0gPiAxKToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgcmVzdWx0WyJtb2RlIl0gPSByZXN1bHRbInRvcCJdCiAgICAgICAgICAgIGV4Y2VwdCBLZXlFcnJvcjoKICAgICAgICAgICAgICAgIHJlc3VsdFsibW9kZSJdID0gMAogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIHJlc3VsdFsibW9kZSJdID0gcmVzdWx0WyJ2YWx1ZV9jb3VudHMiXS5pbmRleFswXQogICAgICAgICAgICBleGNlcHQgS2V5RXJyb3I6CiAgICAgICAgICAgICAgICByZXN1bHRbIm1vZGUiXSA9IDAKICAgICAgICAgICAgZXhjZXB0IEluZGV4RXJyb3I6CiAgICAgICAgICAgICAgICByZXN1bHRbIm1vZGUiXSA9ICJNSVNTSU5HIgoKICAgICAgICBpZiBsb29rdXBfY29uZmlnOgogICAgICAgICAgICBsb29rdXBfb2JqZWN0ID0gbG9va3VwX2NvbmZpZ1snb2JqZWN0J10KICAgICAgICAgICAgY29sX25hbWVfaW5fZGIgPSBsb29rdXBfY29uZmlnWydjb2xfbmFtZV9pbl9kYiddIGlmICdjb2xfbmFtZV9pbl9kYicgaW4gbG9va3VwX2NvbmZpZyBlbHNlIE5vbmUKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgbWF0Y2hlZCwgdW5tYXRjaGVkID0gbG9va3VwX29iamVjdC5sb29rdXAoZGYuc2VsZWN0KGNvbHVtbiksIGNvbF9uYW1lX2luX2RiKQogICAgICAgICAgICAgICAgcmVzdWx0Wydsb29rZWR1cF92YWx1ZXMnXSA9IHN0cihtYXRjaGVkLmNvdW50KCkpICsgIi8iICsgc3RyKGRmLnNlbGVjdChjb2x1bW4pLmNvdW50KCkpCiAgICAgICAgICAgIGV4Y2VwdDoKICAgICAgICAgICAgICAgIHJlc3VsdFsnbG9va2VkdXBfdmFsdWVzJ10gPSAnRkFJTEVEJwogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHJlc3VsdFsnbG9va2VkdXBfdmFsdWVzJ10gPSAnJwoKICAgICAgICByZXR1cm4gcmVzdWx0CgoKICAgIGxkZXNjID0ge30KICAgIGZvciBjb2x1bSBpbiBkZi5jb2x1bW5zOgogICAgICAgIGlmIGNvbHVtIGluIGNvbmZpZzoKICAgICAgICAgICAgaWYgJ2xvb2t1cCcgaW4gY29uZmlnW2NvbHVtXToKICAgICAgICAgICAgICAgIGxvb2t1cF9jb25maWcgPSBjb25maWdbY29sdW1dWydsb29rdXAnXQogICAgICAgICAgICAgICAgZGVzYyA9IGRlc2NyaWJlXzFkKGRmLCBjb2x1bSwgdGFibGVfc3RhdHNbIm4iXSwgbG9va3VwX2NvbmZpZz1sb29rdXBfY29uZmlnKQogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgZGVzYyA9IGRlc2NyaWJlXzFkKGRmLCBjb2x1bSwgdGFibGVfc3RhdHNbIm4iXSkKICAgICAgICBlbHNlOgogICAgICAgICAgICBkZXNjID0gZGVzY3JpYmVfMWQoZGYsIGNvbHVtLCB0YWJsZV9zdGF0c1sibiJdKQogICAgICAgIGxkZXNjLnVwZGF0ZSh7Y29sdW06IGRlc2N9KQoKICAgIGlmIGNvcnJfcmVqZWN0IGlzIG5vdCBOb25lOgogICAgICAgIGNvbXB1dGFibGVfY29ycnMgPSBbY29sdW0gZm9yIGNvbHVtIGluIGxkZXNjIGlmIGxkZXNjW2NvbHVtXVsidHlwZSJdIGluIHsiTlVNIn1dCgogICAgICAgIGlmIGxlbihjb21wdXRhYmxlX2NvcnJzKSA+IDA6CiAgICAgICAgICAgIGNvcnIgPSBjb3JyX21hdHJpeChkZiwgY29sdW1ucz1jb21wdXRhYmxlX2NvcnJzKQogICAgICAgICAgICBmb3IgeCwgY29ycl94IGluIGNvcnIuaXRlcnJvd3MoKToKICAgICAgICAgICAgICAgIGZvciB5LCBjb3JyIGluIGNvcnJfeC5pdGVyaXRlbXMoKToKICAgICAgICAgICAgICAgICAgICBpZiB4ID09IHk6CiAgICAgICAgICAgICAgICAgICAgICAgIGJyZWFrCgogICAgdmFyaWFibGVfc3RhdHMgPSBwZC5EYXRhRnJhbWUobGRlc2MpCgogICAgdGFibGVfc3RhdHNbIm52YXIiXSA9IGxlbihkZi5jb2x1bW5zKQogICAgdGFibGVfc3RhdHNbInRvdGFsX21pc3NpbmciXSA9IGZsb2F0KHZhcmlhYmxlX3N0YXRzLmxvY1sibl9taXNzaW5nIl0uc3VtKCkpIC8gKHRhYmxlX3N0YXRzWyJuIl0gKiB0YWJsZV9zdGF0c1sibnZhciJdKQogICAgbWVtc2l6ZSA9IDAKICAgIHRhYmxlX3N0YXRzWydtZW1zaXplJ10gPSBmbXRfYnl0ZXNpemUobWVtc2l6ZSkKICAgIHRhYmxlX3N0YXRzWydyZWNvcmRzaXplJ10gPSBmbXRfYnl0ZXNpemUobWVtc2l6ZSAvIHRhYmxlX3N0YXRzWyduJ10pCiAgICB0YWJsZV9zdGF0cy51cGRhdGUoe2s6IDAgZm9yIGsgaW4gKCJOVU0iLCAiREFURSIsICJDT05TVCIsICJDQVQiLCAiVU5JUVVFIiwgIkNPUlIiKX0pCiAgICB0YWJsZV9zdGF0cy51cGRhdGUoZGljdCh2YXJpYWJsZV9zdGF0cy5sb2NbJ3R5cGUnXS52YWx1ZV9jb3VudHMoKSkpCiAgICB0YWJsZV9zdGF0c1snUkVKRUNURUQnXSA9IHRhYmxlX3N0YXRzWydDT05TVCddICsgdGFibGVfc3RhdHNbJ0NPUlInXQoKICAgIGZyZXFfZGljdCA9IHt9CiAgICBmb3IgdmFyIGluIHZhcmlhYmxlX3N0YXRzOgogICAgICAgIGlmICJ2YWx1ZV9jb3VudHMiIG5vdCBpbiB2YXJpYWJsZV9zdGF0c1t2YXJdOgogICAgICAgICAgICBwYXNzCiAgICAgICAgZWxpZiBub3QodmFyaWFibGVfc3RhdHNbdmFyXVsidmFsdWVfY291bnRzIl0gaXMgbnAubmFuKToKICAgICAgICAgICAgZnJlcV9kaWN0W3Zhcl0gPSB2YXJpYWJsZV9zdGF0c1t2YXJdWyJ2YWx1ZV9jb3VudHMiXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHBhc3MKICAgIHRyeToKICAgICAgICB2YXJpYWJsZV9zdGF0cyA9IHZhcmlhYmxlX3N0YXRzLmRyb3AoInZhbHVlX2NvdW50cyIpCiAgICBleGNlcHQgKFZhbHVlRXJyb3IsIEtleUVycm9yKToKICAgICAgICBwYXNzCgogICAgcmV0dXJuIHRhYmxlX3N0YXRzLCB2YXJpYWJsZV9zdGF0cy5ULCBmcmVxX2RpY3QKCmltcG9ydCBudW1weSBhcyBucApmcm9tIHB5c3Bhcmsuc3FsLmZ1bmN0aW9ucyBpbXBvcnQgYWJzIGFzIGFic291CgpTS0VXTkVTU19DVVRPRkYgPSAyMApERUZBVUxUX0ZMT0FUX0ZPUk1BVFRFUiA9IHUnc3BhcmtfZGZfcHJvZmlsaW5nLl9fZGVmYXVsdF9mbG9hdF9mb3JtYXR0ZXInCgoKZGVmIGdyYWRpZW50X2Zvcm1hdCh2YWx1ZSwgbGltaXQxLCBsaW1pdDIsIGMxLCBjMik6CiAgICBkZWYgTGVycENvbG91cihjMSxjMix0KToKICAgICAgICByZXR1cm4gKGludChjMVswXSsoYzJbMF0tYzFbMF0pKnQpLGludChjMVsxXSsoYzJbMV0tYzFbMV0pKnQpLGludChjMVsyXSsoYzJbMl0tYzFbMl0pKnQpKQogICAgYyA9IExlcnBDb2xvdXIoYzEsIGMyLCAodmFsdWUtbGltaXQxKS8obGltaXQyLWxpbWl0MSkpCiAgICByZXR1cm4gZm10X2NvbG9yKHZhbHVlLCJyZ2J7fSIuZm9ybWF0KHN0cihjKSkpCgoKZGVmIGZtdF9jb2xvcih0ZXh0LCBjb2xvcik6CiAgICByZXR1cm4odSc8c3BhbiBzdHlsZT0iY29sb3I6e2NvbG9yfSI+e3RleHR9PC9zcGFuPicuZm9ybWF0KGNvbG9yPWNvbG9yLHRleHQ9c3RyKHRleHQpKSkKCgpkZWYgZm10X2NsYXNzKHRleHQsIGNscyk6CiAgICByZXR1cm4odSc8c3BhbiBjbGFzcz0ie2Nsc30iPnt0ZXh0fTwvc3Bhbj4nLmZvcm1hdChjbHM9Y2xzLHRleHQ9c3RyKHRleHQpKSkKCgpkZWYgZm10X2J5dGVzaXplKG51bSwgc3VmZml4PSdCJyk6CiAgICBmb3IgdW5pdCBpbiBbJycsJ0tpJywnTWknLCdHaScsJ1RpJywnUGknLCdFaScsJ1ppJ106CiAgICAgICAgaWYgbnVtIDwgMDoKICAgICAgICAgICAgbnVtID0gbnVtKi0xCiAgICAgICAgICAgIGlmIG51bSA8IDEwMjQuMDoKICAgICAgICAgICAgICAgIHJldHVybiAiJTMuMWYgJXMlcyIgJSAobnVtLCB1bml0LCBzdWZmaXgpCiAgICAgICAgICAgIG51bSAvPSAxMDI0LjAKICAgIHJldHVybiAiJS4xZiAlcyVzIiAlIChudW0sICdZaScsIHN1ZmZpeCkKCgpkZWYgZm10X3BlcmNlbnQodik6CiAgICByZXR1cm4gICJ7OjIuMWZ9JSIuZm9ybWF0KHYqMTAwKQoKZGVmIGZtdF92YXJuYW1lKHYpOgogICAgcmV0dXJuIHUnPGNvZGU+ezB9PC9jb2RlPicuZm9ybWF0KHYpCgoKdmFsdWVfZm9ybWF0dGVycz17CiAgICAgICAgdSdmcmVxJzogKGxhbWJkYSB2OiBncmFkaWVudF9mb3JtYXQodiwgMCwgNjIwMDAsICgzMCwgMTk4LCAyNDQpLCAoOTksIDIwMCwgNzIpKSksCiAgICAgICAgdSdwX21pc3NpbmcnOiBmbXRfcGVyY2VudCwKICAgICAgICB1J3BfaW5maW5pdGUnOiBmbXRfcGVyY2VudCwKICAgICAgICB1J3BfdW5pcXVlJzogZm10X3BlcmNlbnQsCiAgICAgICAgdSdwX3plcm9zJzogZm10X3BlcmNlbnQsCiAgICAgICAgdSdtZW1vcnlzaXplJzogZm10X2J5dGVzaXplLAogICAgICAgIHUndG90YWxfbWlzc2luZyc6IGZtdF9wZXJjZW50LAogICAgICAgIERFRkFVTFRfRkxPQVRfRk9STUFUVEVSOiBsYW1iZGEgdjogc3RyKGZsb2F0KCd7Oi41Z30nLmZvcm1hdCh2KSkpLnJzdHJpcCgnMCcpLnJzdHJpcCgnLicpLAogICAgICAgIHUnY29ycmVsYXRpb25fdmFyJzogbGFtYmRhIHY6IGZtdF92YXJuYW1lKHYpLAogICAgICAgIHUndW5wYXJzZWRfanNvbl90eXBlcyc6IGxhbWJkYSB2OiAnLCAnLmpvaW4oW3MuX19uYW1lX18gZm9yIHMgaW4gdl0pCiAgICAgICAgfQoKZGVmIGZtdF9yb3dfc2V2ZXJpdHkodik6CiAgICBpZiBucC5pc25hbih2KSBvciB2PD0gMC4wMToKICAgICAgICByZXR1cm4gImlnbm9yZSIKICAgIGVsc2U6CiAgICAgICAgcmV0dXJuICJhbGVydCIKCmRlZiBmbXRfc2tld25lc3Modik6CiAgICBpZiBub3QgbnAuaXNuYW4odikgYW5kICh2PC1TS0VXTkVTU19DVVRPRkYgb3Igdj4gU0tFV05FU1NfQ1VUT0ZGKToKICAgICAgICByZXR1cm4gImFsZXJ0IgogICAgZWxzZToKICAgICAgICByZXR1cm4gIiIKCnJvd19mb3JtYXR0ZXJzPXsKICAgIHUncF96ZXJvcyc6IGZtdF9yb3dfc2V2ZXJpdHksCiAgICB1J3BfbWlzc2luZyc6IGZtdF9yb3dfc2V2ZXJpdHksCiAgICB1J3BfaW5maW5pdGUnOiBmbXRfcm93X3NldmVyaXR5LAogICAgdSduX2R1cGxpY2F0ZXMnOiBmbXRfcm93X3NldmVyaXR5LAogICAgdSdza2V3bmVzcyc6IGZtdF9za2V3bmVzcywKfQoKcnVuKFsiL2Jpbi9iYXNoIiwgIi9ldGMvY29uZmlnL3YzaW8vdjNpby1zcGFyay1vcGVyYXRvci5zaCJdKQoKZGVmIGRlc2NyaWJlX3NwYXJrKGNvbnRleHQ6IE1MQ2xpZW50Q3R4LCAKICAgICAgICAgICAgICAgICAgIGRhdGFzZXQ6IERhdGFJdGVtLCAKICAgICAgICAgICAgICAgICAgIGFydGlmYWN0X3BhdGgsCiAgICAgICAgICAgICAgICAgICBiaW5zOiBpbnQ9MzAsCiAgICAgICAgICAgICAgICAgICBkZXNjcmliZV9leHRlbmRlZDogYm9vbD1UcnVlKToKICAgIAogICAgbG9jYXRpb24gPSBkYXRhc2V0LmxvY2FsKCkKICAgIAogICAgc3BhcmsgPSBTcGFya1Nlc3Npb24uYnVpbGRlci5hcHBOYW1lKCJTcGFyayBqb2IiKS5nZXRPckNyZWF0ZSgpCiAgICAKICAgIGRmID0gc3BhcmsucmVhZC5jc3YobG9jYXRpb24sIGhlYWRlcj1UcnVlLCBpbmZlclNjaGVtYT0gVHJ1ZSkKCiAgICBrd2FyZ3MgPSBbXQogICAgCiAgICBmbG9hdF9jb2xzID0gW2l0ZW1bMF0gZm9yIGl0ZW0gaW4gZGYuZHR5cGVzIGlmIGl0ZW1bMV0uc3RhcnRzd2l0aCgnZmxvYXQnKSBvciBpdGVtWzFdLnN0YXJ0c3dpdGgoJ2RvdWJsZScpXQogICAgCiAgICBpZiBkZXNjcmliZV9leHRlbmRlZCA9PSBUcnVlOgogICAgICAgIAogICAgICAgIHRhYmxlLCB2YXJpYWJsZXMsIGZyZXEgPSBkZXNjcmliZShkZiwgYmlucywgZmxvYXRfY29scywga3dhcmdzKQoKICAgICAgICB0YmxfMSA9IHZhcmlhYmxlcy5yZXNldF9pbmRleCgpCgogICAgICAgIGlmIGxlbihmcmVxKSAhPSAwOgogICAgICAgICAgICB0YmxfMiA9IHBkLkRhdGFGcmFtZS5mcm9tX2RpY3QoZnJlcSwgb3JpZW50ID0gImluZGV4Iikuc29ydF9pbmRleCgpLnN0YWNrKCkucmVzZXRfaW5kZXgoKQogICAgICAgICAgICB0YmxfMi5jb2x1bW5zID0gWydjb2wnLCAna2V5JywgJ3ZhbCddCiAgICAgICAgICAgIHRibF8yWydNZXJnZWQnXSA9IFt7a2V5OiB2YWx9IGZvciBrZXksIHZhbCBpbiB6aXAodGJsXzIua2V5LCB0YmxfMi52YWwpXQogICAgICAgICAgICB0YmxfMiA9IHRibF8yLmdyb3VwYnkoJ2NvbCcsIGFzX2luZGV4PUZhbHNlKS5hZ2cobGFtYmRhIHg6IHR1cGxlKHgpKVtbJ2NvbCcsJ01lcmdlZCddXQoKICAgICAgICAgICAgc3VtbWFyeSA9IHBkLm1lcmdlKHRibF8xLCB0YmxfMiwgaG93PSdsZWZ0JywgbGVmdF9vbj0naW5kZXgnLCByaWdodF9vbj0nY29sJykKCiAgICAgICAgZWxzZToKICAgICAgICAgICAgc3VtbWFyeSA9IHRibF8xCgogICAgICAgIGNvbnRleHQubG9nX2RhdGFzZXQoInN1bW1hcnlfc3RhdHMiLCAKICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRmPXN1bW1hcnksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb3JtYXQ9ImNzdiIsIGluZGV4PUZhbHNlLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgYXJ0aWZhY3RfcGF0aD1jb250ZXh0LmFydGlmYWN0X3N1YnBhdGgoJ2RhdGEnKSkKCiAgICAgICAgY29udGV4dC5sb2dfcmVzdWx0cyh0YWJsZSkKICAgIAogICAgZWxzZToKICAgICAgICB0YmxfMSA9IGRmLmRlc2NyaWJlKCkudG9QYW5kYXMoKQogICAgICAgIAogICAgICAgIHN1bW1hcnkgPSB0YmxfMS5UCiAgICAgICAgCiAgICAgICAgY29udGV4dC5sb2dfZGF0YXNldCgic3VtbWFyeV9zdGF0cyIsIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgZGY9c3VtbWFyeSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZvcm1hdD0iY3N2IiwgaW5kZXg9RmFsc2UsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBhcnRpZmFjdF9wYXRoPWNvbnRleHQuYXJ0aWZhY3Rfc3VicGF0aCgnZGF0YScpKQogICAgCiAgICBzcGFyay5zdG9wKCkKCg== - commands: [] - code_origin: https://github.com/daniels290813/functions.git#55a79c32be5d233cc11efcf40cd3edbe309bfdef:/home/kali/functions/describe_spark/describe_spark.py - affinity: null -verbose: false + default_handler: describe_spark diff --git a/functions/src/feature_selection/feature_selection.py b/functions/src/feature_selection/feature_selection.py index a046143da..af828ad7f 100644 --- a/functions/src/feature_selection/feature_selection.py +++ b/functions/src/feature_selection/feature_selection.py @@ -23,12 +23,16 @@ import plotly.express as px from mlrun.artifacts import PlotlyArtifact from mlrun.datastore.targets import ParquetTarget + # MLRun utils from mlrun.utils.helpers import create_class + # Feature selection strategies from sklearn.feature_selection import SelectFromModel, SelectKBest + # Scale feature scoresgit st from sklearn.preprocessing import MinMaxScaler + # SKLearn estimators list from sklearn.utils import all_estimators @@ -194,7 +198,7 @@ def feature_selection( selected_models = {} for model_name, model in model_filters.items(): if ".json" in model: - current_model = json.load(open(model, "r")) + current_model = json.load(open(model)) classifier_class = create_class(current_model["META"]["class"]) selected_models[model_name] = classifier_class(**current_model["CLASS"]) elif model in all_sklearn_estimators: @@ -211,7 +215,6 @@ def feature_selection( # Run model filters models_df = pd.DataFrame(index=X.columns) for model_name, model in selected_models.items(): - if model_name == "LogisticRegression": model.set_params(solver="liblinear") diff --git a/functions/src/feature_selection/function.yaml b/functions/src/feature_selection/function.yaml index 1724428d0..8cc5aaa19 100644 --- a/functions/src/feature_selection/function.yaml +++ b/functions/src/feature_selection/function.yaml @@ -1,6 +1,19 @@ +metadata: + tag: '' + name: feature-selection + categories: + - data-preparation + - machine-learning +verbose: false +kind: job spec: + image: mlrun/mlrun disable_auto_mount: false - command: '' + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IGpzb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi5mZWF0dXJlX3N0b3JlIGFzIGZzCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgbnVtcHkgYXMgbnAKaW1wb3J0IHBhbmRhcyBhcyBwZAppbXBvcnQgcGxvdGx5LmV4cHJlc3MgYXMgcHgKZnJvbSBtbHJ1bi5hcnRpZmFjdHMgaW1wb3J0IFBsb3RseUFydGlmYWN0CmZyb20gbWxydW4uZGF0YXN0b3JlLnRhcmdldHMgaW1wb3J0IFBhcnF1ZXRUYXJnZXQKCiMgTUxSdW4gdXRpbHMKZnJvbSBtbHJ1bi51dGlscy5oZWxwZXJzIGltcG9ydCBjcmVhdGVfY2xhc3MKCiMgRmVhdHVyZSBzZWxlY3Rpb24gc3RyYXRlZ2llcwpmcm9tIHNrbGVhcm4uZmVhdHVyZV9zZWxlY3Rpb24gaW1wb3J0IFNlbGVjdEZyb21Nb2RlbCwgU2VsZWN0S0Jlc3QKCiMgU2NhbGUgZmVhdHVyZSBzY29yZXNnaXQgc3QKZnJvbSBza2xlYXJuLnByZXByb2Nlc3NpbmcgaW1wb3J0IE1pbk1heFNjYWxlcgoKIyBTS0xlYXJuIGVzdGltYXRvcnMgbGlzdApmcm9tIHNrbGVhcm4udXRpbHMgaW1wb3J0IGFsbF9lc3RpbWF0b3JzCgpERUZBVUxUX1NUQVRfRklMVEVSUyA9IFsiZl9jbGFzc2lmIiwgIm11dHVhbF9pbmZvX2NsYXNzaWYiLCAiY2hpMiIsICJmX3JlZ3Jlc3Npb24iXQpERUZBVUxUX01PREVMX0ZJTFRFUlMgPSB7CiAgICAiTGluZWFyU1ZDIjogIkxpbmVhclNWQyIsCiAgICAiTG9naXN0aWNSZWdyZXNzaW9uIjogIkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAiRXh0cmFUcmVlc0NsYXNzaWZpZXIiOiAiRXh0cmFUcmVlc0NsYXNzaWZpZXIiLAp9CgoKZGVmIHNob3dfdmFsdWVzX29uX2JhcnMoYXhzLCBoX3Y9InYiLCBzcGFjZT0wLjQpOgogICAgZGVmIF9zaG93X29uX3NpbmdsZV9wbG90KGF4Xyk6CiAgICAgICAgaWYgaF92ID09ICJ2IjoKICAgICAgICAgICAgZm9yIHAgaW4gYXhfLnBhdGNoZXM6CiAgICAgICAgICAgICAgICBfeCA9IHAuZ2V0X3goKSArIHAuZ2V0X3dpZHRoKCkgLyAyCiAgICAgICAgICAgICAgICBfeSA9IHAuZ2V0X3koKSArIHAuZ2V0X2hlaWdodCgpCiAgICAgICAgICAgICAgICB2YWx1ZSA9IGludChwLmdldF9oZWlnaHQoKSkKICAgICAgICAgICAgICAgIGF4Xy50ZXh0KF94LCBfeSwgdmFsdWUsIGhhPSJjZW50ZXIiKQogICAgICAgIGVsaWYgaF92ID09ICJoIjoKICAgICAgICAgICAgZm9yIHAgaW4gYXhfLnBhdGNoZXM6CiAgICAgICAgICAgICAgICBfeCA9IHAuZ2V0X3goKSArIHAuZ2V0X3dpZHRoKCkgKyBmbG9hdChzcGFjZSkKICAgICAgICAgICAgICAgIF95ID0gcC5nZXRfeSgpICsgcC5nZXRfaGVpZ2h0KCkKICAgICAgICAgICAgICAgIHZhbHVlID0gaW50KHAuZ2V0X3dpZHRoKCkpCiAgICAgICAgICAgICAgICBheF8udGV4dChfeCwgX3ksIHZhbHVlLCBoYT0ibGVmdCIpCgogICAgaWYgaXNpbnN0YW5jZShheHMsIG5wLm5kYXJyYXkpOgogICAgICAgIGZvciBpZHgsIGF4IGluIG5wLm5kZW51bWVyYXRlKGF4cyk6CiAgICAgICAgICAgIF9zaG93X29uX3NpbmdsZV9wbG90KGF4KQogICAgZWxzZToKICAgICAgICBfc2hvd19vbl9zaW5nbGVfcGxvdChheHMpCgoKZGVmIHBsb3Rfc3RhdChjb250ZXh0LCBzdGF0X25hbWUsIHN0YXRfZGYpOgogICAgc29ydGVkX2RmID0gc3RhdF9kZi5zb3J0X3ZhbHVlcyhzdGF0X25hbWUpCiAgICBmaWcgPSBweC5iYXIoCiAgICAgICAgZGF0YV9mcmFtZT1zb3J0ZWRfZGYsCiAgICAgICAgeD1zdGF0X25hbWUsCiAgICAgICAgeT1zb3J0ZWRfZGYuaW5kZXgsCiAgICAgICAgdGl0bGU9ZiJ7c3RhdF9uYW1lfSBmZWF0dXJlIHNjb3JlcyIsCiAgICAgICAgY29sb3I9c3RhdF9uYW1lLAogICAgKQogICAgY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgaXRlbT1QbG90bHlBcnRpZmFjdChrZXk9c3RhdF9uYW1lLCBmaWd1cmU9ZmlnKSwKICAgICAgICBsb2NhbF9wYXRoPWYie3N0YXRfbmFtZX0uaHRtbCIsCiAgICApCgoKZGVmIGZlYXR1cmVfc2VsZWN0aW9uKAogICAgY29udGV4dCwKICAgIGRmX2FydGlmYWN0LAogICAgazogaW50ID0gNSwKICAgIG1pbl92b3RlczogZmxvYXQgPSAwLjUsCiAgICBsYWJlbF9jb2x1bW46IHN0ciA9IE5vbmUsCiAgICBzdGF0X2ZpbHRlcnM6IGxpc3QgPSBOb25lLAogICAgbW9kZWxfZmlsdGVyczogZGljdCA9IE5vbmUsCiAgICBtYXhfc2NhbGVkX3Njb3JlczogYm9vbCA9IFRydWUsCiAgICBzYW1wbGVfcmF0aW86IGZsb2F0ID0gTm9uZSwKICAgIG91dHB1dF92ZWN0b3JfbmFtZTogZmxvYXQgPSBOb25lLAogICAgaWdub3JlX3R5cGVfZXJyb3JzOiBib29sID0gRmFsc2UsCik6CiAgICAiIiIKICAgIEFwcGxpZXMgc2VsZWN0ZWQgZmVhdHVyZSBzZWxlY3Rpb24gc3RhdGlzdGljYWwgZnVuY3Rpb25zIG9yIG1vZGVscyBvbiBvdXIgJ2RmX2FydGlmYWN0Jy4KCiAgICBFYWNoIHN0YXRpc3RpY2FsIGZ1bmN0aW9uIG9yIG1vZGVsIHdpbGwgdm90ZSBmb3IgaXQncyBiZXN0IEsgc2VsZWN0ZWQgZmVhdHVyZXMuCiAgICBJZiBhIGZlYXR1cmUgaGFzID49ICdtaW5fdm90ZXMnIHZvdGVzLCBpdCB3aWxsIGJlIHNlbGVjdGVkLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICB0aGUgZnVuY3Rpb24gY29udGV4dC4KICAgIDpwYXJhbSBkZl9hcnRpZmFjdDogICAgICAgICBkYXRhZnJhbWUgdG8gcGFzcyBhcyBpbnB1dC4KICAgIDpwYXJhbSBrOiAgICAgICAgICAgICAgICAgICBudW1iZXIgb2YgdG9wIGZlYXR1cmVzIHRvIHNlbGVjdCBmcm9tIGVhY2ggc3RhdGlzdGljYWwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmdW5jdGlvbiBvciBtb2RlbC4KICAgIDpwYXJhbSBtaW5fdm90ZXM6ICAgICAgICAgICBtaW5pbWFsIG51bWJlciBvZiB2b3RlcyAoZnJvbSBhIG1vZGVsIG9yIGJ5IHN0YXRpc3RpY2FsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZnVuY3Rpb24pIG5lZWRlZCBmb3IgYSBmZWF0dXJlIHRvIGJlIHNlbGVjdGVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENhbiBiZSBzcGVjaWZpZWQgYnkgcGVyY2VudGFnZSBvZiB2b3RlcyBvciBhYnNvbHV0ZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG51bWJlciBvZiB2b3Rlcy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW46ICAgICAgICBncm91bmQtdHJ1dGggKHkpIGxhYmVscy4KICAgIDpwYXJhbSBzdGF0X2ZpbHRlcnM6ICAgICAgICBzdGF0aXN0aWNhbCBmdW5jdGlvbnMgdG8gYXBwbHkgdG8gdGhlIGZlYXR1cmVzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKGZyb20gc2tsZWFybi5mZWF0dXJlX3NlbGVjdGlvbikuCiAgICA6cGFyYW0gbW9kZWxfZmlsdGVyczogICAgICAgbW9kZWxzIHRvIHVzZSBmb3IgZmVhdHVyZSBldmFsdWF0aW9uLCBjYW4gYmUgc3BlY2lmaWVkIGJ5CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kZWwgbmFtZSAoZXguIExpbmVhclNWQyksIGZvcm1hbGl6ZWQganNvbiAoY29udGFpbnMgJ0NMQVNTJywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnRklUJywgJ01FVEEnKSBvciBhIHBhdGggdG8gc3VjaCBqc29uIGZpbGUuCiAgICA6cGFyYW0gbWF4X3NjYWxlZF9zY29yZXM6ICAgcHJvZHVjZSBmZWF0dXJlIHNjb3JlcyB0YWJsZSBzY2FsZWQgd2l0aCBtYXhfc2NhbGVyLgogICAgOnBhcmFtIHNhbXBsZV9yYXRpbzogICAgICAgIHBlcmNlbnRhZ2Ugb2YgdGhlIGRhdGFzZXQgdGhlIHVzZXIgd2lzaGVzIHRvIGNvbXB1dGUgdGhlIGZlYXR1cmUgc2VsZWN0aW9uIHByb2Nlc3Mgb24uCiAgICA6cGFyYW0gb3V0cHV0X3ZlY3Rvcl9uYW1lOiAgY3JlYXRlcyBhIG5ldyBmZWF0dXJlIHZlY3RvciBjb250YWluaW5nIG9ubHkgdGhlIGlkZW50aWZpZXMgZmVhdHVyZXMuCiAgICA6cGFyYW0gaWdub3JlX3R5cGVfZXJyb3JzOiAgc2tpcHMgZGF0YXR5cGVzIHRoYXQgYXJlIG5laXRoZXIgZmxvYXQgbm9yIGludCB3aXRoaW4gdGhlIGZlYXR1cmUgdmVjdG9yLgogICAgIiIiCiAgICBzdGF0X2ZpbHRlcnMgPSBzdGF0X2ZpbHRlcnMgb3IgREVGQVVMVF9TVEFUX0ZJTFRFUlMKICAgIG1vZGVsX2ZpbHRlcnMgPSBtb2RlbF9maWx0ZXJzIG9yIERFRkFVTFRfTU9ERUxfRklMVEVSUwogICAgIyBDaGVjayBpZiBkZi5tZXRhIGlzIHZhbGlkLCBpZiBpdCBpcywgbG9vayBmb3IgYSBmZWF0dXJlIHZlY3RvcgogICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGZfYXJ0aWZhY3QuYXJ0aWZhY3RfdXJsKQogICAgaXNfZmVhdHVyZV92ZWN0b3IgPSBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXgKCiAgICAjIExvb2sgaW5zaWRlIG1ldGEuc3BlYy5sYWJlbF9mZWF0dXJlIHRvIGlkZW50aWZ5IHRoZSBsYWJlbF9jb2x1bW4gaWYgdGhlIHVzZXIgZGlkIG5vdCBzcGVjaWZ5IGl0CiAgICBpZiBsYWJlbF9jb2x1bW4gaXMgTm9uZToKICAgICAgICBpZiBpc19mZWF0dXJlX3ZlY3RvcjoKICAgICAgICAgICAgbGFiZWxfY29sdW1uID0gZGZfYXJ0aWZhY3QubWV0YS5zcGVjLmxhYmVsX2ZlYXR1cmUuc3BsaXQoIi4iKVsxXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoIk5vIGxhYmVsX2NvbHVtbiB3YXMgZ2l2ZW4sIHBsZWFzZSBhZGQgYSBsYWJlbF9jb2x1bW4uIikKCiAgICAjIFVzZSB0aGUgZmVhdHVyZSB2ZWN0b3IgYXMgZGF0YWZyYW1lCiAgICBkZiA9IGRmX2FydGlmYWN0LmFzX2RmKCkKCiAgICAjIEVuc3VyZSBrIGlzIG5vdCBiaWdnZXIgdGhhbiB0aGUgdG90YWwgbnVtYmVyIG9mIGZlYXR1cmVzCiAgICBpZiBrID4gZGYuc2hhcGVbMV06CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJLIGNhbm5vdCBiZSBiaWdnZXIgdGhhbiB0aGUgdG90YWwgbnVtYmVyIG9mIGZlYXR1cmVzICh7ZGYuc2hhcGVbMV19KS4gUGxlYXNlIGNob29zZSBhIHNtYWxsZXIgSy4iCiAgICAgICAgKQogICAgZWxpZiBrIDwgMToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJLIGNhbm5vdCBiZSBzbWFsbGVyIHRoYW4gMS4gUGxlYXNlIGNob29zZSBhIGJpZ2dlciBLLiIpCgogICAgIyBDcmVhdGUgYSBzYW1wbGUgZGF0YWZyYW1lIG9mIHRoZSBvcmlnaW5hbCBmZWF0dXJlIHZlY3RvcgogICAgaWYgc2FtcGxlX3JhdGlvOgogICAgICAgIGRmID0gKAogICAgICAgICAgICBkZi5ncm91cGJ5KGxhYmVsX2NvbHVtbikKICAgICAgICAgICAgLmFwcGx5KGxhbWJkYSB4OiB4LnNhbXBsZShmcmFjPXNhbXBsZV9yYXRpbykpCiAgICAgICAgICAgIC5yZXNldF9pbmRleChkcm9wPVRydWUpCiAgICAgICAgKQogICAgICAgIGRmID0gZGYuZHJvcG5hKCkKCiAgICAjIFNldCBmZWF0dXJlIHZlY3RvciBhbmQgbGFiZWxzCiAgICB5ID0gZGYucG9wKGxhYmVsX2NvbHVtbikKICAgIFggPSBkZgoKICAgIGlmIG5wLm9iamVjdF8gaW4gbGlzdChYLmR0eXBlcykgYW5kIGlnbm9yZV90eXBlX2Vycm9ycyBpcyBGYWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIntkZi5zZWxlY3RfZHR5cGVzKGluY2x1ZGU9WydvYmplY3QnXSkuY29sdW1ucy50b2xpc3QoKX0gYXJlIG5laXRoZXIgZmxvYXQgb3IgaW50LiIKICAgICAgICApCgogICAgIyBDcmVhdGUgc2VsZWN0ZWQgc3RhdGlzdGljYWwgZXN0aW1hdG9ycwogICAgc3RhdF9mdW5jdGlvbnNfbGlzdCA9IHsKICAgICAgICBzdGF0X25hbWU6IFNlbGVjdEtCZXN0KAogICAgICAgICAgICBzY29yZV9mdW5jPWNyZWF0ZV9jbGFzcyhmInNrbGVhcm4uZmVhdHVyZV9zZWxlY3Rpb24ue3N0YXRfbmFtZX0iKSwgaz1rCiAgICAgICAgKQogICAgICAgIGZvciBzdGF0X25hbWUgaW4gc3RhdF9maWx0ZXJzCiAgICB9CiAgICByZXF1aXJlc19hYnMgPSBbImNoaTIiXQoKICAgICMgUnVuIHN0YXRpc3RpYyBmaWx0ZXJzCiAgICBzZWxlY3RlZF9mZWF0dXJlc19hZ2cgPSB7fQogICAgc3RhdHNfZGYgPSBwZC5EYXRhRnJhbWUoaW5kZXg9WC5jb2x1bW5zKS5kcm9wbmEoKQoKICAgIGZvciBzdGF0X25hbWUsIHN0YXRfZnVuYyBpbiBzdGF0X2Z1bmN0aW9uc19saXN0Lml0ZW1zKCk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICBwYXJhbXMgPSAoWCwgeSkgaWYgc3RhdF9uYW1lIGluIHJlcXVpcmVzX2FicyBlbHNlIChhYnMoWCksIHkpCiAgICAgICAgICAgIHN0YXQgPSBzdGF0X2Z1bmMuZml0KCpwYXJhbXMpCgogICAgICAgICAgICAjIENvbGxlY3Qgc3RhdCBmdW5jdGlvbiByZXN1bHRzCiAgICAgICAgICAgIHN0YXRfZGYgPSBwZC5EYXRhRnJhbWUoCiAgICAgICAgICAgICAgICBpbmRleD1YLmNvbHVtbnMsIGNvbHVtbnM9W3N0YXRfbmFtZV0sIGRhdGE9c3RhdC5zY29yZXNfCiAgICAgICAgICAgICkKICAgICAgICAgICAgcGxvdF9zdGF0KGNvbnRleHQsIHN0YXRfbmFtZSwgc3RhdF9kZikKICAgICAgICAgICAgc3RhdHNfZGYgPSBzdGF0c19kZi5qb2luKHN0YXRfZGYpCgogICAgICAgICAgICAjIFNlbGVjdCBLIEJlc3QgZmVhdHVyZXMKICAgICAgICAgICAgc2VsZWN0ZWRfZmVhdHVyZXMgPSBYLmNvbHVtbnNbc3RhdF9mdW5jLmdldF9zdXBwb3J0KCldCiAgICAgICAgICAgIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZ1tzdGF0X25hbWVdID0gc2VsZWN0ZWRfZmVhdHVyZXMKCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiQ291bGRuJ3QgY2FsY3VsYXRlIHtzdGF0X25hbWV9IGJlY2F1c2Ugb2Y6IHtlfSIpCgogICAgIyBDcmVhdGUgbW9kZWxzIGZyb20gY2xhc3MgbmFtZSAvIGpzb24gZmlsZSAvIGpzb24gcGFyYW1zCiAgICBhbGxfc2tsZWFybl9lc3RpbWF0b3JzID0gZGljdChhbGxfZXN0aW1hdG9ycygpKSBpZiBsZW4obW9kZWxfZmlsdGVycykgPiAwIGVsc2Uge30KICAgIHNlbGVjdGVkX21vZGVscyA9IHt9CiAgICBmb3IgbW9kZWxfbmFtZSwgbW9kZWwgaW4gbW9kZWxfZmlsdGVycy5pdGVtcygpOgogICAgICAgIGlmICIuanNvbiIgaW4gbW9kZWw6CiAgICAgICAgICAgIGN1cnJlbnRfbW9kZWwgPSBqc29uLmxvYWQob3Blbihtb2RlbCkpCiAgICAgICAgICAgIGNsYXNzaWZpZXJfY2xhc3MgPSBjcmVhdGVfY2xhc3MoY3VycmVudF9tb2RlbFsiTUVUQSJdWyJjbGFzcyJdKQogICAgICAgICAgICBzZWxlY3RlZF9tb2RlbHNbbW9kZWxfbmFtZV0gPSBjbGFzc2lmaWVyX2NsYXNzKCoqY3VycmVudF9tb2RlbFsiQ0xBU1MiXSkKICAgICAgICBlbGlmIG1vZGVsIGluIGFsbF9za2xlYXJuX2VzdGltYXRvcnM6CiAgICAgICAgICAgIHNlbGVjdGVkX21vZGVsc1ttb2RlbF9uYW1lXSA9IGFsbF9za2xlYXJuX2VzdGltYXRvcnNbbW9kZWxfbmFtZV0oKQoKICAgICAgICBlbHNlOgogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICBjdXJyZW50X21vZGVsID0ganNvbi5sb2Fkcyhtb2RlbCkKICAgICAgICAgICAgICAgIGNsYXNzaWZpZXJfY2xhc3MgPSBjcmVhdGVfY2xhc3MoY3VycmVudF9tb2RlbFsiTUVUQSJdWyJjbGFzcyJdKQogICAgICAgICAgICAgICAgc2VsZWN0ZWRfbW9kZWxzW21vZGVsX25hbWVdID0gY2xhc3NpZmllcl9jbGFzcygqKmN1cnJlbnRfbW9kZWxbIkNMQVNTIl0pCiAgICAgICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJ1bmFibGUgdG8gbG9hZCB7bW9kZWx9IGJlY2F1c2Ugb2Y6IHtlfSIpCgogICAgIyBSdW4gbW9kZWwgZmlsdGVycwogICAgbW9kZWxzX2RmID0gcGQuRGF0YUZyYW1lKGluZGV4PVguY29sdW1ucykKICAgIGZvciBtb2RlbF9uYW1lLCBtb2RlbCBpbiBzZWxlY3RlZF9tb2RlbHMuaXRlbXMoKToKICAgICAgICBpZiBtb2RlbF9uYW1lID09ICJMb2dpc3RpY1JlZ3Jlc3Npb24iOgogICAgICAgICAgICBtb2RlbC5zZXRfcGFyYW1zKHNvbHZlcj0ibGlibGluZWFyIikKCiAgICAgICAgIyBUcmFpbiBtb2RlbCBhbmQgZ2V0IGZlYXR1cmUgaW1wb3J0YW5jZQogICAgICAgIHNlbGVjdF9mcm9tX21vZGVsID0gU2VsZWN0RnJvbU1vZGVsKG1vZGVsKS5maXQoWCwgeSkKICAgICAgICBmZWF0dXJlX2lkeCA9IHNlbGVjdF9mcm9tX21vZGVsLmdldF9zdXBwb3J0KCkKICAgICAgICBmZWF0dXJlX25hbWVzID0gWC5jb2x1bW5zW2ZlYXR1cmVfaWR4XQogICAgICAgIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZ1ttb2RlbF9uYW1lXSA9IGZlYXR1cmVfbmFtZXMudG9saXN0KCkKCiAgICAgICAgIyBDb2xsZWN0IG1vZGVsIGZlYXR1cmUgaW1wb3J0YW5jZQogICAgICAgIGlmIGhhc2F0dHIoc2VsZWN0X2Zyb21fbW9kZWwuZXN0aW1hdG9yXywgImNvZWZfIik6CiAgICAgICAgICAgIHN0YXRfZGYgPSBzZWxlY3RfZnJvbV9tb2RlbC5lc3RpbWF0b3JfLmNvZWZfCiAgICAgICAgZWxpZiBoYXNhdHRyKHNlbGVjdF9mcm9tX21vZGVsLmVzdGltYXRvcl8sICJmZWF0dXJlX2ltcG9ydGFuY2VzXyIpOgogICAgICAgICAgICBzdGF0X2RmID0gc2VsZWN0X2Zyb21fbW9kZWwuZXN0aW1hdG9yXy5mZWF0dXJlX2ltcG9ydGFuY2VzXwoKICAgICAgICBzdGF0X2RmID0gcGQuRGF0YUZyYW1lKGluZGV4PVguY29sdW1ucywgY29sdW1ucz1bbW9kZWxfbmFtZV0sIGRhdGE9c3RhdF9kZlswXSkKICAgICAgICBtb2RlbHNfZGYgPSBtb2RlbHNfZGYuam9pbihzdGF0X2RmKQoKICAgICAgICBwbG90X3N0YXQoY29udGV4dCwgbW9kZWxfbmFtZSwgc3RhdF9kZikKCiAgICAjIENyZWF0ZSBmZWF0dXJlX3Njb3JlcyBERiB3aXRoIHN0YXQgJiBtb2RlbCBmaWx0ZXJzIHNjb3JlcwogICAgcmVzdWx0X21hdHJpeF9kZiA9IHBkLmNvbmNhdChbc3RhdHNfZGYsIG1vZGVsc19kZl0sIGF4aXM9MSwgc29ydD1GYWxzZSkKICAgIGNvbnRleHQubG9nX2RhdGFzZXQoCiAgICAgICAga2V5PSJmZWF0dXJlX3Njb3JlcyIsCiAgICAgICAgZGY9cmVzdWx0X21hdHJpeF9kZiwKICAgICAgICBsb2NhbF9wYXRoPSJmZWF0dXJlX3Njb3Jlcy5wYXJxdWV0IiwKICAgICAgICBmb3JtYXQ9InBhcnF1ZXQiLAogICAgKQogICAgaWYgbWF4X3NjYWxlZF9zY29yZXM6CiAgICAgICAgbm9ybWFsaXplZF9kZiA9IHJlc3VsdF9tYXRyaXhfZGYucmVwbGFjZShbbnAuaW5mLCAtbnAuaW5mXSwgbnAubmFuKS52YWx1ZXMKICAgICAgICBtaW5fbWF4X3NjYWxlciA9IE1pbk1heFNjYWxlcigpCiAgICAgICAgbm9ybWFsaXplZF9kZiA9IG1pbl9tYXhfc2NhbGVyLmZpdF90cmFuc2Zvcm0obm9ybWFsaXplZF9kZikKICAgICAgICBub3JtYWxpemVkX2RmID0gcGQuRGF0YUZyYW1lKAogICAgICAgICAgICBkYXRhPW5vcm1hbGl6ZWRfZGYsCiAgICAgICAgICAgIGNvbHVtbnM9cmVzdWx0X21hdHJpeF9kZi5jb2x1bW5zLAogICAgICAgICAgICBpbmRleD1yZXN1bHRfbWF0cml4X2RmLmluZGV4LAogICAgICAgICkKICAgICAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgICAgICBrZXk9Im1heF9zY2FsZWRfc2NvcmVzX2ZlYXR1cmVfc2NvcmVzIiwKICAgICAgICAgICAgZGY9bm9ybWFsaXplZF9kZiwKICAgICAgICAgICAgbG9jYWxfcGF0aD0ibWF4X3NjYWxlZF9zY29yZXNfZmVhdHVyZV9zY29yZXMucGFycXVldCIsCiAgICAgICAgICAgIGZvcm1hdD0icGFycXVldCIsCiAgICAgICAgKQoKICAgICMgQ3JlYXRlIGZlYXR1cmUgY291bnQgRGF0YUZyYW1lCiAgICBmb3IgdGVzdF9uYW1lIGluIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZzoKICAgICAgICByZXN1bHRfbWF0cml4X2RmW3Rlc3RfbmFtZV0gPSBbCiAgICAgICAgICAgIDEgaWYgeCBpbiBzZWxlY3RlZF9mZWF0dXJlc19hZ2dbdGVzdF9uYW1lXSBlbHNlIDAgZm9yIHggaW4gWC5jb2x1bW5zCiAgICAgICAgXQogICAgcmVzdWx0X21hdHJpeF9kZi5sb2NbOiwgIm51bV92b3RlcyJdID0gcmVzdWx0X21hdHJpeF9kZi5zdW0oYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldCgKICAgICAgICBrZXk9InNlbGVjdGVkX2ZlYXR1cmVzX2NvdW50IiwKICAgICAgICBkZj1yZXN1bHRfbWF0cml4X2RmLAogICAgICAgIGxvY2FsX3BhdGg9InNlbGVjdGVkX2ZlYXR1cmVzX2NvdW50LnBhcnF1ZXQiLAogICAgICAgIGZvcm1hdD0icGFycXVldCIsCiAgICApCgogICAgIyBIb3cgbWFueSB2b3RlcyBhcmUgbmVlZGVkIGZvciBhIGZlYXR1cmUgdG8gYmUgc2VsZWN0ZWQ/CiAgICBpZiBpc2luc3RhbmNlKG1pbl92b3RlcywgaW50KToKICAgICAgICB2b3Rlc19uZWVkZWQgPSBtaW5fdm90ZXMKICAgIGVsc2U6CiAgICAgICAgbnVtX2ZpbHRlcnMgPSBsZW4oc3RhdF9maWx0ZXJzKSArIGxlbihtb2RlbF9maWx0ZXJzKQogICAgICAgIHZvdGVzX25lZWRlZCA9IGludChucC5mbG9vcihudW1fZmlsdGVycyAqIG1heChtaW4obWluX3ZvdGVzLCAxKSwgMCkpKQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInZvdGVzIG5lZWRlZCB0byBiZSBzZWxlY3RlZDoge3ZvdGVzX25lZWRlZH0iKQoKICAgICMgQ3JlYXRlIGZpbmFsIGZlYXR1cmUgZGF0YWZyYW1lCiAgICBzZWxlY3RlZF9mZWF0dXJlcyA9IHJlc3VsdF9tYXRyaXhfZGZbCiAgICAgICAgcmVzdWx0X21hdHJpeF9kZi5udW1fdm90ZXMgPj0gdm90ZXNfbmVlZGVkCiAgICBdLmluZGV4LnRvbGlzdCgpCiAgICBnb29kX2ZlYXR1cmVfZGYgPSBkZi5sb2NbOiwgc2VsZWN0ZWRfZmVhdHVyZXNdCiAgICBmaW5hbF9kZiA9IHBkLmNvbmNhdChbZ29vZF9mZWF0dXJlX2RmLCB5XSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldCgKICAgICAgICBrZXk9InNlbGVjdGVkX2ZlYXR1cmVzIiwKICAgICAgICBkZj1maW5hbF9kZiwKICAgICAgICBsb2NhbF9wYXRoPSJzZWxlY3RlZF9mZWF0dXJlcy5wYXJxdWV0IiwKICAgICAgICBmb3JtYXQ9InBhcnF1ZXQiLAogICAgKQoKICAgICMgQ3JlYXRpbmcgYSBuZXcgZmVhdHVyZSB2ZWN0b3IgY29udGFpbmluZyBvbmx5IHRoZSBpZGVudGlmaWVkIHRvcCBmZWF0dXJlcwogICAgaWYgaXNfZmVhdHVyZV92ZWN0b3IgYW5kIGRmX2FydGlmYWN0Lm1ldGEuc3BlYy5mZWF0dXJlcyBhbmQgb3V0cHV0X3ZlY3Rvcl9uYW1lOgogICAgICAgICMgU2VsZWN0aW5nIHRoZSB0b3AgSyBmZWF0dXJlcyBmcm9tIG91ciB0b3AgZmVhdHVyZSBkYXRhZnJhbWUKICAgICAgICBzZWxlY3RlZF9mZWF0dXJlcyA9IHJlc3VsdF9tYXRyaXhfZGYuaGVhZChrKS5pbmRleAoKICAgICAgICAjIE1hdGNoIHRoZSBzZWxlY3RlZCBmZWF0dXJlIG5hbWVzIHRvIHRoZSBGUyBGZWF0dXJlIGFubm90YXRpb25zCiAgICAgICAgbWF0Y2hlZF9zZWxlY3Rpb25zID0gWwogICAgICAgICAgICBmZWF0dXJlCiAgICAgICAgICAgIGZvciBmZWF0dXJlIGluIGxpc3QoZGZfYXJ0aWZhY3QubWV0YS5zcGVjLmZlYXR1cmVzKQogICAgICAgICAgICBmb3Igc2VsZWN0ZWQgaW4gbGlzdChzZWxlY3RlZF9mZWF0dXJlcykKICAgICAgICAgICAgaWYgZmVhdHVyZS5lbmRzd2l0aChzZWxlY3RlZCkKICAgICAgICBdCgogICAgICAgICMgRGVmaW5pbmcgb3VyIG5ldyBmZWF0dXJlIHZlY3RvcgogICAgICAgIHRvcF9mZWF0dXJlc19mdiA9IGZzLkZlYXR1cmVWZWN0b3IoCiAgICAgICAgICAgIG91dHB1dF92ZWN0b3JfbmFtZSwKICAgICAgICAgICAgbWF0Y2hlZF9zZWxlY3Rpb25zLAogICAgICAgICAgICBsYWJlbF9mZWF0dXJlPSJsYWJlbHMubGFiZWwiLAogICAgICAgICAgICBkZXNjcmlwdGlvbj0iZmVhdHVyZSB2ZWN0b3IgY29tcG9zZWQgc3RyaWN0bHkgb2Ygb3VyIHRvcCBmZWF0dXJlcyIsCiAgICAgICAgKQoKICAgICAgICAjIFNhdmluZwogICAgICAgIHRvcF9mZWF0dXJlc19mdi5zYXZlKCkKICAgICAgICB0b3BfZmVhdHVyZXNfZnYuZ2V0X29mZmxpbmVfZmVhdHVyZXModGFyZ2V0PVBhcnF1ZXRUYXJnZXQoKSkKCiAgICAgICAgIyBMb2dnaW5nIG91ciBuZXcgZmVhdHVyZSB2ZWN0b3IgVVJJCiAgICAgICAgY29udGV4dC5sb2dfcmVzdWx0KCJ0b3BfZmVhdHVyZXNfdmVjdG9yIiwgdG9wX2ZlYXR1cmVzX2Z2LnVyaSkK + code_origin: '' + filename: feature_selection.py entry_points: show_values_on_bars: parameters: @@ -10,20 +23,20 @@ spec: - name: space default: 0.4 name: show_values_on_bars - lineno: 43 + doc: '' has_kwargs: false has_varargs: false - doc: '' + lineno: 47 plot_stat: parameters: - name: context - name: stat_name - name: stat_df name: plot_stat - lineno: 65 + doc: '' has_kwargs: false has_varargs: false - doc: '' + lineno: 69 feature_selection: parameters: - name: context @@ -72,9 +85,6 @@ spec: doc: skips datatypes that are neither float nor int within the feature vector. default: false name: feature_selection - lineno: 80 - has_kwargs: false - has_varargs: false doc: 'Applies selected feature selection statistical functions or models on our ''df_artifact''. @@ -82,18 +92,9 @@ spec: Each statistical function or model will vote for it''s best K selected features. If a feature has >= ''min_votes'' votes, it will be selected.' - image: mlrun/mlrun - build: - origin_filename: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IGpzb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi5mZWF0dXJlX3N0b3JlIGFzIGZzCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgbnVtcHkgYXMgbnAKaW1wb3J0IHBhbmRhcyBhcyBwZAppbXBvcnQgcGxvdGx5LmV4cHJlc3MgYXMgcHgKZnJvbSBtbHJ1bi5hcnRpZmFjdHMgaW1wb3J0IFBsb3RseUFydGlmYWN0CmZyb20gbWxydW4uZGF0YXN0b3JlLnRhcmdldHMgaW1wb3J0IFBhcnF1ZXRUYXJnZXQKIyBNTFJ1biB1dGlscwpmcm9tIG1scnVuLnV0aWxzLmhlbHBlcnMgaW1wb3J0IGNyZWF0ZV9jbGFzcwojIEZlYXR1cmUgc2VsZWN0aW9uIHN0cmF0ZWdpZXMKZnJvbSBza2xlYXJuLmZlYXR1cmVfc2VsZWN0aW9uIGltcG9ydCBTZWxlY3RGcm9tTW9kZWwsIFNlbGVjdEtCZXN0CiMgU2NhbGUgZmVhdHVyZSBzY29yZXNnaXQgc3QKZnJvbSBza2xlYXJuLnByZXByb2Nlc3NpbmcgaW1wb3J0IE1pbk1heFNjYWxlcgojIFNLTGVhcm4gZXN0aW1hdG9ycyBsaXN0CmZyb20gc2tsZWFybi51dGlscyBpbXBvcnQgYWxsX2VzdGltYXRvcnMKCkRFRkFVTFRfU1RBVF9GSUxURVJTID0gWyJmX2NsYXNzaWYiLCAibXV0dWFsX2luZm9fY2xhc3NpZiIsICJjaGkyIiwgImZfcmVncmVzc2lvbiJdCkRFRkFVTFRfTU9ERUxfRklMVEVSUyA9IHsKICAgICJMaW5lYXJTVkMiOiAiTGluZWFyU1ZDIiwKICAgICJMb2dpc3RpY1JlZ3Jlc3Npb24iOiAiTG9naXN0aWNSZWdyZXNzaW9uIiwKICAgICJFeHRyYVRyZWVzQ2xhc3NpZmllciI6ICJFeHRyYVRyZWVzQ2xhc3NpZmllciIsCn0KCgpkZWYgc2hvd192YWx1ZXNfb25fYmFycyhheHMsIGhfdj0idiIsIHNwYWNlPTAuNCk6CiAgICBkZWYgX3Nob3dfb25fc2luZ2xlX3Bsb3QoYXhfKToKICAgICAgICBpZiBoX3YgPT0gInYiOgogICAgICAgICAgICBmb3IgcCBpbiBheF8ucGF0Y2hlczoKICAgICAgICAgICAgICAgIF94ID0gcC5nZXRfeCgpICsgcC5nZXRfd2lkdGgoKSAvIDIKICAgICAgICAgICAgICAgIF95ID0gcC5nZXRfeSgpICsgcC5nZXRfaGVpZ2h0KCkKICAgICAgICAgICAgICAgIHZhbHVlID0gaW50KHAuZ2V0X2hlaWdodCgpKQogICAgICAgICAgICAgICAgYXhfLnRleHQoX3gsIF95LCB2YWx1ZSwgaGE9ImNlbnRlciIpCiAgICAgICAgZWxpZiBoX3YgPT0gImgiOgogICAgICAgICAgICBmb3IgcCBpbiBheF8ucGF0Y2hlczoKICAgICAgICAgICAgICAgIF94ID0gcC5nZXRfeCgpICsgcC5nZXRfd2lkdGgoKSArIGZsb2F0KHNwYWNlKQogICAgICAgICAgICAgICAgX3kgPSBwLmdldF95KCkgKyBwLmdldF9oZWlnaHQoKQogICAgICAgICAgICAgICAgdmFsdWUgPSBpbnQocC5nZXRfd2lkdGgoKSkKICAgICAgICAgICAgICAgIGF4Xy50ZXh0KF94LCBfeSwgdmFsdWUsIGhhPSJsZWZ0IikKCiAgICBpZiBpc2luc3RhbmNlKGF4cywgbnAubmRhcnJheSk6CiAgICAgICAgZm9yIGlkeCwgYXggaW4gbnAubmRlbnVtZXJhdGUoYXhzKToKICAgICAgICAgICAgX3Nob3dfb25fc2luZ2xlX3Bsb3QoYXgpCiAgICBlbHNlOgogICAgICAgIF9zaG93X29uX3NpbmdsZV9wbG90KGF4cykKCgpkZWYgcGxvdF9zdGF0KGNvbnRleHQsIHN0YXRfbmFtZSwgc3RhdF9kZik6CiAgICBzb3J0ZWRfZGYgPSBzdGF0X2RmLnNvcnRfdmFsdWVzKHN0YXRfbmFtZSkKICAgIGZpZyA9IHB4LmJhcigKICAgICAgICBkYXRhX2ZyYW1lPXNvcnRlZF9kZiwKICAgICAgICB4PXN0YXRfbmFtZSwKICAgICAgICB5PXNvcnRlZF9kZi5pbmRleCwKICAgICAgICB0aXRsZT1mIntzdGF0X25hbWV9IGZlYXR1cmUgc2NvcmVzIiwKICAgICAgICBjb2xvcj1zdGF0X25hbWUsCiAgICApCiAgICBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBpdGVtPVBsb3RseUFydGlmYWN0KGtleT1zdGF0X25hbWUsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7c3RhdF9uYW1lfS5odG1sIiwKICAgICkKCgpkZWYgZmVhdHVyZV9zZWxlY3Rpb24oCiAgICBjb250ZXh0LAogICAgZGZfYXJ0aWZhY3QsCiAgICBrOiBpbnQgPSA1LAogICAgbWluX3ZvdGVzOiBmbG9hdCA9IDAuNSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gTm9uZSwKICAgIHN0YXRfZmlsdGVyczogbGlzdCA9IE5vbmUsCiAgICBtb2RlbF9maWx0ZXJzOiBkaWN0ID0gTm9uZSwKICAgIG1heF9zY2FsZWRfc2NvcmVzOiBib29sID0gVHJ1ZSwKICAgIHNhbXBsZV9yYXRpbzogZmxvYXQgPSBOb25lLAogICAgb3V0cHV0X3ZlY3Rvcl9uYW1lOiBmbG9hdCA9IE5vbmUsCiAgICBpZ25vcmVfdHlwZV9lcnJvcnM6IGJvb2wgPSBGYWxzZSwKKToKICAgICIiIgogICAgQXBwbGllcyBzZWxlY3RlZCBmZWF0dXJlIHNlbGVjdGlvbiBzdGF0aXN0aWNhbCBmdW5jdGlvbnMgb3IgbW9kZWxzIG9uIG91ciAnZGZfYXJ0aWZhY3QnLgoKICAgIEVhY2ggc3RhdGlzdGljYWwgZnVuY3Rpb24gb3IgbW9kZWwgd2lsbCB2b3RlIGZvciBpdCdzIGJlc3QgSyBzZWxlY3RlZCBmZWF0dXJlcy4KICAgIElmIGEgZmVhdHVyZSBoYXMgPj0gJ21pbl92b3Rlcycgdm90ZXMsIGl0IHdpbGwgYmUgc2VsZWN0ZWQuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgIHRoZSBmdW5jdGlvbiBjb250ZXh0LgogICAgOnBhcmFtIGRmX2FydGlmYWN0OiAgICAgICAgIGRhdGFmcmFtZSB0byBwYXNzIGFzIGlucHV0LgogICAgOnBhcmFtIGs6ICAgICAgICAgICAgICAgICAgIG51bWJlciBvZiB0b3AgZmVhdHVyZXMgdG8gc2VsZWN0IGZyb20gZWFjaCBzdGF0aXN0aWNhbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZ1bmN0aW9uIG9yIG1vZGVsLgogICAgOnBhcmFtIG1pbl92b3RlczogICAgICAgICAgIG1pbmltYWwgbnVtYmVyIG9mIHZvdGVzIChmcm9tIGEgbW9kZWwgb3IgYnkgc3RhdGlzdGljYWwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmdW5jdGlvbikgbmVlZGVkIGZvciBhIGZlYXR1cmUgdG8gYmUgc2VsZWN0ZWQuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2FuIGJlIHNwZWNpZmllZCBieSBwZXJjZW50YWdlIG9mIHZvdGVzIG9yIGFic29sdXRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbnVtYmVyIG9mIHZvdGVzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICAgIGdyb3VuZC10cnV0aCAoeSkgbGFiZWxzLgogICAgOnBhcmFtIHN0YXRfZmlsdGVyczogICAgICAgIHN0YXRpc3RpY2FsIGZ1bmN0aW9ucyB0byBhcHBseSB0byB0aGUgZmVhdHVyZXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoZnJvbSBza2xlYXJuLmZlYXR1cmVfc2VsZWN0aW9uKS4KICAgIDpwYXJhbSBtb2RlbF9maWx0ZXJzOiAgICAgICBtb2RlbHMgdG8gdXNlIGZvciBmZWF0dXJlIGV2YWx1YXRpb24sIGNhbiBiZSBzcGVjaWZpZWQgYnkKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlbCBuYW1lIChleC4gTGluZWFyU1ZDKSwgZm9ybWFsaXplZCBqc29uIChjb250YWlucyAnQ0xBU1MnLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICdGSVQnLCAnTUVUQScpIG9yIGEgcGF0aCB0byBzdWNoIGpzb24gZmlsZS4KICAgIDpwYXJhbSBtYXhfc2NhbGVkX3Njb3JlczogICBwcm9kdWNlIGZlYXR1cmUgc2NvcmVzIHRhYmxlIHNjYWxlZCB3aXRoIG1heF9zY2FsZXIuCiAgICA6cGFyYW0gc2FtcGxlX3JhdGlvOiAgICAgICAgcGVyY2VudGFnZSBvZiB0aGUgZGF0YXNldCB0aGUgdXNlciB3aXNoZXMgdG8gY29tcHV0ZSB0aGUgZmVhdHVyZSBzZWxlY3Rpb24gcHJvY2VzcyBvbi4KICAgIDpwYXJhbSBvdXRwdXRfdmVjdG9yX25hbWU6ICBjcmVhdGVzIGEgbmV3IGZlYXR1cmUgdmVjdG9yIGNvbnRhaW5pbmcgb25seSB0aGUgaWRlbnRpZmllcyBmZWF0dXJlcy4KICAgIDpwYXJhbSBpZ25vcmVfdHlwZV9lcnJvcnM6ICBza2lwcyBkYXRhdHlwZXMgdGhhdCBhcmUgbmVpdGhlciBmbG9hdCBub3IgaW50IHdpdGhpbiB0aGUgZmVhdHVyZSB2ZWN0b3IuCiAgICAiIiIKICAgIHN0YXRfZmlsdGVycyA9IHN0YXRfZmlsdGVycyBvciBERUZBVUxUX1NUQVRfRklMVEVSUwogICAgbW9kZWxfZmlsdGVycyA9IG1vZGVsX2ZpbHRlcnMgb3IgREVGQVVMVF9NT0RFTF9GSUxURVJTCiAgICAjIENoZWNrIGlmIGRmLm1ldGEgaXMgdmFsaWQsIGlmIGl0IGlzLCBsb29rIGZvciBhIGZlYXR1cmUgdmVjdG9yCiAgICBzdG9yZV91cmlfcHJlZml4LCBfID0gbWxydW4uZGF0YXN0b3JlLnBhcnNlX3N0b3JlX3VyaShkZl9hcnRpZmFjdC5hcnRpZmFjdF91cmwpCiAgICBpc19mZWF0dXJlX3ZlY3RvciA9IG1scnVuLnV0aWxzLlN0b3JlUHJlZml4LkZlYXR1cmVWZWN0b3IgPT0gc3RvcmVfdXJpX3ByZWZpeAoKICAgICMgTG9vayBpbnNpZGUgbWV0YS5zcGVjLmxhYmVsX2ZlYXR1cmUgdG8gaWRlbnRpZnkgdGhlIGxhYmVsX2NvbHVtbiBpZiB0aGUgdXNlciBkaWQgbm90IHNwZWNpZnkgaXQKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBOb25lOgogICAgICAgIGlmIGlzX2ZlYXR1cmVfdmVjdG9yOgogICAgICAgICAgICBsYWJlbF9jb2x1bW4gPSBkZl9hcnRpZmFjdC5tZXRhLnNwZWMubGFiZWxfZmVhdHVyZS5zcGxpdCgiLiIpWzFdCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigiTm8gbGFiZWxfY29sdW1uIHdhcyBnaXZlbiwgcGxlYXNlIGFkZCBhIGxhYmVsX2NvbHVtbi4iKQoKICAgICMgVXNlIHRoZSBmZWF0dXJlIHZlY3RvciBhcyBkYXRhZnJhbWUKICAgIGRmID0gZGZfYXJ0aWZhY3QuYXNfZGYoKQoKICAgICMgRW5zdXJlIGsgaXMgbm90IGJpZ2dlciB0aGFuIHRoZSB0b3RhbCBudW1iZXIgb2YgZmVhdHVyZXMKICAgIGlmIGsgPiBkZi5zaGFwZVsxXToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIksgY2Fubm90IGJlIGJpZ2dlciB0aGFuIHRoZSB0b3RhbCBudW1iZXIgb2YgZmVhdHVyZXMgKHtkZi5zaGFwZVsxXX0pLiBQbGVhc2UgY2hvb3NlIGEgc21hbGxlciBLLiIKICAgICAgICApCiAgICBlbGlmIGsgPCAxOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoIksgY2Fubm90IGJlIHNtYWxsZXIgdGhhbiAxLiBQbGVhc2UgY2hvb3NlIGEgYmlnZ2VyIEsuIikKCiAgICAjIENyZWF0ZSBhIHNhbXBsZSBkYXRhZnJhbWUgb2YgdGhlIG9yaWdpbmFsIGZlYXR1cmUgdmVjdG9yCiAgICBpZiBzYW1wbGVfcmF0aW86CiAgICAgICAgZGYgPSAoCiAgICAgICAgICAgIGRmLmdyb3VwYnkobGFiZWxfY29sdW1uKQogICAgICAgICAgICAuYXBwbHkobGFtYmRhIHg6IHguc2FtcGxlKGZyYWM9c2FtcGxlX3JhdGlvKSkKICAgICAgICAgICAgLnJlc2V0X2luZGV4KGRyb3A9VHJ1ZSkKICAgICAgICApCiAgICAgICAgZGYgPSBkZi5kcm9wbmEoKQoKICAgICMgU2V0IGZlYXR1cmUgdmVjdG9yIGFuZCBsYWJlbHMKICAgIHkgPSBkZi5wb3AobGFiZWxfY29sdW1uKQogICAgWCA9IGRmCgogICAgaWYgbnAub2JqZWN0XyBpbiBsaXN0KFguZHR5cGVzKSBhbmQgaWdub3JlX3R5cGVfZXJyb3JzIGlzIEZhbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgIGYie2RmLnNlbGVjdF9kdHlwZXMoaW5jbHVkZT1bJ29iamVjdCddKS5jb2x1bW5zLnRvbGlzdCgpfSBhcmUgbmVpdGhlciBmbG9hdCBvciBpbnQuIgogICAgICAgICkKCiAgICAjIENyZWF0ZSBzZWxlY3RlZCBzdGF0aXN0aWNhbCBlc3RpbWF0b3JzCiAgICBzdGF0X2Z1bmN0aW9uc19saXN0ID0gewogICAgICAgIHN0YXRfbmFtZTogU2VsZWN0S0Jlc3QoCiAgICAgICAgICAgIHNjb3JlX2Z1bmM9Y3JlYXRlX2NsYXNzKGYic2tsZWFybi5mZWF0dXJlX3NlbGVjdGlvbi57c3RhdF9uYW1lfSIpLCBrPWsKICAgICAgICApCiAgICAgICAgZm9yIHN0YXRfbmFtZSBpbiBzdGF0X2ZpbHRlcnMKICAgIH0KICAgIHJlcXVpcmVzX2FicyA9IFsiY2hpMiJdCgogICAgIyBSdW4gc3RhdGlzdGljIGZpbHRlcnMKICAgIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZyA9IHt9CiAgICBzdGF0c19kZiA9IHBkLkRhdGFGcmFtZShpbmRleD1YLmNvbHVtbnMpLmRyb3BuYSgpCgogICAgZm9yIHN0YXRfbmFtZSwgc3RhdF9mdW5jIGluIHN0YXRfZnVuY3Rpb25zX2xpc3QuaXRlbXMoKToKICAgICAgICB0cnk6CiAgICAgICAgICAgIHBhcmFtcyA9IChYLCB5KSBpZiBzdGF0X25hbWUgaW4gcmVxdWlyZXNfYWJzIGVsc2UgKGFicyhYKSwgeSkKICAgICAgICAgICAgc3RhdCA9IHN0YXRfZnVuYy5maXQoKnBhcmFtcykKCiAgICAgICAgICAgICMgQ29sbGVjdCBzdGF0IGZ1bmN0aW9uIHJlc3VsdHMKICAgICAgICAgICAgc3RhdF9kZiA9IHBkLkRhdGFGcmFtZSgKICAgICAgICAgICAgICAgIGluZGV4PVguY29sdW1ucywgY29sdW1ucz1bc3RhdF9uYW1lXSwgZGF0YT1zdGF0LnNjb3Jlc18KICAgICAgICAgICAgKQogICAgICAgICAgICBwbG90X3N0YXQoY29udGV4dCwgc3RhdF9uYW1lLCBzdGF0X2RmKQogICAgICAgICAgICBzdGF0c19kZiA9IHN0YXRzX2RmLmpvaW4oc3RhdF9kZikKCiAgICAgICAgICAgICMgU2VsZWN0IEsgQmVzdCBmZWF0dXJlcwogICAgICAgICAgICBzZWxlY3RlZF9mZWF0dXJlcyA9IFguY29sdW1uc1tzdGF0X2Z1bmMuZ2V0X3N1cHBvcnQoKV0KICAgICAgICAgICAgc2VsZWN0ZWRfZmVhdHVyZXNfYWdnW3N0YXRfbmFtZV0gPSBzZWxlY3RlZF9mZWF0dXJlcwoKICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJDb3VsZG4ndCBjYWxjdWxhdGUge3N0YXRfbmFtZX0gYmVjYXVzZSBvZjoge2V9IikKCiAgICAjIENyZWF0ZSBtb2RlbHMgZnJvbSBjbGFzcyBuYW1lIC8ganNvbiBmaWxlIC8ganNvbiBwYXJhbXMKICAgIGFsbF9za2xlYXJuX2VzdGltYXRvcnMgPSBkaWN0KGFsbF9lc3RpbWF0b3JzKCkpIGlmIGxlbihtb2RlbF9maWx0ZXJzKSA+IDAgZWxzZSB7fQogICAgc2VsZWN0ZWRfbW9kZWxzID0ge30KICAgIGZvciBtb2RlbF9uYW1lLCBtb2RlbCBpbiBtb2RlbF9maWx0ZXJzLml0ZW1zKCk6CiAgICAgICAgaWYgIi5qc29uIiBpbiBtb2RlbDoKICAgICAgICAgICAgY3VycmVudF9tb2RlbCA9IGpzb24ubG9hZChvcGVuKG1vZGVsLCAiciIpKQogICAgICAgICAgICBjbGFzc2lmaWVyX2NsYXNzID0gY3JlYXRlX2NsYXNzKGN1cnJlbnRfbW9kZWxbIk1FVEEiXVsiY2xhc3MiXSkKICAgICAgICAgICAgc2VsZWN0ZWRfbW9kZWxzW21vZGVsX25hbWVdID0gY2xhc3NpZmllcl9jbGFzcygqKmN1cnJlbnRfbW9kZWxbIkNMQVNTIl0pCiAgICAgICAgZWxpZiBtb2RlbCBpbiBhbGxfc2tsZWFybl9lc3RpbWF0b3JzOgogICAgICAgICAgICBzZWxlY3RlZF9tb2RlbHNbbW9kZWxfbmFtZV0gPSBhbGxfc2tsZWFybl9lc3RpbWF0b3JzW21vZGVsX25hbWVdKCkKCiAgICAgICAgZWxzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgY3VycmVudF9tb2RlbCA9IGpzb24ubG9hZHMobW9kZWwpCiAgICAgICAgICAgICAgICBjbGFzc2lmaWVyX2NsYXNzID0gY3JlYXRlX2NsYXNzKGN1cnJlbnRfbW9kZWxbIk1FVEEiXVsiY2xhc3MiXSkKICAgICAgICAgICAgICAgIHNlbGVjdGVkX21vZGVsc1ttb2RlbF9uYW1lXSA9IGNsYXNzaWZpZXJfY2xhc3MoKipjdXJyZW50X21vZGVsWyJDTEFTUyJdKQogICAgICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidW5hYmxlIHRvIGxvYWQge21vZGVsfSBiZWNhdXNlIG9mOiB7ZX0iKQoKICAgICMgUnVuIG1vZGVsIGZpbHRlcnMKICAgIG1vZGVsc19kZiA9IHBkLkRhdGFGcmFtZShpbmRleD1YLmNvbHVtbnMpCiAgICBmb3IgbW9kZWxfbmFtZSwgbW9kZWwgaW4gc2VsZWN0ZWRfbW9kZWxzLml0ZW1zKCk6CgogICAgICAgIGlmIG1vZGVsX25hbWUgPT0gIkxvZ2lzdGljUmVncmVzc2lvbiI6CiAgICAgICAgICAgIG1vZGVsLnNldF9wYXJhbXMoc29sdmVyPSJsaWJsaW5lYXIiKQoKICAgICAgICAjIFRyYWluIG1vZGVsIGFuZCBnZXQgZmVhdHVyZSBpbXBvcnRhbmNlCiAgICAgICAgc2VsZWN0X2Zyb21fbW9kZWwgPSBTZWxlY3RGcm9tTW9kZWwobW9kZWwpLmZpdChYLCB5KQogICAgICAgIGZlYXR1cmVfaWR4ID0gc2VsZWN0X2Zyb21fbW9kZWwuZ2V0X3N1cHBvcnQoKQogICAgICAgIGZlYXR1cmVfbmFtZXMgPSBYLmNvbHVtbnNbZmVhdHVyZV9pZHhdCiAgICAgICAgc2VsZWN0ZWRfZmVhdHVyZXNfYWdnW21vZGVsX25hbWVdID0gZmVhdHVyZV9uYW1lcy50b2xpc3QoKQoKICAgICAgICAjIENvbGxlY3QgbW9kZWwgZmVhdHVyZSBpbXBvcnRhbmNlCiAgICAgICAgaWYgaGFzYXR0cihzZWxlY3RfZnJvbV9tb2RlbC5lc3RpbWF0b3JfLCAiY29lZl8iKToKICAgICAgICAgICAgc3RhdF9kZiA9IHNlbGVjdF9mcm9tX21vZGVsLmVzdGltYXRvcl8uY29lZl8KICAgICAgICBlbGlmIGhhc2F0dHIoc2VsZWN0X2Zyb21fbW9kZWwuZXN0aW1hdG9yXywgImZlYXR1cmVfaW1wb3J0YW5jZXNfIik6CiAgICAgICAgICAgIHN0YXRfZGYgPSBzZWxlY3RfZnJvbV9tb2RlbC5lc3RpbWF0b3JfLmZlYXR1cmVfaW1wb3J0YW5jZXNfCgogICAgICAgIHN0YXRfZGYgPSBwZC5EYXRhRnJhbWUoaW5kZXg9WC5jb2x1bW5zLCBjb2x1bW5zPVttb2RlbF9uYW1lXSwgZGF0YT1zdGF0X2RmWzBdKQogICAgICAgIG1vZGVsc19kZiA9IG1vZGVsc19kZi5qb2luKHN0YXRfZGYpCgogICAgICAgIHBsb3Rfc3RhdChjb250ZXh0LCBtb2RlbF9uYW1lLCBzdGF0X2RmKQoKICAgICMgQ3JlYXRlIGZlYXR1cmVfc2NvcmVzIERGIHdpdGggc3RhdCAmIG1vZGVsIGZpbHRlcnMgc2NvcmVzCiAgICByZXN1bHRfbWF0cml4X2RmID0gcGQuY29uY2F0KFtzdGF0c19kZiwgbW9kZWxzX2RmXSwgYXhpcz0xLCBzb3J0PUZhbHNlKQogICAgY29udGV4dC5sb2dfZGF0YXNldCgKICAgICAgICBrZXk9ImZlYXR1cmVfc2NvcmVzIiwKICAgICAgICBkZj1yZXN1bHRfbWF0cml4X2RmLAogICAgICAgIGxvY2FsX3BhdGg9ImZlYXR1cmVfc2NvcmVzLnBhcnF1ZXQiLAogICAgICAgIGZvcm1hdD0icGFycXVldCIsCiAgICApCiAgICBpZiBtYXhfc2NhbGVkX3Njb3JlczoKICAgICAgICBub3JtYWxpemVkX2RmID0gcmVzdWx0X21hdHJpeF9kZi5yZXBsYWNlKFtucC5pbmYsIC1ucC5pbmZdLCBucC5uYW4pLnZhbHVlcwogICAgICAgIG1pbl9tYXhfc2NhbGVyID0gTWluTWF4U2NhbGVyKCkKICAgICAgICBub3JtYWxpemVkX2RmID0gbWluX21heF9zY2FsZXIuZml0X3RyYW5zZm9ybShub3JtYWxpemVkX2RmKQogICAgICAgIG5vcm1hbGl6ZWRfZGYgPSBwZC5EYXRhRnJhbWUoCiAgICAgICAgICAgIGRhdGE9bm9ybWFsaXplZF9kZiwKICAgICAgICAgICAgY29sdW1ucz1yZXN1bHRfbWF0cml4X2RmLmNvbHVtbnMsCiAgICAgICAgICAgIGluZGV4PXJlc3VsdF9tYXRyaXhfZGYuaW5kZXgsCiAgICAgICAgKQogICAgICAgIGNvbnRleHQubG9nX2RhdGFzZXQoCiAgICAgICAgICAgIGtleT0ibWF4X3NjYWxlZF9zY29yZXNfZmVhdHVyZV9zY29yZXMiLAogICAgICAgICAgICBkZj1ub3JtYWxpemVkX2RmLAogICAgICAgICAgICBsb2NhbF9wYXRoPSJtYXhfc2NhbGVkX3Njb3Jlc19mZWF0dXJlX3Njb3Jlcy5wYXJxdWV0IiwKICAgICAgICAgICAgZm9ybWF0PSJwYXJxdWV0IiwKICAgICAgICApCgogICAgIyBDcmVhdGUgZmVhdHVyZSBjb3VudCBEYXRhRnJhbWUKICAgIGZvciB0ZXN0X25hbWUgaW4gc2VsZWN0ZWRfZmVhdHVyZXNfYWdnOgogICAgICAgIHJlc3VsdF9tYXRyaXhfZGZbdGVzdF9uYW1lXSA9IFsKICAgICAgICAgICAgMSBpZiB4IGluIHNlbGVjdGVkX2ZlYXR1cmVzX2FnZ1t0ZXN0X25hbWVdIGVsc2UgMCBmb3IgeCBpbiBYLmNvbHVtbnMKICAgICAgICBdCiAgICByZXN1bHRfbWF0cml4X2RmLmxvY1s6LCAibnVtX3ZvdGVzIl0gPSByZXN1bHRfbWF0cml4X2RmLnN1bShheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgIGtleT0ic2VsZWN0ZWRfZmVhdHVyZXNfY291bnQiLAogICAgICAgIGRmPXJlc3VsdF9tYXRyaXhfZGYsCiAgICAgICAgbG9jYWxfcGF0aD0ic2VsZWN0ZWRfZmVhdHVyZXNfY291bnQucGFycXVldCIsCiAgICAgICAgZm9ybWF0PSJwYXJxdWV0IiwKICAgICkKCiAgICAjIEhvdyBtYW55IHZvdGVzIGFyZSBuZWVkZWQgZm9yIGEgZmVhdHVyZSB0byBiZSBzZWxlY3RlZD8KICAgIGlmIGlzaW5zdGFuY2UobWluX3ZvdGVzLCBpbnQpOgogICAgICAgIHZvdGVzX25lZWRlZCA9IG1pbl92b3RlcwogICAgZWxzZToKICAgICAgICBudW1fZmlsdGVycyA9IGxlbihzdGF0X2ZpbHRlcnMpICsgbGVuKG1vZGVsX2ZpbHRlcnMpCiAgICAgICAgdm90ZXNfbmVlZGVkID0gaW50KG5wLmZsb29yKG51bV9maWx0ZXJzICogbWF4KG1pbihtaW5fdm90ZXMsIDEpLCAwKSkpCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidm90ZXMgbmVlZGVkIHRvIGJlIHNlbGVjdGVkOiB7dm90ZXNfbmVlZGVkfSIpCgogICAgIyBDcmVhdGUgZmluYWwgZmVhdHVyZSBkYXRhZnJhbWUKICAgIHNlbGVjdGVkX2ZlYXR1cmVzID0gcmVzdWx0X21hdHJpeF9kZlsKICAgICAgICByZXN1bHRfbWF0cml4X2RmLm51bV92b3RlcyA+PSB2b3Rlc19uZWVkZWQKICAgIF0uaW5kZXgudG9saXN0KCkKICAgIGdvb2RfZmVhdHVyZV9kZiA9IGRmLmxvY1s6LCBzZWxlY3RlZF9mZWF0dXJlc10KICAgIGZpbmFsX2RmID0gcGQuY29uY2F0KFtnb29kX2ZlYXR1cmVfZGYsIHldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgIGtleT0ic2VsZWN0ZWRfZmVhdHVyZXMiLAogICAgICAgIGRmPWZpbmFsX2RmLAogICAgICAgIGxvY2FsX3BhdGg9InNlbGVjdGVkX2ZlYXR1cmVzLnBhcnF1ZXQiLAogICAgICAgIGZvcm1hdD0icGFycXVldCIsCiAgICApCgogICAgIyBDcmVhdGluZyBhIG5ldyBmZWF0dXJlIHZlY3RvciBjb250YWluaW5nIG9ubHkgdGhlIGlkZW50aWZpZWQgdG9wIGZlYXR1cmVzCiAgICBpZiBpc19mZWF0dXJlX3ZlY3RvciBhbmQgZGZfYXJ0aWZhY3QubWV0YS5zcGVjLmZlYXR1cmVzIGFuZCBvdXRwdXRfdmVjdG9yX25hbWU6CiAgICAgICAgIyBTZWxlY3RpbmcgdGhlIHRvcCBLIGZlYXR1cmVzIGZyb20gb3VyIHRvcCBmZWF0dXJlIGRhdGFmcmFtZQogICAgICAgIHNlbGVjdGVkX2ZlYXR1cmVzID0gcmVzdWx0X21hdHJpeF9kZi5oZWFkKGspLmluZGV4CgogICAgICAgICMgTWF0Y2ggdGhlIHNlbGVjdGVkIGZlYXR1cmUgbmFtZXMgdG8gdGhlIEZTIEZlYXR1cmUgYW5ub3RhdGlvbnMKICAgICAgICBtYXRjaGVkX3NlbGVjdGlvbnMgPSBbCiAgICAgICAgICAgIGZlYXR1cmUKICAgICAgICAgICAgZm9yIGZlYXR1cmUgaW4gbGlzdChkZl9hcnRpZmFjdC5tZXRhLnNwZWMuZmVhdHVyZXMpCiAgICAgICAgICAgIGZvciBzZWxlY3RlZCBpbiBsaXN0KHNlbGVjdGVkX2ZlYXR1cmVzKQogICAgICAgICAgICBpZiBmZWF0dXJlLmVuZHN3aXRoKHNlbGVjdGVkKQogICAgICAgIF0KCiAgICAgICAgIyBEZWZpbmluZyBvdXIgbmV3IGZlYXR1cmUgdmVjdG9yCiAgICAgICAgdG9wX2ZlYXR1cmVzX2Z2ID0gZnMuRmVhdHVyZVZlY3RvcigKICAgICAgICAgICAgb3V0cHV0X3ZlY3Rvcl9uYW1lLAogICAgICAgICAgICBtYXRjaGVkX3NlbGVjdGlvbnMsCiAgICAgICAgICAgIGxhYmVsX2ZlYXR1cmU9ImxhYmVscy5sYWJlbCIsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJmZWF0dXJlIHZlY3RvciBjb21wb3NlZCBzdHJpY3RseSBvZiBvdXIgdG9wIGZlYXR1cmVzIiwKICAgICAgICApCgogICAgICAgICMgU2F2aW5nCiAgICAgICAgdG9wX2ZlYXR1cmVzX2Z2LnNhdmUoKQogICAgICAgIHRvcF9mZWF0dXJlc19mdi5nZXRfb2ZmbGluZV9mZWF0dXJlcyh0YXJnZXQ9UGFycXVldFRhcmdldCgpKQoKICAgICAgICAjIExvZ2dpbmcgb3VyIG5ldyBmZWF0dXJlIHZlY3RvciBVUkkKICAgICAgICBjb250ZXh0LmxvZ19yZXN1bHQoInRvcF9mZWF0dXJlc192ZWN0b3IiLCB0b3BfZmVhdHVyZXNfZnYudXJpKQo= - code_origin: '' + has_kwargs: false + has_varargs: false + lineno: 84 + command: '' description: Select features through multiple Statistical and Model filters default_handler: feature_selection -kind: job -metadata: - categories: - - data-preparation - - machine-learning - name: feature-selection - tag: '' -verbose: false diff --git a/functions/src/feature_selection/item.yaml b/functions/src/feature_selection/item.yaml index 4f9a3a5dd..8e0911229 100644 --- a/functions/src/feature_selection/item.yaml +++ b/functions/src/feature_selection/item.yaml @@ -12,7 +12,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.8.0-rc40 +mlrunVersion: 1.8.0 name: feature-selection platformVersion: 3.6.0 spec: diff --git a/functions/src/gen_class_data/function.yaml b/functions/src/gen_class_data/function.yaml index 1769bec07..b4d175d67 100644 --- a/functions/src/gen_class_data/function.yaml +++ b/functions/src/gen_class_data/function.yaml @@ -1,14 +1,20 @@ metadata: - categories: - - data-generation tag: '' name: gen-class-data + categories: + - data-generation +verbose: false +kind: job spec: - description: Create a binary classification sample dataset and save. - default_handler: gen_class_data + image: mlrun/mlrun + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKCmltcG9ydCBwYW5kYXMgYXMgcGQKZnJvbSBtbHJ1bi5leGVjdXRpb24gaW1wb3J0IE1MQ2xpZW50Q3R4CmZyb20gc2tsZWFybi5kYXRhc2V0cyBpbXBvcnQgbWFrZV9jbGFzc2lmaWNhdGlvbgoKCmRlZiBnZW5fY2xhc3NfZGF0YSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbl9zYW1wbGVzOiBpbnQsCiAgICBtX2ZlYXR1cmVzOiBpbnQsCiAgICBrX2NsYXNzZXM6IGludCwKICAgIGhlYWRlcjogbGlzdFtzdHJdIHwgTm9uZSwKICAgIGxhYmVsX2NvbHVtbjogc3RyIHwgTm9uZSA9ICJsYWJlbHMiLAogICAgd2VpZ2h0OiBmbG9hdCA9IDAuNSwKICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gMSwKICAgIGtleTogc3RyID0gImNsYXNzaWZpZXItZGF0YSIsCiAgICBmaWxlX2V4dDogc3RyID0gInBhcnF1ZXQiLAogICAgc2tfcGFyYW1zPXt9LAopOgogICAgIiIiQ3JlYXRlIGEgYmluYXJ5IGNsYXNzaWZpY2F0aW9uIHNhbXBsZSBkYXRhc2V0IGFuZCBzYXZlLgogICAgSWYgbm8gZmlsZW5hbWUgaXMgZ2l2ZW4gaXQgd2lsbCBkZWZhdWx0IHRvOgogICAgInNpbWRhdGEte25fc2FtcGxlc31Ye21fZmVhdHVyZXN9LnBhcnF1ZXQiLgoKICAgIEFkZGl0aW9uYWwgc2Npa2l0LWxlYXJuIHBhcmFtZXRlcnMgY2FuIGJlIHNldCB1c2luZyAqKnNrX3BhcmFtcywgcGxlYXNlIHNlZSBodHRwczovL3NjaWtpdC1sZWFybi5vcmcvc3RhYmxlL21vZHVsZXMvZ2VuZXJhdGVkL3NrbGVhcm4uZGF0YXNldHMubWFrZV9jbGFzc2lmaWNhdGlvbi5odG1sIGZvciBtb3JlIGRldGFpbHMuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgIGZ1bmN0aW9uIGNvbnRleHQKICAgIDpwYXJhbSBuX3NhbXBsZXM6ICAgICBudW1iZXIgb2Ygcm93cy9zYW1wbGVzCiAgICA6cGFyYW0gbV9mZWF0dXJlczogICAgbnVtYmVyIG9mIGNvbHMvZmVhdHVyZXMKICAgIDpwYXJhbSBrX2NsYXNzZXM6ICAgICBudW1iZXIgb2YgY2xhc3NlcwogICAgOnBhcmFtIGhlYWRlcjogICAgICAgIGhlYWRlciBmb3IgZmVhdHVyZXMgYXJyYXkKICAgIDpwYXJhbSBsYWJlbF9jb2x1bW46ICBjb2x1bW4gbmFtZSBvZiBncm91bmQtdHJ1dGggc2VyaWVzCiAgICA6cGFyYW0gd2VpZ2h0OiAgICAgICAgZnJhY3Rpb24gb2Ygc2FtcGxlIG5lZ2F0aXZlIHZhbHVlIChncm91bmQtdHJ1dGg9MCkKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICBybmcgc2VlZCAoc2VlIGh0dHBzOi8vc2Npa2l0LWxlYXJuLm9yZy9zdGFibGUvZ2xvc3NhcnkuaHRtbCN0ZXJtLXJhbmRvbS1zdGF0ZSkKICAgIDpwYXJhbSBrZXk6ICAgICAgICAgICBrZXkgb2YgZGF0YSBpbiBhcnRpZmFjdCBzdG9yZQogICAgOnBhcmFtIGZpbGVfZXh0OiAgICAgIChwcXQpIGV4dGVuc2lvbiBmb3IgcGFycXVldCBmaWxlCiAgICA6cGFyYW0gc2tfcGFyYW1zOiAgICAgYWRkaXRpb25hbCBwYXJhbWV0ZXJzIGZvciBgc2tsZWFybi5kYXRhc2V0cy5tYWtlX2NsYXNzaWZpY2F0aW9uYAogICAgIiIiCiAgICBmZWF0dXJlcywgbGFiZWxzID0gbWFrZV9jbGFzc2lmaWNhdGlvbigKICAgICAgICBuX3NhbXBsZXM9bl9zYW1wbGVzLAogICAgICAgIG5fZmVhdHVyZXM9bV9mZWF0dXJlcywKICAgICAgICB3ZWlnaHRzPXdlaWdodCwKICAgICAgICBuX2NsYXNzZXM9a19jbGFzc2VzLAogICAgICAgIHJhbmRvbV9zdGF0ZT1yYW5kb21fc3RhdGUsCiAgICAgICAgKipza19wYXJhbXMsCiAgICApCgogICAgIyBtYWtlIGRhdGFmcmFtZXMsIGFkZCBjb2x1bW4gbmFtZXMsIGNvbmNhdGVuYXRlIChYLCB5KQogICAgWCA9IHBkLkRhdGFGcmFtZShmZWF0dXJlcykKICAgIGlmIG5vdCBoZWFkZXI6CiAgICAgICAgWC5jb2x1bW5zID0gWyJmZWF0XyIgKyBzdHIoeCkgZm9yIHggaW4gcmFuZ2UobV9mZWF0dXJlcyldCiAgICBlbHNlOgogICAgICAgIFguY29sdW1ucyA9IGhlYWRlcgoKICAgIHkgPSBwZC5EYXRhRnJhbWUobGFiZWxzLCBjb2x1bW5zPVtsYWJlbF9jb2x1bW5dKQogICAgZGF0YSA9IHBkLmNvbmNhdChbWCwgeV0sIGF4aXM9MSkKCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KGtleSwgZGY9ZGF0YSwgZm9ybWF0PWZpbGVfZXh0LCBpbmRleD1GYWxzZSkK + code_origin: '' + filename: gen_class_data.py entry_points: gen_class_data: - has_kwargs: false parameters: - name: context type: MLClientCtx @@ -23,10 +29,8 @@ spec: type: int doc: number of classes - name: header - type: Optional[List[str]] doc: header for features array - name: label_column - type: Optional[str] doc: column name of ground-truth series default: labels - name: weight @@ -48,7 +52,7 @@ spec: - name: sk_params doc: additional parameters for `sklearn.datasets.make_classification` default: {} - lineno: 22 + name: gen_class_data doc: 'Create a binary classification sample dataset and save. If no filename is given it will default to: @@ -59,14 +63,9 @@ spec: Additional scikit-learn parameters can be set using **sk_params, please see https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_classification.html for more details.' + has_kwargs: false has_varargs: false - name: gen_class_data + lineno: 21 command: '' - disable_auto_mount: false - image: mlrun/mlrun - build: - origin_filename: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IHBhbmRhcyBhcyBwZApmcm9tIHR5cGluZyBpbXBvcnQgT3B0aW9uYWwsIExpc3QKZnJvbSBza2xlYXJuLmRhdGFzZXRzIGltcG9ydCBtYWtlX2NsYXNzaWZpY2F0aW9uCgpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKCgpkZWYgZ2VuX2NsYXNzX2RhdGEoCiAgICAgICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICAgICAgbl9zYW1wbGVzOiBpbnQsCiAgICAgICAgbV9mZWF0dXJlczogaW50LAogICAgICAgIGtfY2xhc3NlczogaW50LAogICAgICAgIGhlYWRlcjogT3B0aW9uYWxbTGlzdFtzdHJdXSwKICAgICAgICBsYWJlbF9jb2x1bW46IE9wdGlvbmFsW3N0cl0gPSAibGFiZWxzIiwKICAgICAgICB3ZWlnaHQ6IGZsb2F0ID0gMC41LAogICAgICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gMSwKICAgICAgICBrZXk6IHN0ciA9ICJjbGFzc2lmaWVyLWRhdGEiLAogICAgICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICAgICAgc2tfcGFyYW1zPXt9Cik6CiAgICAiIiJDcmVhdGUgYSBiaW5hcnkgY2xhc3NpZmljYXRpb24gc2FtcGxlIGRhdGFzZXQgYW5kIHNhdmUuCiAgICBJZiBubyBmaWxlbmFtZSBpcyBnaXZlbiBpdCB3aWxsIGRlZmF1bHQgdG86CiAgICAic2ltZGF0YS17bl9zYW1wbGVzfVh7bV9mZWF0dXJlc30ucGFycXVldCIuCgogICAgQWRkaXRpb25hbCBzY2lraXQtbGVhcm4gcGFyYW1ldGVycyBjYW4gYmUgc2V0IHVzaW5nICoqc2tfcGFyYW1zLCBwbGVhc2Ugc2VlIGh0dHBzOi8vc2Npa2l0LWxlYXJuLm9yZy9zdGFibGUvbW9kdWxlcy9nZW5lcmF0ZWQvc2tsZWFybi5kYXRhc2V0cy5tYWtlX2NsYXNzaWZpY2F0aW9uLmh0bWwgZm9yIG1vcmUgZGV0YWlscy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgZnVuY3Rpb24gY29udGV4dAogICAgOnBhcmFtIG5fc2FtcGxlczogICAgIG51bWJlciBvZiByb3dzL3NhbXBsZXMKICAgIDpwYXJhbSBtX2ZlYXR1cmVzOiAgICBudW1iZXIgb2YgY29scy9mZWF0dXJlcwogICAgOnBhcmFtIGtfY2xhc3NlczogICAgIG51bWJlciBvZiBjbGFzc2VzCiAgICA6cGFyYW0gaGVhZGVyOiAgICAgICAgaGVhZGVyIGZvciBmZWF0dXJlcyBhcnJheQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogIGNvbHVtbiBuYW1lIG9mIGdyb3VuZC10cnV0aCBzZXJpZXMKICAgIDpwYXJhbSB3ZWlnaHQ6ICAgICAgICBmcmFjdGlvbiBvZiBzYW1wbGUgbmVnYXRpdmUgdmFsdWUgKGdyb3VuZC10cnV0aD0wKQogICAgOnBhcmFtIHJhbmRvbV9zdGF0ZTogIHJuZyBzZWVkIChzZWUgaHR0cHM6Ly9zY2lraXQtbGVhcm4ub3JnL3N0YWJsZS9nbG9zc2FyeS5odG1sI3Rlcm0tcmFuZG9tLXN0YXRlKQogICAgOnBhcmFtIGtleTogICAgICAgICAgIGtleSBvZiBkYXRhIGluIGFydGlmYWN0IHN0b3JlCiAgICA6cGFyYW0gZmlsZV9leHQ6ICAgICAgKHBxdCkgZXh0ZW5zaW9uIGZvciBwYXJxdWV0IGZpbGUKICAgIDpwYXJhbSBza19wYXJhbXM6ICAgICBhZGRpdGlvbmFsIHBhcmFtZXRlcnMgZm9yIGBza2xlYXJuLmRhdGFzZXRzLm1ha2VfY2xhc3NpZmljYXRpb25gCiAgICAiIiIKICAgIGZlYXR1cmVzLCBsYWJlbHMgPSBtYWtlX2NsYXNzaWZpY2F0aW9uKAogICAgICAgIG5fc2FtcGxlcz1uX3NhbXBsZXMsCiAgICAgICAgbl9mZWF0dXJlcz1tX2ZlYXR1cmVzLAogICAgICAgIHdlaWdodHM9d2VpZ2h0LAogICAgICAgIG5fY2xhc3Nlcz1rX2NsYXNzZXMsCiAgICAgICAgcmFuZG9tX3N0YXRlPXJhbmRvbV9zdGF0ZSwKICAgICAgICAqKnNrX3BhcmFtcykKCiAgICAjIG1ha2UgZGF0YWZyYW1lcywgYWRkIGNvbHVtbiBuYW1lcywgY29uY2F0ZW5hdGUgKFgsIHkpCiAgICBYID0gcGQuRGF0YUZyYW1lKGZlYXR1cmVzKQogICAgaWYgbm90IGhlYWRlcjoKICAgICAgICBYLmNvbHVtbnMgPSBbImZlYXRfIiArIHN0cih4KSBmb3IgeCBpbiByYW5nZShtX2ZlYXR1cmVzKV0KICAgIGVsc2U6CiAgICAgICAgWC5jb2x1bW5zID0gaGVhZGVyCgogICAgeSA9IHBkLkRhdGFGcmFtZShsYWJlbHMsIGNvbHVtbnM9W2xhYmVsX2NvbHVtbl0pCiAgICBkYXRhID0gcGQuY29uY2F0KFtYLCB5XSwgYXhpcz0xKQoKICAgIGNvbnRleHQubG9nX2RhdGFzZXQoa2V5LCBkZj1kYXRhLCBmb3JtYXQ9ZmlsZV9leHQsIGluZGV4PUZhbHNlKQo= - code_origin: '' -kind: job -verbose: false + description: Create a binary classification sample dataset and save. + default_handler: gen_class_data diff --git a/functions/src/gen_class_data/gen_class_data.py b/functions/src/gen_class_data/gen_class_data.py index 2e5ab1073..8e8774f00 100644 --- a/functions/src/gen_class_data/gen_class_data.py +++ b/functions/src/gen_class_data/gen_class_data.py @@ -12,25 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import pandas as pd -from typing import Optional, List -from sklearn.datasets import make_classification +import pandas as pd from mlrun.execution import MLClientCtx +from sklearn.datasets import make_classification def gen_class_data( - context: MLClientCtx, - n_samples: int, - m_features: int, - k_classes: int, - header: Optional[List[str]], - label_column: Optional[str] = "labels", - weight: float = 0.5, - random_state: int = 1, - key: str = "classifier-data", - file_ext: str = "parquet", - sk_params={} + context: MLClientCtx, + n_samples: int, + m_features: int, + k_classes: int, + header: list[str] | None, + label_column: str | None = "labels", + weight: float = 0.5, + random_state: int = 1, + key: str = "classifier-data", + file_ext: str = "parquet", + sk_params={}, ): """Create a binary classification sample dataset and save. If no filename is given it will default to: @@ -56,7 +55,8 @@ def gen_class_data( weights=weight, n_classes=k_classes, random_state=random_state, - **sk_params) + **sk_params, + ) # make dataframes, add column names, concatenate (X, y) X = pd.DataFrame(features) diff --git a/functions/src/gen_class_data/test_gen_class_data.py b/functions/src/gen_class_data/test_gen_class_data.py index e06eeb16b..deb354dc0 100644 --- a/functions/src/gen_class_data/test_gen_class_data.py +++ b/functions/src/gen_class_data/test_gen_class_data.py @@ -12,13 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from mlrun import code_to_function import os +from mlrun import code_to_function + def test_gen_class_data(): fn = code_to_function( - name='test_gen_class_data', + name="test_gen_class_data", filename="gen_class_data.py", handler="gen_class_data", kind="job", @@ -32,8 +33,11 @@ def test_gen_class_data(): "header": None, "weight": [0.5, 0.5], "sk_params": {"n_informative": 2}, - "file_ext": "csv"}, + "file_ext": "csv", + }, local=True, artifact_path="./artifacts", - ) - assert os.path.isfile(run.status.artifacts[0]['spec']['target_path']), 'dataset is not available' + ) + assert os.path.isfile(run.status.artifacts[0]["spec"]["target_path"]), ( + "dataset is not available" + ) diff --git a/functions/src/github_utils/function.yaml b/functions/src/github_utils/function.yaml index 2d5d93aab..68b5afd8f 100644 --- a/functions/src/github_utils/function.yaml +++ b/functions/src/github_utils/function.yaml @@ -1,64 +1,52 @@ -kind: job metadata: - name: github-utils tag: '' - hash: d8e639af306794ce6f59eb246f0b845c016c9da4 - project: '' - labels: - author: Iguazio + name: github-utils categories: - utils +verbose: false +kind: job spec: - command: '' - args: [] image: mlrun/mlrun - env: [] - default_handler: run_summary_comment + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IG9zCgppbXBvcnQgcmVxdWVzdHMKZnJvbSBtbHJ1biBpbXBvcnQgRGF0YUl0ZW0sIGdldF9ydW5fZGIKCgpkZWYgcHJfY29tbWVudCgKICAgIGNvbnRleHQsIHJlcG86IHN0ciwgaXNzdWU6IGludCwgbWVzc2FnZTogc3RyID0gIiIsIG1lc3NhZ2VfZmlsZTogRGF0YUl0ZW0gPSBOb25lCik6CiAgICB0b2tlbiA9IGNvbnRleHQuZ2V0X3NlY3JldCgiR0lUSFVCX1RPS0VOIikgb3Igb3MuZW52aXJvbi5nZXQoIkdJVEhVQl9UT0tFTiIpCiAgICBpZiBtZXNzYWdlX2ZpbGUgYW5kIG5vdCBtZXNzYWdlOgogICAgICAgIG1lc3NhZ2UgPSBtZXNzYWdlX2ZpbGUuZ2V0KCkKICAgIGVsaWYgbm90IG1lc3NhZ2UgYW5kIG5vdCBtZXNzYWdlX2ZpbGU6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigicHIgbWVzc2FnZSBvciBtZXNzYWdlIGZpbGUgbXVzdCBiZSBwcm92aWRlZCIpCgogICAgaGVhZGVycyA9IHsKICAgICAgICAiQWNjZXB0IjogImFwcGxpY2F0aW9uL3ZuZC5naXRodWIudjMranNvbiIsCiAgICAgICAgIkF1dGhvcml6YXRpb24iOiBmInRva2VuIHt0b2tlbn0iLAogICAgfQogICAgdXJsID0gZiJodHRwczovL2FwaS5naXRodWIuY29tL3JlcG9zL3tyZXBvfS9pc3N1ZXMve2lzc3VlfS9jb21tZW50cyIKCiAgICByZXNwID0gcmVxdWVzdHMucG9zdCh1cmw9dXJsLCBqc29uPXsiYm9keSI6IHN0cihtZXNzYWdlKX0sIGhlYWRlcnM9aGVhZGVycykKICAgIGlmIG5vdCByZXNwLm9rOgogICAgICAgIGVycm1zZyA9IGYiYmFkIHByIGNvbW1lbnQgcmVzcCEhXG57cmVzcC50ZXh0fSIKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcihlcnJtc2cpCiAgICAgICAgcmFpc2UgT1NFcnJvcihlcnJtc2cpCgoKZGVmIHJ1bl9zdW1tYXJ5X2NvbW1lbnQoY29udGV4dCwgd29ya2Zsb3dfaWQsIHJlcG86IHN0ciwgaXNzdWU6IGludCwgcHJvamVjdD0iIik6CiAgICBkYiA9IGdldF9ydW5fZGIoKS5jb25uZWN0KCkKICAgIHByb2plY3QgPSBwcm9qZWN0IG9yIGNvbnRleHQucHJvamVjdAogICAgcnVucyA9IGRiLmxpc3RfcnVucyhwcm9qZWN0PXByb2plY3QsIGxhYmVscz1mIndvcmtmbG93PXt3b3JrZmxvd19pZH0iKQoKICAgIGhhZF9lcnJvcnMgPSBpID0gMAogICAgZm9yIHIgaW4gcnVuczoKICAgICAgICBuYW1lID0gclsibWV0YWRhdGEiXVsibmFtZSJdCiAgICAgICAgaWYgclsic3RhdHVzIl0uZ2V0KCJzdGF0ZSIsICIiKSA9PSAiZXJyb3IiOgogICAgICAgICAgICBoYWRfZXJyb3JzICs9IDEKICAgICAgICBpZiBuYW1lID09IGNvbnRleHQubmFtZToKICAgICAgICAgICAgZGVsIHJ1bnNbaV0KICAgICAgICBpICs9IDEKCiAgICBwcmludCgiZXJyb3JzOiIsIGhhZF9lcnJvcnMpCgogICAgaHRtbCA9IGYiIyMjIFJ1biBSZXN1bHRzXG5Xb3JrZmxvdyB7d29ya2Zsb3dfaWR9IGZpbmlzaGVkIHdpdGgge2hhZF9lcnJvcnN9IGVycm9ycyIKICAgIGh0bWwgKz0gIjxicj5jbGljayB0aGUgaHlwZXIgbGlua3MgYmVsb3cgdG8gc2VlIGRldGFpbGVkIHJlc3VsdHM8YnI+IgogICAgaHRtbCArPSBydW5zLnNob3coZGlzcGxheT1GYWxzZSwgc2hvcnQ9VHJ1ZSkKICAgIGlmIHJlcG86CiAgICAgICAgcHJfY29tbWVudChjb250ZXh0LCByZXBvLCBpc3N1ZSwgaHRtbCkKICAgIGVsc2U6CiAgICAgICAgcHJpbnQoInJlcG8gbm90IGRlZmluZWQiKQogICAgICAgIHByaW50KGh0bWwpCg== + code_origin: '' + filename: github_utils.py entry_points: pr_comment: - name: pr_comment - doc: '' parameters: - name: context - default: '' - name: repo type: str - default: '' - name: issue type: int - default: '' - name: message type: str default: '' - name: message_file type: DataItem default: null - outputs: - - default: '' - lineno: 8 - run_summary_comment: - name: run_summary_comment + name: pr_comment doc: '' + has_kwargs: false + has_varargs: false + lineno: 23 + run_summary_comment: parameters: - name: context - default: '' - name: workflow_id - default: '' - name: repo type: str - default: '' - name: issue type: int - default: '' - name: project default: '' - outputs: - - default: '' - lineno: 31 + name: run_summary_comment + doc: '' + has_kwargs: false + has_varargs: false + lineno: 45 + command: '' description: add comments to github pull request - build: - functionSourceCode: IyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHJlcXVlc3RzCmltcG9ydCBvcwpmcm9tIG1scnVuIGltcG9ydCBEYXRhSXRlbSwgZ2V0X3J1bl9kYiwgbWxjb25mCgoKZGVmIHByX2NvbW1lbnQoCiAgICBjb250ZXh0LCByZXBvOiBzdHIsIGlzc3VlOiBpbnQsIG1lc3NhZ2U6IHN0ciA9ICIiLCBtZXNzYWdlX2ZpbGU6IERhdGFJdGVtID0gTm9uZQopOgoKICAgIHRva2VuID0gY29udGV4dC5nZXRfc2VjcmV0KCJHSVRIVUJfVE9LRU4iKSBvciBvcy5lbnZpcm9uLmdldCgiR0lUSFVCX1RPS0VOIikKICAgIGlmIG1lc3NhZ2VfZmlsZSBhbmQgbm90IG1lc3NhZ2U6CiAgICAgICAgbWVzc2FnZSA9IG1lc3NhZ2VfZmlsZS5nZXQoKQogICAgZWxpZiBub3QgbWVzc2FnZSBhbmQgbm90IG1lc3NhZ2VfZmlsZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJwciBtZXNzYWdlIG9yIG1lc3NhZ2UgZmlsZSBtdXN0IGJlIHByb3ZpZGVkIikKCiAgICBoZWFkZXJzID0gewogICAgICAgICJBY2NlcHQiOiAiYXBwbGljYXRpb24vdm5kLmdpdGh1Yi52Mytqc29uIiwKICAgICAgICAiQXV0aG9yaXphdGlvbiI6IGYidG9rZW4ge3Rva2VufSIsCiAgICB9CiAgICB1cmwgPSBmImh0dHBzOi8vYXBpLmdpdGh1Yi5jb20vcmVwb3Mve3JlcG99L2lzc3Vlcy97aXNzdWV9L2NvbW1lbnRzIgoKICAgIHJlc3AgPSByZXF1ZXN0cy5wb3N0KHVybD11cmwsIGpzb249eyJib2R5Ijogc3RyKG1lc3NhZ2UpfSwgaGVhZGVycz1oZWFkZXJzKQogICAgaWYgbm90IHJlc3Aub2s6CiAgICAgICAgZXJybXNnID0gZiJiYWQgcHIgY29tbWVudCByZXNwISFcbntyZXNwLnRleHR9IgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKGVycm1zZykKICAgICAgICByYWlzZSBJT0Vycm9yKGVycm1zZykKCgpkZWYgcnVuX3N1bW1hcnlfY29tbWVudChjb250ZXh0LCB3b3JrZmxvd19pZCwgcmVwbzogc3RyLCBpc3N1ZTogaW50LCBwcm9qZWN0PSIiKToKICAgIGRiID0gZ2V0X3J1bl9kYigpLmNvbm5lY3QoKQogICAgcHJvamVjdCA9IHByb2plY3Qgb3IgY29udGV4dC5wcm9qZWN0CiAgICBydW5zID0gZGIubGlzdF9ydW5zKHByb2plY3Q9cHJvamVjdCwgbGFiZWxzPWYid29ya2Zsb3c9e3dvcmtmbG93X2lkfSIpCgogICAgaGFkX2Vycm9ycyA9IGkgPSAwCiAgICBmb3IgciBpbiBydW5zOgogICAgICAgIG5hbWUgPSByWyJtZXRhZGF0YSJdWyJuYW1lIl0KICAgICAgICBpZiByWyJzdGF0dXMiXS5nZXQoInN0YXRlIiwgIiIpID09ICJlcnJvciI6CiAgICAgICAgICAgIGhhZF9lcnJvcnMgKz0gMQogICAgICAgIGlmIG5hbWUgPT0gY29udGV4dC5uYW1lOgogICAgICAgICAgICBkZWwgcnVuc1tpXQogICAgICAgIGkgKz0gMQoKICAgIHByaW50KCJlcnJvcnM6IiwgaGFkX2Vycm9ycykKCiAgICBodG1sID0gIiMjIyBSdW4gUmVzdWx0c1xuV29ya2Zsb3cge30gZmluaXNoZWQgd2l0aCB7fSBlcnJvcnMiLmZvcm1hdCgKICAgICAgICB3b3JrZmxvd19pZCwgaGFkX2Vycm9ycwogICAgKQogICAgaHRtbCArPSAiPGJyPmNsaWNrIHRoZSBoeXBlciBsaW5rcyBiZWxvdyB0byBzZWUgZGV0YWlsZWQgcmVzdWx0czxicj4iCiAgICBodG1sICs9IHJ1bnMuc2hvdyhkaXNwbGF5PUZhbHNlLCBzaG9ydD1UcnVlKQogICAgaWYgcmVwbzoKICAgICAgICBwcl9jb21tZW50KGNvbnRleHQsIHJlcG8sIGlzc3VlLCBodG1sKQogICAgZWxzZToKICAgICAgICBwcmludCgicmVwbyBub3QgZGVmaW5lZCIpCiAgICAgICAgcHJpbnQoaHRtbCkK - commands: [] - code_origin: https://github.com/daniels290813/functions.git#55a79c32be5d233cc11efcf40cd3edbe309bfdef:/home/kali/functions/github_utils/github_utils.py - affinity: null -verbose: false + default_handler: run_summary_comment diff --git a/functions/src/github_utils/github_utils.py b/functions/src/github_utils/github_utils.py index dc70456a9..09ed6a7bb 100644 --- a/functions/src/github_utils/github_utils.py +++ b/functions/src/github_utils/github_utils.py @@ -14,15 +14,15 @@ # # Generated by nuclio.export.NuclioExporter -import requests import os -from mlrun import DataItem, get_run_db, mlconf + +import requests +from mlrun import DataItem, get_run_db def pr_comment( context, repo: str, issue: int, message: str = "", message_file: DataItem = None ): - token = context.get_secret("GITHUB_TOKEN") or os.environ.get("GITHUB_TOKEN") if message_file and not message: message = message_file.get() @@ -39,7 +39,7 @@ def pr_comment( if not resp.ok: errmsg = f"bad pr comment resp!!\n{resp.text}" context.logger.error(errmsg) - raise IOError(errmsg) + raise OSError(errmsg) def run_summary_comment(context, workflow_id, repo: str, issue: int, project=""): @@ -58,9 +58,7 @@ def run_summary_comment(context, workflow_id, repo: str, issue: int, project="") print("errors:", had_errors) - html = "### Run Results\nWorkflow {} finished with {} errors".format( - workflow_id, had_errors - ) + html = f"### Run Results\nWorkflow {workflow_id} finished with {had_errors} errors" html += "
click the hyper links below to see detailed results
" html += runs.show(display=False, short=True) if repo: diff --git a/functions/src/hugging_face_serving/function.yaml b/functions/src/hugging_face_serving/function.yaml index a628d7ab7..3da9128a9 100644 --- a/functions/src/hugging_face_serving/function.yaml +++ b/functions/src/hugging_face_serving/function.yaml @@ -1,31 +1,32 @@ metadata: + tag: '' name: hugging-face-serving categories: - genai - model-serving - tag: '' +verbose: false +kind: serving spec: - default_handler: '' - min_replicas: 1 - source: '' image: mlrun/ml-models + disable_auto_mount: false build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKCmZyb20gYWJjIGltcG9ydCBBQkMKZnJvbSBpbXBvcnRsaWIgaW1wb3J0IGltcG9ydF9tb2R1bGUKZnJvbSB0eXBpbmcgaW1wb3J0IExpc3QKCmZyb20gdHJhbnNmb3JtZXJzIGltcG9ydCBwaXBlbGluZQoKaW1wb3J0IG1scnVuLnNlcnZpbmcKClBBQ0tBR0VfTU9EVUxFID0gInRyYW5zZm9ybWVycyIKU0VSSUFMSVpBQkxFX1RZUEVTID0gW2RpY3QsIGxpc3QsIHR1cGxlLCBzdHIsIGludCwgZmxvYXRdCgoKY2xhc3MgSHVnZ2luZ0ZhY2VNb2RlbFNlcnZlcihtbHJ1bi5zZXJ2aW5nLlYyTW9kZWxTZXJ2ZXIsIEFCQyk6CiAgICAiIiIKICAgIEh1Z2dpbmcgRmFjZSBNb2RlbCBzZXJ2aW5nIGNsYXNzLCBpbmhlcml0aW5nIHRoZSBWMk1vZGVsU2VydmVyIGNsYXNzIGZvciBiZWluZyBpbml0aWFsaXplZCBhdXRvbWF0aWNhbGx5IGJ5IHRoZQogICAgbW9kZWwgc2VydmVyIGFuZCBiZSBhYmxlIHRvIHJ1biBsb2NhbGx5IGFzIHBhcnQgb2YgYSBudWNsaW8gc2VydmVybGVzcyBmdW5jdGlvbiwgb3IgYXMgcGFydCBvZiBhIHJlYWwtdGltZSBwaXBlbGluZS4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgICAgIG5hbWU6IHN0ciwKICAgICAgICB0YXNrOiBzdHIsCiAgICAgICAgbW9kZWxfcGF0aDogc3RyID0gTm9uZSwKICAgICAgICBtb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG1vZGVsX2NsYXNzOiBzdHIgPSBOb25lLAogICAgICAgIHRva2VuaXplcl9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIHRva2VuaXplcl9jbGFzczogc3RyID0gTm9uZSwKICAgICAgICBmcmFtZXdvcms6IHN0ciA9IE5vbmUsCiAgICAgICAgKipjbGFzc19hcmdzLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIGEgc2VydmluZyBjbGFzcyBmb3IgYSBIdWdnaW5nIGZhY2UgbW9kZWwuCgogICAgICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIFRoZSBtbHJ1biBjb250ZXh0IHRvIHdvcmsgd2l0aAogICAgICAgIDpwYXJhbSBuYW1lOiAgICAgICAgICAgIFRoZSBuYW1lIG9mIHRoaXMgc2VydmVyIHRvIGJlIGluaXRpYWxpemVkCiAgICAgICAgOnBhcmFtIG1vZGVsX3BhdGg6ICAgICAgTm90IGluIHVzZS4gV2hlbiBhZGRpbmcgYSBtb2RlbCBwYXNzIGFueSBzdHJpbmcgdmFsdWUKICAgICAgICA6cGFyYW0gbW9kZWxfbmFtZTogICAgICBUaGUgbW9kZWwncyBuYW1lIGluIHRoZSBIdWdnaW5nIEZhY2UgaHViCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZS5nLiwgYG5scHRvd24vYmVydC1iYXNlLW11bHRpbGluZ3VhbC11bmNhc2VkLXNlbnRpbWVudGAKICAgICAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICBUaGUgbW9kZWwncyBjbGFzcyB0eXBlIG9iamVjdCB3aGljaCBjYW4gYmUgcGFzc2VkIGFzIHRoZSBjbGFzcydzIG5hbWUgKHN0cmluZykuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTXVzdCBiZSBwcm92aWRlZCBhbmQgdG8gYmUgbWF0Y2hlZCB3aXRoIGBtb2RlbF9uYW1lYC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBlLmcuLCBgQXV0b01vZGVsRm9yU2VxdWVuY2VDbGFzc2lmaWNhdGlvbmAKICAgICAgICA6cGFyYW0gdG9rZW5pemVyX25hbWU6ICBUaGUgdG9rZW5pemVyJ3MgbmFtZSBpbiB0aGUgSHVnZ2luZyBGYWNlIGh1YgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGUuZy4sIGBubHB0b3duL2JlcnQtYmFzZS1tdWx0aWxpbmd1YWwtdW5jYXNlZC1zZW50aW1lbnRgCiAgICAgICAgOnBhcmFtIHRva2VuaXplcl9jbGFzczogVGhlIG1vZGVsJ3MgY2xhc3MgdHlwZSBvYmplY3Qgd2hpY2ggY2FuIGJlIHBhc3NlZCBhcyB0aGUgY2xhc3MncyBuYW1lIChzdHJpbmcpLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE11c3QgYmUgcHJvdmlkZWQgYW5kIHRvIGJlIG1hdGNoZWQgd2l0aCBgbW9kZWxfbmFtZWAuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZS5nLiwgYEF1dG9Ub2tlbml6ZXJgCiAgICAgICAgOnBhcmFtIGZyYW1ld29yazogICAgICAgVGhlIGZyYW1ld29yayB0byB1c2UsIGVpdGhlciBgInB0ImAgZm9yIFB5VG9yY2ggb3IgYCJ0ZiJgIGZvciBUZW5zb3JGbG93LiBUaGUgc3BlY2lmaWVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZnJhbWV3b3JrIG11c3QgYmUgaW5zdGFsbGVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIElmIG5vIGZyYW1ld29yayBpcyBzcGVjaWZpZWQsIHdpbGwgZGVmYXVsdCB0byB0aGUgb25lIGN1cnJlbnRseSBpbnN0YWxsZWQuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgbm8gZnJhbWV3b3JrIGlzIHNwZWNpZmllZCBhbmQgYm90aCBmcmFtZXdvcmtzIGFyZSBpbnN0YWxsZWQsIHdpbGwgZGVmYXVsdCB0byB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmcmFtZXdvcmsgb2YgdGhlIGBtb2RlbGAsIG9yIHRvIFB5VG9yY2ggaWYgbm8gbW9kZWwgaXMgcHJvdmlkZWQuCiAgICAgICAgOnBhcmFtIGNsYXNzX2FyZ3M6ICAgICAgLQogICAgICAgICIiIgogICAgICAgIHN1cGVyKEh1Z2dpbmdGYWNlTW9kZWxTZXJ2ZXIsIHNlbGYpLl9faW5pdF9fKAogICAgICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgICAgIG5hbWU9bmFtZSwKICAgICAgICAgICAgbW9kZWxfcGF0aD1tb2RlbF9wYXRoLAogICAgICAgICAgICAqKmNsYXNzX2FyZ3MsCiAgICAgICAgKQogICAgICAgIHNlbGYudGFzayA9IHRhc2sKICAgICAgICBzZWxmLm1vZGVsID0gTm9uZQogICAgICAgIHNlbGYudG9rZW5pemVyID0gTm9uZQogICAgICAgIHNlbGYubW9kZWxfbmFtZSA9IG1vZGVsX25hbWUKICAgICAgICBzZWxmLnRva2VuaXplcl9uYW1lID0gdG9rZW5pemVyX25hbWUKICAgICAgICBzZWxmLm1vZGVsX2NsYXNzID0gbW9kZWxfY2xhc3MKICAgICAgICBzZWxmLnRva2VuaXplcl9jbGFzcyA9IHRva2VuaXplcl9jbGFzcwogICAgICAgIHNlbGYuZnJhbWV3b3JrID0gZnJhbWV3b3JrCiAgICAgICAgc2VsZi5waXBlID0gTm9uZQoKICAgIGRlZiBsb2FkKHNlbGYpOgogICAgICAgICIiImxvYWQgYW5kIGluaXRpYWxpemUgdGhlIG1vZGVsIGFuZC9vciBvdGhlciBlbGVtZW50cyIiIgogICAgICAgIGlmIHNlbGYubW9kZWxfY2xhc3M6CiAgICAgICAgICAgIG1vZGVsX29iamVjdCA9IGdldGF0dHIoaW1wb3J0X21vZHVsZShQQUNLQUdFX01PRFVMRSksIHNlbGYubW9kZWxfY2xhc3MpCiAgICAgICAgICAgIHNlbGYubW9kZWwgPSBtb2RlbF9vYmplY3QuZnJvbV9wcmV0cmFpbmVkKHNlbGYubW9kZWxfbmFtZSkKICAgICAgICBpZiBzZWxmLnRva2VuaXplcl9jbGFzczoKICAgICAgICAgICAgdG9rZW5pemVyX29iamVjdCA9IGdldGF0dHIoCiAgICAgICAgICAgICAgICBpbXBvcnRfbW9kdWxlKFBBQ0tBR0VfTU9EVUxFKSwgc2VsZi50b2tlbml6ZXJfY2xhc3MKICAgICAgICAgICAgKQogICAgICAgICAgICBzZWxmLnRva2VuaXplciA9IHRva2VuaXplcl9vYmplY3QuZnJvbV9wcmV0cmFpbmVkKHNlbGYudG9rZW5pemVyX25hbWUpCiAgICAgICAgc2VsZi5waXBlID0gcGlwZWxpbmUoCiAgICAgICAgICAgIHRhc2s9c2VsZi50YXNrLAogICAgICAgICAgICBtb2RlbD1zZWxmLm1vZGVsIG9yIHNlbGYubW9kZWxfbmFtZSwKICAgICAgICAgICAgdG9rZW5pemVyPXNlbGYudG9rZW5pemVyLAogICAgICAgICAgICBmcmFtZXdvcms9c2VsZi5mcmFtZXdvcmssCiAgICAgICAgKQoKICAgIGRlZiBwcmVkaWN0KHNlbGYsIGJvZHk6IGRpY3QpIC0+IExpc3Q6CiAgICAgICAgIiIiR2VuZXJhdGUgbW9kZWwgcHJlZGljdGlvbnMgZnJvbSBzYW1wbGUuIiIiCiAgICAgICAgaWYgc2VsZi5waXBlIGlzIE5vbmU6CiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoIlBsZWFzZSB1c2UgYC5sb2FkKClgIikKICAgICAgICB0cnk6CiAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoYm9keVsiaW5wdXRzIl1bMF0sIGRpY3QpOgogICAgICAgICAgICAgICAgcmVzdWx0ID0gW3NlbGYucGlwZSgqKl9pbnB1dCkgZm9yIF9pbnB1dCBpbiBib2R5WyJpbnB1dHMiXV0KICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIHJlc3VsdCA9IHNlbGYucGlwZShib2R5WyJpbnB1dHMiXSkKICAgICAgICAgICAgIyByZXBsYWNlIGxpc3Qgb2YgbGlzdHMgb2YgZGljdHMgaW50byBhIGxpc3Qgb2YgZGljdHM6CiAgICAgICAgICAgIGlmIGFsbChpc2luc3RhbmNlKHJlcywgbGlzdCkgZm9yIHJlcyBpbiByZXN1bHQpOgogICAgICAgICAgICAgICAgbmV3X3Jlc3VsdCA9IFtyZXNbMF0gZm9yIHJlcyBpbiByZXN1bHRdCiAgICAgICAgICAgICAgICByZXN1bHQgPSBuZXdfcmVzdWx0CgogICAgICAgICAgICBub25fc2VyaWFsaXphYmxlX3R5cGVzID0gW10KICAgICAgICAgICAgZm9yIHJlcyBpbiByZXN1bHQ6CiAgICAgICAgICAgICAgICBmb3Iga2V5LCB2YWwgaW4gcmVzLml0ZW1zKCk6CiAgICAgICAgICAgICAgICAgICAgaWYgdHlwZSh2YWwpIG5vdCBpbiBTRVJJQUxJWkFCTEVfVFlQRVM6CiAgICAgICAgICAgICAgICAgICAgICAgIG5vbl9zZXJpYWxpemFibGVfdHlwZXMuYXBwZW5kKHN0cih0eXBlKHZhbCkpKQogICAgICAgICAgICAgICAgICAgICAgICByZXNba2V5XSA9IHN0cih2YWwpCiAgICAgICAgICAgIGlmIG5vbl9zZXJpYWxpemFibGVfdHlwZXM6CiAgICAgICAgICAgICAgICBzZWxmLmNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJOb24tc2VyaWFsaXphYmxlIHR5cGVzOiB7bm9uX3NlcmlhbGl6YWJsZV90eXBlc30gd2VyZSBjYXN0ZWQgdG8gc3RyaW5ncyIKICAgICAgICAgICAgICAgICkKICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgICAgIHJhaXNlIEV4Y2VwdGlvbigiRmFpbGVkIHRvIHByZWRpY3QgJXMiICUgZSkKICAgICAgICByZXR1cm4gcmVzdWx0Cgpmcm9tIG1scnVuLnJ1bnRpbWVzIGltcG9ydCBudWNsaW9faW5pdF9ob29rCmRlZiBpbml0X2NvbnRleHQoY29udGV4dCk6CiAgICBudWNsaW9faW5pdF9ob29rKGNvbnRleHQsIGdsb2JhbHMoKSwgJ3NlcnZpbmdfdjInKQoKZGVmIGhhbmRsZXIoY29udGV4dCwgZXZlbnQpOgogICAgcmV0dXJuIGNvbnRleHQubWxydW5faGFuZGxlcihjb250ZXh0LCBldmVudCkK - code_origin: '' origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKCmZyb20gYWJjIGltcG9ydCBBQkMKZnJvbSBpbXBvcnRsaWIgaW1wb3J0IGltcG9ydF9tb2R1bGUKCmltcG9ydCBtbHJ1bi5zZXJ2aW5nCmZyb20gdHJhbnNmb3JtZXJzIGltcG9ydCBwaXBlbGluZQoKUEFDS0FHRV9NT0RVTEUgPSAidHJhbnNmb3JtZXJzIgpTRVJJQUxJWkFCTEVfVFlQRVMgPSBbZGljdCwgbGlzdCwgdHVwbGUsIHN0ciwgaW50LCBmbG9hdF0KCgpjbGFzcyBIdWdnaW5nRmFjZU1vZGVsU2VydmVyKG1scnVuLnNlcnZpbmcuVjJNb2RlbFNlcnZlciwgQUJDKToKICAgICIiIgogICAgSHVnZ2luZyBGYWNlIE1vZGVsIHNlcnZpbmcgY2xhc3MsIGluaGVyaXRpbmcgdGhlIFYyTW9kZWxTZXJ2ZXIgY2xhc3MgZm9yIGJlaW5nIGluaXRpYWxpemVkIGF1dG9tYXRpY2FsbHkgYnkgdGhlCiAgICBtb2RlbCBzZXJ2ZXIgYW5kIGJlIGFibGUgdG8gcnVuIGxvY2FsbHkgYXMgcGFydCBvZiBhIG51Y2xpbyBzZXJ2ZXJsZXNzIGZ1bmN0aW9uLCBvciBhcyBwYXJ0IG9mIGEgcmVhbC10aW1lIHBpcGVsaW5lLgogICAgIiIiCgogICAgZGVmIF9faW5pdF9fKAogICAgICAgIHNlbGYsCiAgICAgICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICAgICAgbmFtZTogc3RyLAogICAgICAgIHRhc2s6IHN0ciwKICAgICAgICBtb2RlbF9wYXRoOiBzdHIgPSBOb25lLAogICAgICAgIG1vZGVsX25hbWU6IHN0ciA9IE5vbmUsCiAgICAgICAgbW9kZWxfY2xhc3M6IHN0ciA9IE5vbmUsCiAgICAgICAgdG9rZW5pemVyX25hbWU6IHN0ciA9IE5vbmUsCiAgICAgICAgdG9rZW5pemVyX2NsYXNzOiBzdHIgPSBOb25lLAogICAgICAgIGZyYW1ld29yazogc3RyID0gTm9uZSwKICAgICAgICAqKmNsYXNzX2FyZ3MsCiAgICApOgogICAgICAgICIiIgogICAgICAgIEluaXRpYWxpemUgYSBzZXJ2aW5nIGNsYXNzIGZvciBhIEh1Z2dpbmcgZmFjZSBtb2RlbC4KCiAgICAgICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgVGhlIG1scnVuIGNvbnRleHQgdG8gd29yayB3aXRoCiAgICAgICAgOnBhcmFtIG5hbWU6ICAgICAgICAgICAgVGhlIG5hbWUgb2YgdGhpcyBzZXJ2ZXIgdG8gYmUgaW5pdGlhbGl6ZWQKICAgICAgICA6cGFyYW0gbW9kZWxfcGF0aDogICAgICBOb3QgaW4gdXNlLiBXaGVuIGFkZGluZyBhIG1vZGVsIHBhc3MgYW55IHN0cmluZyB2YWx1ZQogICAgICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgICAgIFRoZSBtb2RlbCdzIG5hbWUgaW4gdGhlIEh1Z2dpbmcgRmFjZSBodWIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBlLmcuLCBgbmxwdG93bi9iZXJ0LWJhc2UtbXVsdGlsaW5ndWFsLXVuY2FzZWQtc2VudGltZW50YAogICAgICAgIDpwYXJhbSBtb2RlbF9jbGFzczogICAgIFRoZSBtb2RlbCdzIGNsYXNzIHR5cGUgb2JqZWN0IHdoaWNoIGNhbiBiZSBwYXNzZWQgYXMgdGhlIGNsYXNzJ3MgbmFtZSAoc3RyaW5nKS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBNdXN0IGJlIHByb3ZpZGVkIGFuZCB0byBiZSBtYXRjaGVkIHdpdGggYG1vZGVsX25hbWVgLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGUuZy4sIGBBdXRvTW9kZWxGb3JTZXF1ZW5jZUNsYXNzaWZpY2F0aW9uYAogICAgICAgIDpwYXJhbSB0b2tlbml6ZXJfbmFtZTogIFRoZSB0b2tlbml6ZXIncyBuYW1lIGluIHRoZSBIdWdnaW5nIEZhY2UgaHViCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZS5nLiwgYG5scHRvd24vYmVydC1iYXNlLW11bHRpbGluZ3VhbC11bmNhc2VkLXNlbnRpbWVudGAKICAgICAgICA6cGFyYW0gdG9rZW5pemVyX2NsYXNzOiBUaGUgbW9kZWwncyBjbGFzcyB0eXBlIG9iamVjdCB3aGljaCBjYW4gYmUgcGFzc2VkIGFzIHRoZSBjbGFzcydzIG5hbWUgKHN0cmluZykuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTXVzdCBiZSBwcm92aWRlZCBhbmQgdG8gYmUgbWF0Y2hlZCB3aXRoIGBtb2RlbF9uYW1lYC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBlLmcuLCBgQXV0b1Rva2VuaXplcmAKICAgICAgICA6cGFyYW0gZnJhbWV3b3JrOiAgICAgICBUaGUgZnJhbWV3b3JrIHRvIHVzZSwgZWl0aGVyIGAicHQiYCBmb3IgUHlUb3JjaCBvciBgInRmImAgZm9yIFRlbnNvckZsb3cuIFRoZSBzcGVjaWZpZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmcmFtZXdvcmsgbXVzdCBiZSBpbnN0YWxsZWQuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgbm8gZnJhbWV3b3JrIGlzIHNwZWNpZmllZCwgd2lsbCBkZWZhdWx0IHRvIHRoZSBvbmUgY3VycmVudGx5IGluc3RhbGxlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBubyBmcmFtZXdvcmsgaXMgc3BlY2lmaWVkIGFuZCBib3RoIGZyYW1ld29ya3MgYXJlIGluc3RhbGxlZCwgd2lsbCBkZWZhdWx0IHRvIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZyYW1ld29yayBvZiB0aGUgYG1vZGVsYCwgb3IgdG8gUHlUb3JjaCBpZiBubyBtb2RlbCBpcyBwcm92aWRlZC4KICAgICAgICA6cGFyYW0gY2xhc3NfYXJnczogICAgICAtCiAgICAgICAgIiIiCiAgICAgICAgc3VwZXIoSHVnZ2luZ0ZhY2VNb2RlbFNlcnZlciwgc2VsZikuX19pbml0X18oCiAgICAgICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICAgICAgbmFtZT1uYW1lLAogICAgICAgICAgICBtb2RlbF9wYXRoPW1vZGVsX3BhdGgsCiAgICAgICAgICAgICoqY2xhc3NfYXJncywKICAgICAgICApCiAgICAgICAgc2VsZi50YXNrID0gdGFzawogICAgICAgIHNlbGYubW9kZWwgPSBOb25lCiAgICAgICAgc2VsZi50b2tlbml6ZXIgPSBOb25lCiAgICAgICAgc2VsZi5tb2RlbF9uYW1lID0gbW9kZWxfbmFtZQogICAgICAgIHNlbGYudG9rZW5pemVyX25hbWUgPSB0b2tlbml6ZXJfbmFtZQogICAgICAgIHNlbGYubW9kZWxfY2xhc3MgPSBtb2RlbF9jbGFzcwogICAgICAgIHNlbGYudG9rZW5pemVyX2NsYXNzID0gdG9rZW5pemVyX2NsYXNzCiAgICAgICAgc2VsZi5mcmFtZXdvcmsgPSBmcmFtZXdvcmsKICAgICAgICBzZWxmLnBpcGUgPSBOb25lCgogICAgZGVmIGxvYWQoc2VsZik6CiAgICAgICAgIiIibG9hZCBhbmQgaW5pdGlhbGl6ZSB0aGUgbW9kZWwgYW5kL29yIG90aGVyIGVsZW1lbnRzIiIiCiAgICAgICAgaWYgc2VsZi5tb2RlbF9jbGFzczoKICAgICAgICAgICAgbW9kZWxfb2JqZWN0ID0gZ2V0YXR0cihpbXBvcnRfbW9kdWxlKFBBQ0tBR0VfTU9EVUxFKSwgc2VsZi5tb2RlbF9jbGFzcykKICAgICAgICAgICAgc2VsZi5tb2RlbCA9IG1vZGVsX29iamVjdC5mcm9tX3ByZXRyYWluZWQoc2VsZi5tb2RlbF9uYW1lKQogICAgICAgIGlmIHNlbGYudG9rZW5pemVyX2NsYXNzOgogICAgICAgICAgICB0b2tlbml6ZXJfb2JqZWN0ID0gZ2V0YXR0cigKICAgICAgICAgICAgICAgIGltcG9ydF9tb2R1bGUoUEFDS0FHRV9NT0RVTEUpLCBzZWxmLnRva2VuaXplcl9jbGFzcwogICAgICAgICAgICApCiAgICAgICAgICAgIHNlbGYudG9rZW5pemVyID0gdG9rZW5pemVyX29iamVjdC5mcm9tX3ByZXRyYWluZWQoc2VsZi50b2tlbml6ZXJfbmFtZSkKICAgICAgICBzZWxmLnBpcGUgPSBwaXBlbGluZSgKICAgICAgICAgICAgdGFzaz1zZWxmLnRhc2ssCiAgICAgICAgICAgIG1vZGVsPXNlbGYubW9kZWwgb3Igc2VsZi5tb2RlbF9uYW1lLAogICAgICAgICAgICB0b2tlbml6ZXI9c2VsZi50b2tlbml6ZXIsCiAgICAgICAgICAgIGZyYW1ld29yaz1zZWxmLmZyYW1ld29yaywKICAgICAgICApCgogICAgZGVmIHByZWRpY3Qoc2VsZiwgYm9keTogZGljdCkgLT4gbGlzdDoKICAgICAgICAiIiJHZW5lcmF0ZSBtb2RlbCBwcmVkaWN0aW9ucyBmcm9tIHNhbXBsZS4iIiIKICAgICAgICBpZiBzZWxmLnBpcGUgaXMgTm9uZToKICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigiUGxlYXNlIHVzZSBgLmxvYWQoKWAiKQogICAgICAgIHRyeToKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShib2R5WyJpbnB1dHMiXVswXSwgZGljdCk6CiAgICAgICAgICAgICAgICByZXN1bHQgPSBbc2VsZi5waXBlKCoqX2lucHV0KSBmb3IgX2lucHV0IGluIGJvZHlbImlucHV0cyJdXQogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgcmVzdWx0ID0gc2VsZi5waXBlKGJvZHlbImlucHV0cyJdKQogICAgICAgICAgICAjIHJlcGxhY2UgbGlzdCBvZiBsaXN0cyBvZiBkaWN0cyBpbnRvIGEgbGlzdCBvZiBkaWN0czoKICAgICAgICAgICAgaWYgYWxsKGlzaW5zdGFuY2UocmVzLCBsaXN0KSBmb3IgcmVzIGluIHJlc3VsdCk6CiAgICAgICAgICAgICAgICBuZXdfcmVzdWx0ID0gW3Jlc1swXSBmb3IgcmVzIGluIHJlc3VsdF0KICAgICAgICAgICAgICAgIHJlc3VsdCA9IG5ld19yZXN1bHQKCiAgICAgICAgICAgIG5vbl9zZXJpYWxpemFibGVfdHlwZXMgPSBbXQogICAgICAgICAgICBmb3IgcmVzIGluIHJlc3VsdDoKICAgICAgICAgICAgICAgIGZvciBrZXksIHZhbCBpbiByZXMuaXRlbXMoKToKICAgICAgICAgICAgICAgICAgICBpZiB0eXBlKHZhbCkgbm90IGluIFNFUklBTElaQUJMRV9UWVBFUzoKICAgICAgICAgICAgICAgICAgICAgICAgbm9uX3NlcmlhbGl6YWJsZV90eXBlcy5hcHBlbmQoc3RyKHR5cGUodmFsKSkpCiAgICAgICAgICAgICAgICAgICAgICAgIHJlc1trZXldID0gc3RyKHZhbCkKICAgICAgICAgICAgaWYgbm9uX3NlcmlhbGl6YWJsZV90eXBlczoKICAgICAgICAgICAgICAgIHNlbGYuY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICBmIk5vbi1zZXJpYWxpemFibGUgdHlwZXM6IHtub25fc2VyaWFsaXphYmxlX3R5cGVzfSB3ZXJlIGNhc3RlZCB0byBzdHJpbmdzIgogICAgICAgICAgICAgICAgKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICAgICAgcmFpc2UgRXhjZXB0aW9uKCJGYWlsZWQgdG8gcHJlZGljdCAlcyIgJSBlKQogICAgICAgIHJldHVybiByZXN1bHQKCmZyb20gbWxydW4ucnVudGltZXMgaW1wb3J0IG51Y2xpb19pbml0X2hvb2sKZGVmIGluaXRfY29udGV4dChjb250ZXh0KToKICAgIG51Y2xpb19pbml0X2hvb2soY29udGV4dCwgZ2xvYmFscygpLCAnc2VydmluZ192MicpCgpkZWYgaGFuZGxlcihjb250ZXh0LCBldmVudCk6CiAgICByZXR1cm4gY29udGV4dC5tbHJ1bl9oYW5kbGVyKGNvbnRleHQsIGV2ZW50KQo= requirements: - transformers==4.21.3 - tensorflow==2.9.2 - function_kind: serving_v2 + code_origin: '' + filename: hugging_face_serving.py default_class: HuggingFaceModelServer - base_image_pull: false - max_replicas: 4 + min_replicas: 1 command: '' - disable_auto_mount: false - function_handler: hugging-face-serving-nuclio:handler + default_handler: '' + source: '' + max_replicas: 4 + base_image_pull: false description: Generic Hugging Face model server. + function_kind: serving_v2 + function_handler: hugging-face-serving-nuclio:handler env: - name: MLRUN_HTTPDB__NUCLIO__EXPLICIT_ACK value: enabled -verbose: false -kind: serving diff --git a/functions/src/hugging_face_serving/hugging_face_serving.py b/functions/src/hugging_face_serving/hugging_face_serving.py index 06dc4207f..31ef144d1 100644 --- a/functions/src/hugging_face_serving/hugging_face_serving.py +++ b/functions/src/hugging_face_serving/hugging_face_serving.py @@ -15,11 +15,9 @@ from abc import ABC from importlib import import_module -from typing import List - -from transformers import pipeline import mlrun.serving +from transformers import pipeline PACKAGE_MODULE = "transformers" SERIALIZABLE_TYPES = [dict, list, tuple, str, int, float] @@ -100,7 +98,7 @@ def load(self): framework=self.framework, ) - def predict(self, body: dict) -> List: + def predict(self, body: dict) -> list: """Generate model predictions from sample.""" if self.pipe is None: raise ValueError("Please use `.load()`") diff --git a/functions/src/hugging_face_serving/test_hugging_face_serving.py b/functions/src/hugging_face_serving/test_hugging_face_serving.py index 6fdc02dd3..da1c68ec9 100644 --- a/functions/src/hugging_face_serving/test_hugging_face_serving.py +++ b/functions/src/hugging_face_serving/test_hugging_face_serving.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import mlrun import numpy as np import pytest -import mlrun - CLASS_NAME = "HuggingFaceModelServer" PIPELINES = [ @@ -81,7 +80,7 @@ def test_default_models(pipeline): ) server = serving_function.to_mock_server() result = server.test( - f'/v2/models/{pipeline["task"]}', body={"inputs": [pipeline["example"]]} + f"/v2/models/{pipeline['task']}", body={"inputs": [pipeline["example"]]} ) prediction = result["outputs"][0] assert all( @@ -90,7 +89,6 @@ def test_default_models(pipeline): def test_local_model_serving(): - serving_function = mlrun.import_function("function.yaml") # Adding model: diff --git a/functions/src/load_dataset/function.yaml b/functions/src/load_dataset/function.yaml index 91775a802..5fb3ca19f 100644 --- a/functions/src/load_dataset/function.yaml +++ b/functions/src/load_dataset/function.yaml @@ -1,40 +1,22 @@ -kind: job metadata: - name: load-dataset tag: '' - hash: d05aa41d618533335eeaeab38aa434a14e3e3980 - project: '' - labels: - author: Iguazio - framework: sklearn + name: load-dataset categories: - data-preparation +verbose: false +kind: job spec: - command: '' - args: [] image: mlrun/mlrun + disable_auto_mount: false build: + origin_filename: '' functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IG51bXB5IGFzIG5wCmltcG9ydCBwYW5kYXMgYXMgcGQKZnJvbSBtbHJ1bi5leGVjdXRpb24gaW1wb3J0IE1MQ2xpZW50Q3R4CgoKZGVmIGxvYWRfZGF0YXNldCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogc3RyLAogICAgbmFtZTogc3RyID0gIiIsCiAgICBmaWxlX2V4dDogc3RyID0gInBhcnF1ZXQiLAogICAgcGFyYW1zOiBkaWN0ID0ge30sCikgLT4gTm9uZToKICAgICIiIkxvYWRzIGEgc2Npa2l0LWxlYXJuIHRveSBkYXRhc2V0IGZvciBjbGFzc2lmaWNhdGlvbiBvciByZWdyZXNzaW9uCgogICAgVGhlIGZvbGxvd2luZyBkYXRhc2V0cyBhcmUgYXZhaWxhYmxlICgnbmFtZScgOiBkZXNyaXB0aW9uKToKCiAgICAgICAgJ2Jvc3RvbicgICAgICAgICAgOiBib3N0b24gaG91c2UtcHJpY2VzIGRhdGFzZXQgKHJlZ3Jlc3Npb24pCiAgICAgICAgJ2lyaXMnICAgICAgICAgICAgOiBpcmlzIGRhdGFzZXQgKGNsYXNzaWZpY2F0aW9uKQogICAgICAgICdkaWFiZXRlcycgICAgICAgIDogZGlhYmV0ZXMgZGF0YXNldCAocmVncmVzc2lvbikKICAgICAgICAnZGlnaXRzJyAgICAgICAgICA6IGRpZ2l0cyBkYXRhc2V0IChjbGFzc2lmaWNhdGlvbikKICAgICAgICAnbGlubmVydWQnICAgICAgICA6IGxpbm5lcnVkIGRhdGFzZXQgKG11bHRpdmFyaWF0ZSByZWdyZXNzaW9uKQogICAgICAgICd3aW5lJyAgICAgICAgICAgIDogd2luZSBkYXRhc2V0IChjbGFzc2lmaWNhdGlvbikKICAgICAgICAnYnJlYXN0X2NhbmNlcicgICA6IGJyZWFzdCBjYW5jZXIgd2lzY29uc2luIGRhdGFzZXQgKGNsYXNzaWZpY2F0aW9uKQoKICAgIFRoZSBzY2lraXQtbGVhcm4gZnVuY3Rpb25zIHJldHVybiBhIGRhdGEgYnVuY2ggaW5jbHVkaW5nIHRoZSBmb2xsb3dpbmcgaXRlbXM6CiAgICAtIGRhdGEgICAgICAgICAgICAgIHRoZSBmZWF0dXJlcyBtYXRyaXgKICAgIC0gdGFyZ2V0ICAgICAgICAgICAgdGhlIGdyb3VuZCB0cnV0aCBsYWJlbHMKICAgIC0gREVTQ1IgICAgICAgICAgICAgYSBkZXNjcmlwdGlvbiBvZiB0aGUgZGF0YXNldAogICAgLSBmZWF0dXJlX25hbWVzICAgICBoZWFkZXIgZm9yIGRhdGEKCiAgICBUaGUgZmVhdHVyZXMgKGFuZCB0aGVpciBuYW1lcykgYXJlIHN0b3JlZCB3aXRoIHRoZSB0YXJnZXQgbGFiZWxzIGluIGEgRGF0YUZyYW1lLgoKICAgIEZvciBmdXJ0aGVyIGRldGFpbHMgc2VlIGh0dHBzOi8vc2Npa2l0LWxlYXJuLm9yZy9zdGFibGUvZGF0YXNldHMvaW5kZXguaHRtbCN0b3ktZGF0YXNldHMKCiAgICA6cGFyYW0gY29udGV4dDogICAgZnVuY3Rpb24gZXhlY3V0aW9uIGNvbnRleHQKICAgIDpwYXJhbSBkYXRhc2V0OiAgICBuYW1lIG9mIHRoZSBkYXRhc2V0IHRvIGxvYWQKICAgIDpwYXJhbSBuYW1lOiAgICAgICBhcnRpZmFjdCBuYW1lIChkZWZhdWx0cyB0byBkYXRhc2V0KQogICAgOnBhcmFtIGZpbGVfZXh0OiAgIG91dHB1dCBmaWxlX2V4dDogcGFycXVldCBvciBjc3YKICAgIDpwYXJhbSBwYXJhbXM6ICAgICBwYXJhbXMgb2YgdGhlIHNrbGVhcm4gbG9hZF9kYXRhIG1ldGhvZAogICAgIiIiCiAgICBkYXRhc2V0ID0gc3RyKGRhdGFzZXQpCiAgICBwa2dfbW9kdWxlID0gInNrbGVhcm4uZGF0YXNldHMiCiAgICBmbmFtZSA9IGYibG9hZF97ZGF0YXNldH0iCgogICAgcGtnX21vZHVsZSA9IF9faW1wb3J0X18ocGtnX21vZHVsZSwgZnJvbWxpc3Q9W2ZuYW1lXSkKICAgIGxvYWRfZGF0YV9mbiA9IGdldGF0dHIocGtnX21vZHVsZSwgZm5hbWUpCgogICAgZGF0YSA9IGxvYWRfZGF0YV9mbigqKnBhcmFtcykKICAgIGZlYXR1cmVfbmFtZXMgPSBkYXRhWyJmZWF0dXJlX25hbWVzIl0KCiAgICB4eSA9IG5wLmNvbmNhdGVuYXRlKFtkYXRhWyJkYXRhIl0sIGRhdGFbInRhcmdldCJdLnJlc2hhcGUoLTEsIDEpXSwgYXhpcz0xKQogICAgaWYgaGFzYXR0cihmZWF0dXJlX25hbWVzLCAiYXBwZW5kIik6CiAgICAgICAgZmVhdHVyZV9uYW1lcy5hcHBlbmQoImxhYmVscyIpCiAgICBlbHNlOgogICAgICAgIGZlYXR1cmVfbmFtZXMgPSBucC5hcHBlbmQoZmVhdHVyZV9uYW1lcywgImxhYmVscyIpCiAgICBkZiA9IHBkLkRhdGFGcmFtZShkYXRhPXh5LCBjb2x1bW5zPWZlYXR1cmVfbmFtZXMpCgogICAgY29udGV4dC5sb2dfZGF0YXNldChuYW1lIG9yIGRhdGFzZXQsIGRmPWRmLCBmb3JtYXQ9ZmlsZV9leHQsIGluZGV4PUZhbHNlKQo= - commands: [] code_origin: '' - origin_filename: '' - requirements: [] + filename: load_dataset.py entry_points: load_dataset: - name: load_dataset - doc: "Loads a scikit-learn toy dataset for classification or regression\n\n\ - The following datasets are available ('name' : desription):\n\n 'boston'\ - \ : boston house-prices dataset (regression)\n 'iris' \ - \ : iris dataset (classification)\n 'diabetes' : diabetes dataset\ - \ (regression)\n 'digits' : digits dataset (classification)\n\ - \ 'linnerud' : linnerud dataset (multivariate regression)\n 'wine'\ - \ : wine dataset (classification)\n 'breast_cancer' : breast\ - \ cancer wisconsin dataset (classification)\n\nThe scikit-learn functions\ - \ return a data bunch including the following items:\n- data \ - \ the features matrix\n- target the ground truth labels\n- DESCR\ - \ a description of the dataset\n- feature_names header for\ - \ data\n\nThe features (and their names) are stored with the target labels\ - \ in a DataFrame.\n\nFor further details see https://scikit-learn.org/stable/datasets/index.html#toy-datasets" + outputs: + - type: None parameters: - name: context type: MLClientCtx @@ -54,19 +36,23 @@ spec: type: dict doc: params of the sklearn load_data method default: {} - outputs: - - type: None - lineno: 20 - has_varargs: false + name: load_dataset + doc: "Loads a scikit-learn toy dataset for classification or regression\n\n\ + The following datasets are available ('name' : desription):\n\n 'boston'\ + \ : boston house-prices dataset (regression)\n 'iris' \ + \ : iris dataset (classification)\n 'diabetes' : diabetes dataset\ + \ (regression)\n 'digits' : digits dataset (classification)\n\ + \ 'linnerud' : linnerud dataset (multivariate regression)\n 'wine'\ + \ : wine dataset (classification)\n 'breast_cancer' : breast\ + \ cancer wisconsin dataset (classification)\n\nThe scikit-learn functions\ + \ return a data bunch including the following items:\n- data \ + \ the features matrix\n- target the ground truth labels\n- DESCR\ + \ a description of the dataset\n- feature_names header for\ + \ data\n\nThe features (and their names) are stored with the target labels\ + \ in a DataFrame.\n\nFor further details see https://scikit-learn.org/stable/datasets/index.html#toy-datasets" has_kwargs: false + has_varargs: false + lineno: 20 + command: '' description: load a toy dataset from scikit-learn default_handler: load_dataset - disable_auto_mount: false - clone_target_dir: '' - env: [] - priority_class_name: '' - preemption_mode: prevent - affinity: null - tolerations: null - security_context: {} -verbose: false diff --git a/functions/src/mlflow_utils/function.yaml b/functions/src/mlflow_utils/function.yaml index 623f054fb..96d04602d 100644 --- a/functions/src/mlflow_utils/function.yaml +++ b/functions/src/mlflow_utils/function.yaml @@ -1,32 +1,33 @@ +metadata: + tag: '' + name: mlflow-utils + categories: + - model-serving + - utils verbose: false +kind: serving spec: - command: '' - source: '' - default_class: MLFlowModelServer - function_kind: serving_v2 + image: mlrun/mlrun + disable_auto_mount: false build: - functionSourceCode: aW1wb3J0IHppcGZpbGUKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgRGljdAppbXBvcnQgbWxmbG93CmZyb20gbWxydW4uc2VydmluZy52Ml9zZXJ2aW5nIGltcG9ydCBWMk1vZGVsU2VydmVyCmltcG9ydCBwYW5kYXMgYXMgcGQKCgpjbGFzcyBNTEZsb3dNb2RlbFNlcnZlcihWMk1vZGVsU2VydmVyKToKICAgICIiIgogICAgTUxGbG93IHRyYWNrZXIgTW9kZWwgc2VydmluZyBjbGFzcywgaW5oZXJpdGluZyB0aGUgVjJNb2RlbFNlcnZlciBjbGFzcyBmb3IgYmVpbmcgaW5pdGlhbGl6ZWQgYXV0b21hdGljYWxseSBieSB0aGUgbW9kZWwKICAgIHNlcnZlciBhbmQgYmUgYWJsZSB0byBydW4gbG9jYWxseSBhcyBwYXJ0IG9mIGEgbnVjbGlvIHNlcnZlcmxlc3MgZnVuY3Rpb24sIG9yIGFzIHBhcnQgb2YgYSByZWFsLXRpbWUgcGlwZWxpbmUuCiAgICAiIiIKCiAgICBkZWYgbG9hZChzZWxmKToKICAgICAgICAiIiIKICAgICAgICBsb2FkcyBhIG1vZGVsIHRoYXQgd2FzIGxvZ2dlZCBieSB0aGUgTUxGbG93IHRyYWNrZXIgbW9kZWwKICAgICAgICAiIiIKICAgICAgICAjIFVuemlwIHRoZSBtb2RlbCBkaXIgYW5kIHRoZW4gdXNlIG1sZmxvdydzIGxvYWQgZnVuY3Rpb24KICAgICAgICBtb2RlbF9maWxlLCBfID0gc2VsZi5nZXRfbW9kZWwoIi56aXAiKQogICAgICAgIG1vZGVsX3BhdGhfdW56aXAgPSBtb2RlbF9maWxlLnJlcGxhY2UoIi56aXAiLCAiIikKCiAgICAgICAgd2l0aCB6aXBmaWxlLlppcEZpbGUobW9kZWxfZmlsZSwgInIiKSBhcyB6aXBfcmVmOgogICAgICAgICAgICB6aXBfcmVmLmV4dHJhY3RhbGwobW9kZWxfcGF0aF91bnppcCkKCiAgICAgICAgc2VsZi5tb2RlbCA9IG1sZmxvdy5weWZ1bmMubG9hZF9tb2RlbChtb2RlbF9wYXRoX3VuemlwKQoKICAgIGRlZiBwcmVkaWN0KHNlbGYsIHJlcXVlc3Q6IERpY3Rbc3RyLCBBbnldKSAtPiBsaXN0OgogICAgICAgICIiIgogICAgICAgIEluZmVyIHRoZSBpbnB1dHMgdGhyb3VnaCB0aGUgbW9kZWwuIFRoZSBpbmZlcnJlZCBkYXRhIHdpbGwKICAgICAgICBiZSByZWFkIGZyb20gdGhlICJpbnB1dHMiIGtleSBvZiB0aGUgcmVxdWVzdC4KCiAgICAgICAgOnBhcmFtIHJlcXVlc3Q6IFRoZSByZXF1ZXN0IHRvIHRoZSBtb2RlbCB1c2luZyB4Z2Jvb3N0J3MgcHJlZGljdC4KICAgICAgICAgICAgICAgIFRoZSBpbnB1dCB0byB0aGUgbW9kZWwgd2lsbCBiZSByZWFkIGZyb20gdGhlICJpbnB1dHMiIGtleS4KCiAgICAgICAgOnJldHVybjogVGhlIG1vZGVsJ3MgcHJlZGljdGlvbiBvbiB0aGUgZ2l2ZW4gaW5wdXQuCiAgICAgICAgIiIiCgogICAgICAgICMgR2V0IHRoZSBpbnB1dHMgYW5kIHNldCB0byBhY2NlcHRlZCB0eXBlOgogICAgICAgIGlucHV0cyA9IHBkLkRhdGFGcmFtZShyZXF1ZXN0WyJpbnB1dHMiXSkKCiAgICAgICAgIyBQcmVkaWN0IHVzaW5nIHRoZSBtb2RlbCdzIHByZWRpY3QgZnVuY3Rpb246CiAgICAgICAgcHJlZGljdGlvbnMgPSBzZWxmLm1vZGVsLnByZWRpY3QoaW5wdXRzKQoKICAgICAgICAjIFJldHVybiBhcyBsaXN0OgogICAgICAgIHJldHVybiBwcmVkaWN0aW9ucy50b2xpc3QoKQoKZnJvbSBtbHJ1bi5ydW50aW1lcyBpbXBvcnQgbnVjbGlvX2luaXRfaG9vawpkZWYgaW5pdF9jb250ZXh0KGNvbnRleHQpOgogICAgbnVjbGlvX2luaXRfaG9vayhjb250ZXh0LCBnbG9iYWxzKCksICdzZXJ2aW5nX3YyJykKCmRlZiBoYW5kbGVyKGNvbnRleHQsIGV2ZW50KToKICAgIHJldHVybiBjb250ZXh0Lm1scnVuX2hhbmRsZXIoY29udGV4dCwgZXZlbnQpCg== + origin_filename: '' + functionSourceCode: aW1wb3J0IHppcGZpbGUKZnJvbSB0eXBpbmcgaW1wb3J0IEFueQoKaW1wb3J0IG1sZmxvdwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4uc2VydmluZy52Ml9zZXJ2aW5nIGltcG9ydCBWMk1vZGVsU2VydmVyCgoKY2xhc3MgTUxGbG93TW9kZWxTZXJ2ZXIoVjJNb2RlbFNlcnZlcik6CiAgICAiIiIKICAgIE1MRmxvdyB0cmFja2VyIE1vZGVsIHNlcnZpbmcgY2xhc3MsIGluaGVyaXRpbmcgdGhlIFYyTW9kZWxTZXJ2ZXIgY2xhc3MgZm9yIGJlaW5nIGluaXRpYWxpemVkIGF1dG9tYXRpY2FsbHkgYnkgdGhlIG1vZGVsCiAgICBzZXJ2ZXIgYW5kIGJlIGFibGUgdG8gcnVuIGxvY2FsbHkgYXMgcGFydCBvZiBhIG51Y2xpbyBzZXJ2ZXJsZXNzIGZ1bmN0aW9uLCBvciBhcyBwYXJ0IG9mIGEgcmVhbC10aW1lIHBpcGVsaW5lLgogICAgIiIiCgogICAgZGVmIGxvYWQoc2VsZik6CiAgICAgICAgIiIiCiAgICAgICAgbG9hZHMgYSBtb2RlbCB0aGF0IHdhcyBsb2dnZWQgYnkgdGhlIE1MRmxvdyB0cmFja2VyIG1vZGVsCiAgICAgICAgIiIiCiAgICAgICAgIyBVbnppcCB0aGUgbW9kZWwgZGlyIGFuZCB0aGVuIHVzZSBtbGZsb3cncyBsb2FkIGZ1bmN0aW9uCiAgICAgICAgbW9kZWxfZmlsZSwgXyA9IHNlbGYuZ2V0X21vZGVsKCIuemlwIikKICAgICAgICBtb2RlbF9wYXRoX3VuemlwID0gbW9kZWxfZmlsZS5yZXBsYWNlKCIuemlwIiwgIiIpCgogICAgICAgIHdpdGggemlwZmlsZS5aaXBGaWxlKG1vZGVsX2ZpbGUsICJyIikgYXMgemlwX3JlZjoKICAgICAgICAgICAgemlwX3JlZi5leHRyYWN0YWxsKG1vZGVsX3BhdGhfdW56aXApCgogICAgICAgIHNlbGYubW9kZWwgPSBtbGZsb3cucHlmdW5jLmxvYWRfbW9kZWwobW9kZWxfcGF0aF91bnppcCkKCiAgICBkZWYgcHJlZGljdChzZWxmLCByZXF1ZXN0OiBkaWN0W3N0ciwgQW55XSkgLT4gbGlzdDoKICAgICAgICAiIiIKICAgICAgICBJbmZlciB0aGUgaW5wdXRzIHRocm91Z2ggdGhlIG1vZGVsLiBUaGUgaW5mZXJyZWQgZGF0YSB3aWxsCiAgICAgICAgYmUgcmVhZCBmcm9tIHRoZSAiaW5wdXRzIiBrZXkgb2YgdGhlIHJlcXVlc3QuCgogICAgICAgIDpwYXJhbSByZXF1ZXN0OiBUaGUgcmVxdWVzdCB0byB0aGUgbW9kZWwgdXNpbmcgeGdib29zdCdzIHByZWRpY3QuCiAgICAgICAgICAgICAgICBUaGUgaW5wdXQgdG8gdGhlIG1vZGVsIHdpbGwgYmUgcmVhZCBmcm9tIHRoZSAiaW5wdXRzIiBrZXkuCgogICAgICAgIDpyZXR1cm46IFRoZSBtb2RlbCdzIHByZWRpY3Rpb24gb24gdGhlIGdpdmVuIGlucHV0LgogICAgICAgICIiIgoKICAgICAgICAjIEdldCB0aGUgaW5wdXRzIGFuZCBzZXQgdG8gYWNjZXB0ZWQgdHlwZToKICAgICAgICBpbnB1dHMgPSBwZC5EYXRhRnJhbWUocmVxdWVzdFsiaW5wdXRzIl0pCgogICAgICAgICMgUHJlZGljdCB1c2luZyB0aGUgbW9kZWwncyBwcmVkaWN0IGZ1bmN0aW9uOgogICAgICAgIHByZWRpY3Rpb25zID0gc2VsZi5tb2RlbC5wcmVkaWN0KGlucHV0cykKCiAgICAgICAgIyBSZXR1cm4gYXMgbGlzdDoKICAgICAgICByZXR1cm4gcHJlZGljdGlvbnMudG9saXN0KCkKCmZyb20gbWxydW4ucnVudGltZXMgaW1wb3J0IG51Y2xpb19pbml0X2hvb2sKZGVmIGluaXRfY29udGV4dChjb250ZXh0KToKICAgIG51Y2xpb19pbml0X2hvb2soY29udGV4dCwgZ2xvYmFscygpLCAnc2VydmluZ192MicpCgpkZWYgaGFuZGxlcihjb250ZXh0LCBldmVudCk6CiAgICByZXR1cm4gY29udGV4dC5tbHJ1bl9oYW5kbGVyKGNvbnRleHQsIGV2ZW50KQo= requirements: - - mlflow==2.12.2 + - mlflow~=2.22 - lightgbm - xgboost code_origin: '' - origin_filename: '' - image: mlrun/mlrun - base_image_pull: false + filename: mlflow_utils.py + default_class: MLFlowModelServer + min_replicas: 1 + command: '' default_handler: '' + source: '' max_replicas: 4 - disable_auto_mount: false - min_replicas: 1 + base_image_pull: false description: Mlflow model server, and additional utils. + function_kind: serving_v2 function_handler: mlflow-utils-nuclio:handler env: - name: MLRUN_HTTPDB__NUCLIO__EXPLICIT_ACK value: enabled -metadata: - categories: - - model-serving - - utils - name: mlflow-utils - tag: '' -kind: serving diff --git a/functions/src/mlflow_utils/mlflow_utils.py b/functions/src/mlflow_utils/mlflow_utils.py index fb6124bef..cbcc78381 100644 --- a/functions/src/mlflow_utils/mlflow_utils.py +++ b/functions/src/mlflow_utils/mlflow_utils.py @@ -1,8 +1,9 @@ import zipfile -from typing import Any, Dict +from typing import Any + import mlflow -from mlrun.serving.v2_serving import V2ModelServer import pandas as pd +from mlrun.serving.v2_serving import V2ModelServer class MLFlowModelServer(V2ModelServer): @@ -24,7 +25,7 @@ def load(self): self.model = mlflow.pyfunc.load_model(model_path_unzip) - def predict(self, request: Dict[str, Any]) -> list: + def predict(self, request: dict[str, Any]) -> list: """ Infer the inputs through the model. The inferred data will be read from the "inputs" key of the request. diff --git a/functions/src/mlflow_utils/test_mlflow_utils.py b/functions/src/mlflow_utils/test_mlflow_utils.py index 70d6ce03f..74dcefdbc 100644 --- a/functions/src/mlflow_utils/test_mlflow_utils.py +++ b/functions/src/mlflow_utils/test_mlflow_utils.py @@ -12,23 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os import tempfile import lightgbm as lgb import mlflow import mlflow.environment_variables import mlflow.xgboost + +# os.environ["MLRUN_IGNORE_ENV_FILE"] = "True" #TODO remove before push +import mlrun +import mlrun.launcher.local import pytest import xgboost as xgb from sklearn import datasets from sklearn.metrics import accuracy_score, log_loss from sklearn.model_selection import train_test_split -import os -# os.environ["MLRUN_IGNORE_ENV_FILE"] = "True" #TODO remove before push - -import mlrun -import mlrun.launcher.local # Important: # unlike mlconf which resets back to default after each test run, the mlflow configurations # and env vars don't, so at the end of each test we need to redo anything we set in that test. @@ -36,6 +36,7 @@ # name (last two using mlconf), failing run mid-way, and a run with no handler. # we also test here importing of runs, artifacts and models from a previous run. + # simple mlflow example of lgb logging def lgb_run(): # prepare train and test data @@ -170,10 +171,10 @@ def test_track_run_with_experiment_name(handler): server = serving_func.to_mock_server() # An example taken randomly - result = server.test(f"/v2/models/{model_name}/predict", {"inputs": [[5.1, 3.5, 1.4, 0.2]]}) + result = server.test( + f"/v2/models/{model_name}/predict", {"inputs": [[5.1, 3.5, 1.4, 0.2]]} + ) print(result) assert result # unset mlflow experiment name to default mlflow.environment_variables.MLFLOW_EXPERIMENT_NAME.unset() - - diff --git a/functions/src/model_server/function.yaml b/functions/src/model_server/function.yaml index 83e80823d..20f85bf67 100644 --- a/functions/src/model_server/function.yaml +++ b/functions/src/model_server/function.yaml @@ -1,27 +1,28 @@ -kind: remote +metadata: + tag: '' + name: model-server + categories: + - model-serving + - machine-learning verbose: false +kind: remote spec: + image: mlrun/mlrun disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgppbXBvcnQgbWxydW4KZnJvbSBjbG91ZHBpY2tsZSBpbXBvcnQgbG9hZAoKd2FybmluZ3MuZmlsdGVyd2FybmluZ3MoImlnbm9yZSIpCgppbXBvcnQgbnVtcHkgYXMgbnAKCgpjbGFzcyBDbGFzc2lmaWVyTW9kZWwobWxydW4ucnVudGltZXMuTUxNb2RlbFNlcnZlcik6CiAgICBkZWYgbG9hZChzZWxmKToKICAgICAgICAiIiJMb2FkIG1vZGVsIGZyb20gc3RvcmFnZS4iIiIKICAgICAgICBtb2RlbF9maWxlLCBleHRyYV9kYXRhID0gc2VsZi5nZXRfbW9kZWwoIi5wa2wiKQogICAgICAgIHNlbGYubW9kZWwgPSBsb2FkKG9wZW4obW9kZWxfZmlsZSwgInJiIikpCgogICAgZGVmIHByZWRpY3Qoc2VsZiwgYm9keTogZGljdCkgLT4gbGlzdDoKICAgICAgICAiIiJHZW5lcmF0ZSBtb2RlbCBwcmVkaWN0aW9ucyBmcm9tIHNhbXBsZS4KCiAgICAgICAgOnBhcmFtIGJvZHkgOiBBIGRpY3Qgb2Ygb2JzZXJ2YXRpb25zLCBlYWNoIG9mIHdoaWNoIGlzIGFuIDEtZGltZW5zaW9uYWwgZmVhdHVyZSB2ZWN0b3IuCgogICAgICAgIFJldHVybnMgbW9kZWwgcHJlZGljdGlvbnMgYXMgYSBgTGlzdGAsIG9uZSBmb3IgZWFjaCByb3cgaW4gdGhlIGBib2R5YCBpbnB1dCBgTGlzdGAuCiAgICAgICAgIiIiCiAgICAgICAgdHJ5OgogICAgICAgICAgICBmZWF0cyA9IG5wLmFzYXJyYXkoYm9keVsiaW5zdGFuY2VzIl0pCiAgICAgICAgICAgIHJlc3VsdDogbnAubmRhcnJheSA9IHNlbGYubW9kZWwucHJlZGljdChmZWF0cykKICAgICAgICAgICAgcmVzcCA9IHJlc3VsdC50b2xpc3QoKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICAgICAgcmFpc2UgRXhjZXB0aW9uKGYiRmFpbGVkIHRvIHByZWRpY3Qge2V9IikKCiAgICAgICAgcmV0dXJuIHJlc3AKCmZyb20gbWxydW4ucnVudGltZXMgaW1wb3J0IG51Y2xpb19pbml0X2hvb2sKZGVmIGluaXRfY29udGV4dChjb250ZXh0KToKICAgIG51Y2xpb19pbml0X2hvb2soY29udGV4dCwgZ2xvYmFscygpLCAnc2VydmluZycpCgpkZWYgaGFuZGxlcihjb250ZXh0LCBldmVudCk6CiAgICByZXR1cm4gY29udGV4dC5tbHJ1bl9oYW5kbGVyKGNvbnRleHQsIGV2ZW50KQo= + code_origin: '' + filename: model_server.py min_replicas: 1 - source: '' - description: generic sklearn model server + command: '' default_handler: '' + source: '' max_replicas: 4 - image: mlrun/mlrun + base_image_pull: false + description: generic sklearn model server function_kind: serving function_handler: model-server-nuclio:handler - build: - origin_filename: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IG1scnVuCgpmcm9tIGNsb3VkcGlja2xlIGltcG9ydCBsb2FkCmZyb20gdHlwaW5nIGltcG9ydCBMaXN0CmZyb20gZGF0ZXRpbWUgaW1wb3J0IGRhdGV0aW1lCmZyb20gc2tsZWFybi5kYXRhc2V0cyBpbXBvcnQgbG9hZF9pcmlzCgppbXBvcnQgd2FybmluZ3MKCndhcm5pbmdzLmZpbHRlcndhcm5pbmdzKCJpZ25vcmUiKQoKaW1wb3J0IG9zCmltcG9ydCBudW1weSBhcyBucAoKCmNsYXNzIENsYXNzaWZpZXJNb2RlbChtbHJ1bi5ydW50aW1lcy5NTE1vZGVsU2VydmVyKToKICAgIGRlZiBsb2FkKHNlbGYpOgogICAgICAgICIiIkxvYWQgbW9kZWwgZnJvbSBzdG9yYWdlLiIiIgogICAgICAgIG1vZGVsX2ZpbGUsIGV4dHJhX2RhdGEgPSBzZWxmLmdldF9tb2RlbCgiLnBrbCIpCiAgICAgICAgc2VsZi5tb2RlbCA9IGxvYWQob3Blbihtb2RlbF9maWxlLCAicmIiKSkKCiAgICBkZWYgcHJlZGljdChzZWxmLCBib2R5OiBkaWN0KSAtPiBMaXN0OgogICAgICAgICIiIkdlbmVyYXRlIG1vZGVsIHByZWRpY3Rpb25zIGZyb20gc2FtcGxlLgoKICAgICAgICA6cGFyYW0gYm9keSA6IEEgZGljdCBvZiBvYnNlcnZhdGlvbnMsIGVhY2ggb2Ygd2hpY2ggaXMgYW4gMS1kaW1lbnNpb25hbCBmZWF0dXJlIHZlY3Rvci4KCiAgICAgICAgUmV0dXJucyBtb2RlbCBwcmVkaWN0aW9ucyBhcyBhIGBMaXN0YCwgb25lIGZvciBlYWNoIHJvdyBpbiB0aGUgYGJvZHlgIGlucHV0IGBMaXN0YC4KICAgICAgICAiIiIKICAgICAgICB0cnk6CiAgICAgICAgICAgIGZlYXRzID0gbnAuYXNhcnJheShib2R5WyJpbnN0YW5jZXMiXSkKICAgICAgICAgICAgcmVzdWx0OiBucC5uZGFycmF5ID0gc2VsZi5tb2RlbC5wcmVkaWN0KGZlYXRzKQogICAgICAgICAgICByZXNwID0gcmVzdWx0LnRvbGlzdCgpCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgICAgICByYWlzZSBFeGNlcHRpb24oZiJGYWlsZWQgdG8gcHJlZGljdCB7ZX0iKQoKICAgICAgICByZXR1cm4gcmVzcAoKZnJvbSBtbHJ1bi5ydW50aW1lcyBpbXBvcnQgbnVjbGlvX2luaXRfaG9vawpkZWYgaW5pdF9jb250ZXh0KGNvbnRleHQpOgogICAgbnVjbGlvX2luaXRfaG9vayhjb250ZXh0LCBnbG9iYWxzKCksICdzZXJ2aW5nJykKCmRlZiBoYW5kbGVyKGNvbnRleHQsIGV2ZW50KToKICAgIHJldHVybiBjb250ZXh0Lm1scnVuX2hhbmRsZXIoY29udGV4dCwgZXZlbnQpCg== - code_origin: '' - base_image_pull: false - command: '' env: - name: MLRUN_HTTPDB__NUCLIO__EXPLICIT_ACK value: enabled -metadata: - categories: - - model-serving - - machine-learning - name: model-server - tag: '' diff --git a/functions/src/model_server/model_server.py b/functions/src/model_server/model_server.py index cefdff235..3227a289c 100644 --- a/functions/src/model_server/model_server.py +++ b/functions/src/model_server/model_server.py @@ -14,18 +14,13 @@ # # Generated by nuclio.export.NuclioExporter -import mlrun +import warnings +import mlrun from cloudpickle import load -from typing import List -from datetime import datetime -from sklearn.datasets import load_iris - -import warnings warnings.filterwarnings("ignore") -import os import numpy as np @@ -35,7 +30,7 @@ def load(self): model_file, extra_data = self.get_model(".pkl") self.model = load(open(model_file, "rb")) - def predict(self, body: dict) -> List: + def predict(self, body: dict) -> list: """Generate model predictions from sample. :param body : A dict of observations, each of which is an 1-dimensional feature vector. diff --git a/functions/src/model_server/test_model_server.py b/functions/src/model_server/test_model_server.py index a11726bc7..a930ab736 100644 --- a/functions/src/model_server/test_model_server.py +++ b/functions/src/model_server/test_model_server.py @@ -12,38 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import os +import pickle + +from model_server import ClassifierModel from sklearn.datasets import load_iris -from sklearn.model_selection import train_test_split -from sklearn.preprocessing import StandardScaler from sklearn.linear_model import LogisticRegression from sklearn.metrics import accuracy_score -from model_server import ClassifierModel -import pickle -import mlrun -import os -import requests -import json +from sklearn.model_selection import train_test_split +from sklearn.preprocessing import StandardScaler + def gen_model(): # Getting the data - X,y = load_iris(return_X_y=True) - X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=123) + X, y = load_iris(return_X_y=True) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.2, random_state=123 + ) # transforming the data sc = StandardScaler() X_train = sc.fit_transform(X_train) X_test = sc.transform(X_test) # Getting the model and training it - classifier = LogisticRegression(random_state = 0, solver='lbfgs', multi_class='auto') + classifier = LogisticRegression(random_state=0, solver="lbfgs", multi_class="auto") classifier.fit(X_train, y_train) # saving the model - filename = os.getcwd()+'/model.pkl' - pickle.dump(classifier, open(filename, 'wb')) - return X_test,y_test + filename = os.getcwd() + "/model.pkl" + pickle.dump(classifier, open(filename, "wb")) + return X_test, y_test + def test_remote_model_server(): - x,y = gen_model() - my_class = ClassifierModel('iris',model_dir=os.getcwd()) + x, y = gen_model() + my_class = ClassifierModel("iris", model_dir=os.getcwd()) my_class.load() - my_dict = {'instances':x.tolist()} + my_dict = {"instances": x.tolist()} preds = my_class.predict(my_dict) - assert(accuracy_score(y,preds) > 0.8) + assert accuracy_score(y, preds) > 0.8 diff --git a/functions/src/model_server_tester/function.yaml b/functions/src/model_server_tester/function.yaml index 45934c444..ae176c1e6 100644 --- a/functions/src/model_server_tester/function.yaml +++ b/functions/src/model_server_tester/function.yaml @@ -1,35 +1,29 @@ -kind: job metadata: - name: model-server-tester tag: '' - hash: 3b203a2799e44992539eafd32a4b8979bbcc8001 - project: '' - labels: - author: Iguazio + name: model-server-tester categories: - monitoring - model-serving +verbose: false +kind: job spec: - command: '' - args: [] image: mlrun/mlrun - env: [] - default_handler: model_server_tester + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IGpzb24KZnJvbSBkYXRldGltZSBpbXBvcnQgZGF0ZXRpbWUKCmltcG9ydCBudW1weSBhcyBucAppbXBvcnQgcmVxdWVzdHMKZnJvbSBtbHJ1bi5hcnRpZmFjdHMgaW1wb3J0IENoYXJ0QXJ0aWZhY3QKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCgoKZGVmIG1vZGVsX3NlcnZlcl90ZXN0ZXIoCiAgICBjb250ZXh0LAogICAgdGFibGU6IERhdGFJdGVtLAogICAgYWRkcjogc3RyLAogICAgbGFiZWxfY29sdW1uOiBzdHIgPSAibGFiZWwiLAogICAgbW9kZWw6IHN0ciA9ICIiLAogICAgbWF0Y2hfZXJyOiBib29sID0gRmFsc2UsCiAgICByb3dzOiBpbnQgPSAyMCwKKToKICAgICIiIlRlc3QgYSBtb2RlbCBzZXJ2ZXIKCiAgICA6cGFyYW0gdGFibGU6ICAgICAgICAgY3N2L3BhcnF1ZXQgdGFibGUgd2l0aCB0ZXN0IGRhdGEKICAgIDpwYXJhbSBhZGRyOiAgICAgICAgICBmdW5jdGlvbiBhZGRyZXNzL3VybAogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogIG5hbWUgb2YgdGhlIGxhYmVsIGNvbHVtbiBpbiB0YWJsZQogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgIHRlc3RlZCBtb2RlbCBuYW1lCiAgICA6cGFyYW0gbWF0Y2hfZXJyOiAgICAgcmFpc2UgZXJyb3Igb24gdmFsaWRhdGlvbiAocmVxdWlyZSBwcm9wZXIgdGVzdCBzZXQpCiAgICA6cGFyYW0gcm93czogICAgICAgICAgbnVtYmVyIG9mIHJvd3MgdG8gdXNlIGZyb20gdGVzdCBzZXQKICAgICIiIgoKICAgIHRhYmxlID0gdGFibGUuYXNfZGYoKQoKICAgIHlfbGlzdCA9IHRhYmxlLnBvcChsYWJlbF9jb2x1bW4pLnZhbHVlcy50b2xpc3QoKQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInRlc3Rpbmcgd2l0aCBkYXRhc2V0IGFnYWluc3Qge2FkZHJ9LCBtb2RlbDoge21vZGVsfSIpCiAgICBpZiByb3dzIGFuZCByb3dzIDwgdGFibGUuc2hhcGVbMF06CiAgICAgICAgdGFibGUgPSB0YWJsZS5zYW1wbGUocm93cykKCiAgICBjb3VudCA9IGVycl9jb3VudCA9IG1hdGNoID0gMAogICAgdGltZXMgPSBbXQogICAgZm9yIHgsIHkgaW4gemlwKHRhYmxlLnZhbHVlcywgeV9saXN0KToKICAgICAgICBjb3VudCArPSAxCiAgICAgICAgZXZlbnRfZGF0YSA9IGpzb24uZHVtcHMoeyJpbnN0YW5jZXMiOiBbeC50b2xpc3QoKV19KQogICAgICAgIGhhZF9lcnIgPSBGYWxzZQogICAgICAgIHRyeToKICAgICAgICAgICAgc3RhcnQgPSBkYXRldGltZS5ub3coKQogICAgICAgICAgICByZXNwID0gcmVxdWVzdHMucHV0KGYie2FkZHJ9L3ttb2RlbH0vcHJlZGljdCIsIGpzb249ZXZlbnRfZGF0YSkKICAgICAgICAgICAgaWYgbm90IHJlc3Aub2s6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcihmImJhZCBmdW5jdGlvbiByZXNwISFcbntyZXNwLnRleHR9IikKICAgICAgICAgICAgICAgIGVycl9jb3VudCArPSAxCiAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICB0aW1lcy5hcHBlbmQoKGRhdGV0aW1lLm5vdygpIC0gc3RhcnQpLm1pY3Jvc2Vjb25kcykKCiAgICAgICAgZXhjZXB0IE9TRXJyb3IgYXMgZXJyOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcihmImVycm9yIGluIHJlcXVlc3QsIGRhdGE6e2V2ZW50X2RhdGF9LCBlcnJvcjoge2Vycn0iKQogICAgICAgICAgICBlcnJfY291bnQgKz0gMQogICAgICAgICAgICBjb250aW51ZQoKICAgICAgICB5X3Jlc3AgPSByZXNwLmpzb24oKVswXQogICAgICAgIGlmIHkgPT0geV9yZXNwOgogICAgICAgICAgICBtYXRjaCArPSAxCgogICAgY29udGV4dC5sb2dfcmVzdWx0KCJ0b3RhbF90ZXN0cyIsIGNvdW50KQogICAgY29udGV4dC5sb2dfcmVzdWx0KCJlcnJvcnMiLCBlcnJfY291bnQpCiAgICBjb250ZXh0LmxvZ19yZXN1bHQoIm1hdGNoIiwgbWF0Y2gpCiAgICBpZiBjb3VudCAtIGVycl9jb3VudCA+IDA6CiAgICAgICAgdGltZXNfYXJyID0gbnAuYXJyYXkodGltZXMpCiAgICAgICAgY29udGV4dC5sb2dfcmVzdWx0KCJhdmdfbGF0ZW5jeSIsIGludChucC5tZWFuKHRpbWVzX2FycikpKQogICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdCgibWluX2xhdGVuY3kiLCBpbnQobnAuYW1pbih0aW1lc19hcnIpKSkKICAgICAgICBjb250ZXh0LmxvZ19yZXN1bHQoIm1heF9sYXRlbmN5IiwgaW50KG5wLmFtYXgodGltZXNfYXJyKSkpCgogICAgICAgIGNoYXJ0ID0gQ2hhcnRBcnRpZmFjdCgibGF0ZW5jeSIsIGhlYWRlcj1bIlRlc3QiLCAiTGF0ZW5jeSAobWljcm9zZWMpIl0pCiAgICAgICAgZm9yIGkgaW4gcmFuZ2UobGVuKHRpbWVzKSk6CiAgICAgICAgICAgIGNoYXJ0LmFkZF9yb3coW2kgKyAxLCBpbnQodGltZXNbaV0pXSkKICAgICAgICBjb250ZXh0LmxvZ19hcnRpZmFjdChjaGFydCkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgIGYicnVuIHtjb3VudH0gdGVzdHMsIHtlcnJfY291bnR9IGVycm9ycyBhbmQge21hdGNofSBtYXRjaCBleHBlY3RlZCB2YWx1ZSIKICAgICkKCiAgICBpZiBlcnJfY291bnQ6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcihmImZhaWxlZCBvbiB7ZXJyX2NvdW50fSB0ZXN0cyBvZiB7Y291bnR9IikKCiAgICBpZiBtYXRjaF9lcnIgYW5kIG1hdGNoICE9IGNvdW50OgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoZiJvbmx5IHttYXRjaH0gcmVzdWx0cyBtYXRjaCBvdXQgb2Yge2NvdW50fSIpCg== + code_origin: '' + filename: model_server_tester.py entry_points: model_server_tester: - name: model_server_tester - doc: Test a model server parameters: - name: context - default: '' - name: table type: DataItem doc: csv/parquet table with test data - default: '' - name: addr type: str doc: function address/url - default: '' - name: label_column type: str doc: name of the label column in table @@ -46,13 +40,11 @@ spec: type: int doc: number of rows to use from test set default: 20 - outputs: - - default: '' - lineno: 14 + name: model_server_tester + doc: Test a model server + has_kwargs: false + has_varargs: false + lineno: 26 + command: '' description: test model servers - build: - functionSourceCode: IyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IG9zCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IHJlcXVlc3RzCmltcG9ydCBqc29uCmltcG9ydCBudW1weSBhcyBucAoKZnJvbSBkYXRldGltZSBpbXBvcnQgZGF0ZXRpbWUKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uYXJ0aWZhY3RzIGltcG9ydCBnZXRfbW9kZWwsIENoYXJ0QXJ0aWZhY3QKCgpkZWYgbW9kZWxfc2VydmVyX3Rlc3RlcigKICAgIGNvbnRleHQsCiAgICB0YWJsZTogRGF0YUl0ZW0sCiAgICBhZGRyOiBzdHIsCiAgICBsYWJlbF9jb2x1bW46IHN0ciA9ICJsYWJlbCIsCiAgICBtb2RlbDogc3RyID0gIiIsCiAgICBtYXRjaF9lcnI6IGJvb2wgPSBGYWxzZSwKICAgIHJvd3M6IGludCA9IDIwLAopOgogICAgIiIiVGVzdCBhIG1vZGVsIHNlcnZlcgoKICAgIDpwYXJhbSB0YWJsZTogICAgICAgICBjc3YvcGFycXVldCB0YWJsZSB3aXRoIHRlc3QgZGF0YQogICAgOnBhcmFtIGFkZHI6ICAgICAgICAgIGZ1bmN0aW9uIGFkZHJlc3MvdXJsCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uOiAgbmFtZSBvZiB0aGUgbGFiZWwgY29sdW1uIGluIHRhYmxlCiAgICA6cGFyYW0gbW9kZWw6ICAgICAgICAgdGVzdGVkIG1vZGVsIG5hbWUKICAgIDpwYXJhbSBtYXRjaF9lcnI6ICAgICByYWlzZSBlcnJvciBvbiB2YWxpZGF0aW9uIChyZXF1aXJlIHByb3BlciB0ZXN0IHNldCkKICAgIDpwYXJhbSByb3dzOiAgICAgICAgICBudW1iZXIgb2Ygcm93cyB0byB1c2UgZnJvbSB0ZXN0IHNldAogICAgIiIiCgogICAgdGFibGUgPSB0YWJsZS5hc19kZigpCgogICAgeV9saXN0ID0gdGFibGUucG9wKGxhYmVsX2NvbHVtbikudmFsdWVzLnRvbGlzdCgpCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidGVzdGluZyB3aXRoIGRhdGFzZXQgYWdhaW5zdCB7YWRkcn0sIG1vZGVsOiB7bW9kZWx9IikKICAgIGlmIHJvd3MgYW5kIHJvd3MgPCB0YWJsZS5zaGFwZVswXToKICAgICAgICB0YWJsZSA9IHRhYmxlLnNhbXBsZShyb3dzKQoKICAgIGNvdW50ID0gZXJyX2NvdW50ID0gbWF0Y2ggPSAwCiAgICB0aW1lcyA9IFtdCiAgICBmb3IgeCwgeSBpbiB6aXAodGFibGUudmFsdWVzLCB5X2xpc3QpOgogICAgICAgIGNvdW50ICs9IDEKICAgICAgICBldmVudF9kYXRhID0ganNvbi5kdW1wcyh7Imluc3RhbmNlcyI6IFt4LnRvbGlzdCgpXX0pCiAgICAgICAgaGFkX2VyciA9IEZhbHNlCiAgICAgICAgdHJ5OgogICAgICAgICAgICBzdGFydCA9IGRhdGV0aW1lLm5vdygpCiAgICAgICAgICAgIHJlc3AgPSByZXF1ZXN0cy5wdXQoZiJ7YWRkcn0ve21vZGVsfS9wcmVkaWN0IiwganNvbj1ldmVudF9kYXRhKQogICAgICAgICAgICBpZiBub3QgcmVzcC5vazoKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKGYiYmFkIGZ1bmN0aW9uIHJlc3AhIVxue3Jlc3AudGV4dH0iKQogICAgICAgICAgICAgICAgZXJyX2NvdW50ICs9IDEKICAgICAgICAgICAgICAgIGNvbnRpbnVlCiAgICAgICAgICAgIHRpbWVzLmFwcGVuZCgoZGF0ZXRpbWUubm93KCkgLSBzdGFydCkubWljcm9zZWNvbmRzKQoKICAgICAgICBleGNlcHQgT1NFcnJvciBhcyBlcnI6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKGYiZXJyb3IgaW4gcmVxdWVzdCwgZGF0YTp7ZXZlbnRfZGF0YX0sIGVycm9yOiB7ZXJyfSIpCiAgICAgICAgICAgIGVycl9jb3VudCArPSAxCiAgICAgICAgICAgIGNvbnRpbnVlCgogICAgICAgIHlfcmVzcCA9IHJlc3AuanNvbigpWzBdCiAgICAgICAgaWYgeSA9PSB5X3Jlc3A6CiAgICAgICAgICAgIG1hdGNoICs9IDEKCiAgICBjb250ZXh0LmxvZ19yZXN1bHQoInRvdGFsX3Rlc3RzIiwgY291bnQpCiAgICBjb250ZXh0LmxvZ19yZXN1bHQoImVycm9ycyIsIGVycl9jb3VudCkKICAgIGNvbnRleHQubG9nX3Jlc3VsdCgibWF0Y2giLCBtYXRjaCkKICAgIGlmIGNvdW50IC0gZXJyX2NvdW50ID4gMDoKICAgICAgICB0aW1lc19hcnIgPSBucC5hcnJheSh0aW1lcykKICAgICAgICBjb250ZXh0LmxvZ19yZXN1bHQoImF2Z19sYXRlbmN5IiwgaW50KG5wLm1lYW4odGltZXNfYXJyKSkpCiAgICAgICAgY29udGV4dC5sb2dfcmVzdWx0KCJtaW5fbGF0ZW5jeSIsIGludChucC5hbWluKHRpbWVzX2FycikpKQogICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdCgibWF4X2xhdGVuY3kiLCBpbnQobnAuYW1heCh0aW1lc19hcnIpKSkKCiAgICAgICAgY2hhcnQgPSBDaGFydEFydGlmYWN0KCJsYXRlbmN5IiwgaGVhZGVyPVsiVGVzdCIsICJMYXRlbmN5IChtaWNyb3NlYykiXSkKICAgICAgICBmb3IgaSBpbiByYW5nZShsZW4odGltZXMpKToKICAgICAgICAgICAgY2hhcnQuYWRkX3JvdyhbaSArIDEsIGludCh0aW1lc1tpXSldKQogICAgICAgIGNvbnRleHQubG9nX2FydGlmYWN0KGNoYXJ0KQoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgZiJydW4ge2NvdW50fSB0ZXN0cywge2Vycl9jb3VudH0gZXJyb3JzIGFuZCB7bWF0Y2h9IG1hdGNoIGV4cGVjdGVkIHZhbHVlIgogICAgKQoKICAgIGlmIGVycl9jb3VudDoKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKGYiZmFpbGVkIG9uIHtlcnJfY291bnR9IHRlc3RzIG9mIHtjb3VudH0iKQoKICAgIGlmIG1hdGNoX2VyciBhbmQgbWF0Y2ggIT0gY291bnQ6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcihmIm9ubHkge21hdGNofSByZXN1bHRzIG1hdGNoIG91dCBvZiB7Y291bnR9IikK - commands: [] - code_origin: https://github.com/daniels290813/functions.git#55a79c32be5d233cc11efcf40cd3edbe309bfdef:/home/kali/functions/model_server_tester/model_server_tester.py - affinity: null -verbose: false + default_handler: model_server_tester diff --git a/functions/src/model_server_tester/model_server_tester.py b/functions/src/model_server_tester/model_server_tester.py index 7d83b148d..922030d11 100644 --- a/functions/src/model_server_tester/model_server_tester.py +++ b/functions/src/model_server_tester/model_server_tester.py @@ -14,15 +14,13 @@ # # Generated by nuclio.export.NuclioExporter -import os -import pandas as pd -import requests import json -import numpy as np - from datetime import datetime + +import numpy as np +import requests +from mlrun.artifacts import ChartArtifact from mlrun.datastore import DataItem -from mlrun.artifacts import get_model, ChartArtifact def model_server_tester( diff --git a/functions/src/noise_reduction/function.yaml b/functions/src/noise_reduction/function.yaml index d6d33b8da..e9d494506 100644 --- a/functions/src/noise_reduction/function.yaml +++ b/functions/src/noise_reduction/function.yaml @@ -1,21 +1,27 @@ +metadata: + tag: '' + name: noise-reduction + categories: + - data-preparation + - audio +verbose: false +kind: job spec: + image: '' + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: aW1wb3J0IGxvZ2dpbmcKZnJvbSBhYmMgaW1wb3J0IEFCQ01ldGEsIGFic3RyYWN0bWV0aG9kCmZyb20gbXVsdGlwcm9jZXNzaW5nIGltcG9ydCBQcm9jZXNzLCBRdWV1ZQpmcm9tIHBhdGhsaWIgaW1wb3J0IFBhdGgKCmltcG9ydCBsaWJyb3NhCmltcG9ydCBudW1weSBhcyBucAppbXBvcnQgdG9yY2gKZnJvbSBzY2lweS5pbyBpbXBvcnQgd2F2ZmlsZQpmcm9tIHRxZG0gaW1wb3J0IHRxZG0KCiM6IFRoZSB2YWx1ZSB0byBzZW5kIGludG8gbXVsdGlwcm9jZXNzaW5nIHF1ZXVlcyB0byBzdG9wIHRoZSBwcm9jZXNzOgpfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSyA9ICJTVE9QIgoKIyBHZXQgdGhlIGdsb2JhbCBsb2dnZXI6CnRyeToKICAgIGltcG9ydCBtbHJ1bgoKICAgIF9MT0dHRVIgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eCgibm9pc2VfcmVkdWNlIikubG9nZ2VyCmV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgX0xPR0dFUiA9IGxvZ2dpbmcuZ2V0TG9nZ2VyKCkKCgpjbGFzcyBSZWR1Y2VOb2lzZUJhc2UobWV0YWNsYXNzPUFCQ01ldGEpOgogICAgIiIiCiAgICBCYXNlIGNsYXNzIGZvciBub2lzZSByZWR1Y3Rpb24uCiAgICBUaGlzIGNsYXNzIGlzIGFpbWVkIHRvIGJlIGluaGVyaXRlZCBieSBzcGVjaWZpYyBub2lzZSByZWR1Y3Rpb24gYWxnb3JpdGhtcy4KICAgIFlvdSBtdXN0IGltcGxlbWVudCB0aGUgZm9sbG93aW5nIG1ldGhvZHM6CiAgICAtIGNsZWFuX2F1ZGlvOiAgVGhlIG1ldGhvZCB0byBjbGVhbiB0aGUgYXVkaW8sIHdoZXJlIHRoZSBub2lzZSByZWR1Y3Rpb24gYWxnb3JpdGhtIGlzIGltcGxlbWVudGVkLgogICAgLSBzYXZlX2F1ZGlvOiAgIFRoZSBtZXRob2QgdG8gc2F2ZSB0aGUgYXVkaW8gdG8gYSBmaWxlLgogICAgLSBsb2FkX2F1ZGlvOiAgIFRoZSBtZXRob2QgdG8gbG9hZCB0aGUgYXVkaW8gZnJvbSBhIGZpbGUuCgogICAgQWZ0ZXIgaW1wbGVtZW50aW5nIHRoZSBhYm92ZSBtZXRob2RzLCB5b3UgY2FuIHVzZSB0aGUgcmVkdWNlX25vaXNlIG1ldGhvZCB0byByZWR1Y2Ugbm9pc2UgZnJvbSBhdWRpbyBmaWxlcy4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgIHRhcmdldF9kaXJlY3Rvcnk6IFBhdGgsCiAgICAgICAgdmVyYm9zZTogYm9vbCA9IFRydWUsCiAgICAgICAgc2lsZW5jZV90aHJlc2hvbGQ6IGZsb2F0ID0gTm9uZSwKICAgICk6CiAgICAgICAgc2VsZi50YXJnZXRfZGlyZWN0b3J5ID0gUGF0aCh0YXJnZXRfZGlyZWN0b3J5KQogICAgICAgIHNlbGYudmVyYm9zZSA9IHZlcmJvc2UKICAgICAgICBzZWxmLnNpbGVuY2VfdGhyZXNob2xkID0gc2lsZW5jZV90aHJlc2hvbGQKCiAgICBkZWYgcmVkdWNlX25vaXNlKHNlbGYsIGF1ZGlvX2ZpbGU6IFBhdGgpIC0+IHR1cGxlW2Jvb2wsIHR1cGxlW3N0ciwgc3RyXV06CiAgICAgICAgIiIiCiAgICAgICAgUmVkdWNlIG5vaXNlIGZyb20gdGhlIGdpdmVuIGF1ZGlvIGZpbGUuCgogICAgICAgIDpwYXJhbSBhdWRpb19maWxlOiAgVGhlIGF1ZGlvIGZpbGUgdG8gcmVkdWNlIG5vaXNlIGZyb20uCgogICAgICAgIDpyZXR1cm5zOiBBIHR1cGxlIG9mOgogICAgICAgICAtIGEgYm9vbGVhbiBpbmRpY2F0aW5nIHdoZXRoZXIgYW4gZXJyb3Igb2NjdXJyZWQKICAgICAgICAgLSBhIHR1cGxlIG9mOgogICAgICAgICAgICAtIGF1ZGlvIGZpbGUgbmFtZQogICAgICAgICAgICAtIHRhcmdldCBwYXRoIGluIGNhc2Ugb2Ygc3VjY2VzcyAvIGVycm9yIG1lc3NhZ2UgaW4gY2FzZSBvZiBmYWlsdXJlLgogICAgICAgICIiIgogICAgICAgIHRyeToKICAgICAgICAgICAgaWYgc2VsZi52ZXJib3NlOgogICAgICAgICAgICAgICAgX0xPR0dFUi5pbmZvKGYiUmVkdWNpbmcgbm9pc2UgZnJvbSB7YXVkaW9fZmlsZS5uYW1lfS4iKQoKICAgICAgICAgICAgIyBMb2FkIGF1ZGlvIGRhdGE6CiAgICAgICAgICAgIGF1ZGlvID0gc2VsZi5sb2FkX2F1ZGlvKGZpbGU9c3RyKGF1ZGlvX2ZpbGUpKQoKICAgICAgICAgICAgIyBQZXJmb3JtIG5vaXNlIHJlZHVjdGlvbjoKICAgICAgICAgICAgcmVkdWNlZF9ub2lzZSA9IHNlbGYuY2xlYW5fYXVkaW8oZGF0YT1hdWRpbykKCiAgICAgICAgICAgICMgUmVtb3ZlIHNpbGVuY2UgZnJvbSB0aGUgYXVkaW8gaWYgbmVjZXNzYXJ5OgogICAgICAgICAgICByZWR1Y2VkX25vaXNlID0gc2VsZi5yZW1vdmVfc2lsZW5jZShhdWRpbz1yZWR1Y2VkX25vaXNlKQoKICAgICAgICAgICAgIyBQcmVwYXJlIHRhcmdldCBwYXRoOgogICAgICAgICAgICB0YXJnZXRfcGF0aCA9IHNlbGYudXBkYXRlX3RvX3dhdl9zdWZmaXgoYXVkaW9fZmlsZT1hdWRpb19maWxlKQoKICAgICAgICAgICAgIyBTYXZlIGZpbGU6CiAgICAgICAgICAgIHNlbGYuc2F2ZV9hdWRpbygKICAgICAgICAgICAgICAgIGF1ZGlvPXJlZHVjZWRfbm9pc2UsCiAgICAgICAgICAgICAgICB0YXJnZXRfcGF0aD10YXJnZXRfcGF0aCwKICAgICAgICAgICAgKQoKICAgICAgICAgICAgaWYgc2VsZi52ZXJib3NlOgogICAgICAgICAgICAgICAgX0xPR0dFUi5pbmZvKGYiU2F2ZWQgY2xlYW5lZCBhdWRpbyBmaWxlIHRvIHt0YXJnZXRfcGF0aH0uIikKCiAgICAgICAgICAgIHJldHVybiBGYWxzZSwgKGF1ZGlvX2ZpbGUubmFtZSwgc3RyKHRhcmdldF9wYXRoKSkKICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGV4Y2VwdGlvbjoKICAgICAgICAgICAgaWYgc2VsZi52ZXJib3NlOgogICAgICAgICAgICAgICAgX0xPR0dFUi5lcnJvcihmIkZhaWxlZCB0byByZWR1Y2Ugbm9pc2UgZnJvbSB7YXVkaW9fZmlsZS5uYW1lfS4iKQogICAgICAgICAgICAgICAgX0xPR0dFUi5lcnJvcihmIkVycm9yOiB7ZXhjZXB0aW9ufSIpCiAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgZXJyb3I6CiAgICAgICAgICAgIHJldHVybiBUcnVlLCAoYXVkaW9fZmlsZS5uYW1lLCBzdHIoZXhjZXB0aW9uKSkKCiAgICBAYWJzdHJhY3RtZXRob2QKICAgIGRlZiBjbGVhbl9hdWRpbyhzZWxmLCBkYXRhKSAtPiBucC5uZGFycmF5IHwgdG9yY2guVGVuc29yOgogICAgICAgICIiIgogICAgICAgIENsZWFuIHRoZSBhdWRpbyBmcm9tIG5vaXNlLiBIZXJlIHlvdSBzaG91bGQgaW1wbGVtZW50IHRoZSBub2lzZSByZWR1Y3Rpb24gYWxnb3JpdGhtLgoKICAgICAgICA6cGFyYW0gZGF0YTogICAgVGhlIGF1ZGlvIGRhdGEgdG8gY2xlYW4uCgogICAgICAgIDpyZXR1cm5zOiBUaGUgY2xlYW5lZCBhdWRpby4KICAgICAgICAiIiIKICAgICAgICBwYXNzCgogICAgQGFic3RyYWN0bWV0aG9kCiAgICBkZWYgc2F2ZV9hdWRpbyhzZWxmLCBhdWRpbzogbnAubmRhcnJheSwgdGFyZ2V0X3BhdGg6IFBhdGgpOgogICAgICAgICIiIgogICAgICAgIFNhdmUgdGhlIGF1ZGlvIHRvIGEgZmlsZS4KCiAgICAgICAgOnBhcmFtIGF1ZGlvOiAgICAgICBUaGUgYXVkaW8gdG8gc2F2ZS4KICAgICAgICA6cGFyYW0gdGFyZ2V0X3BhdGg6IFRoZSB0YXJnZXQgcGF0aCB0byBzYXZlIHRoZSBhdWRpbyB0by4KICAgICAgICAiIiIKICAgICAgICBwYXNzCgogICAgQGFic3RyYWN0bWV0aG9kCiAgICBkZWYgbG9hZF9hdWRpbyhzZWxmLCBmaWxlOiBzdHIpIC0+IHR1cGxlW25wLm5kYXJyYXkgfCB0b3JjaC5UZW5zb3IsIGludF06CiAgICAgICAgIiIiCiAgICAgICAgTG9hZCB0aGUgYXVkaW8gZnJvbSBhIGZpbGUuCgogICAgICAgIDpwYXJhbSBmaWxlOiAgICBUaGUgZmlsZSB0byBsb2FkIHRoZSBhdWRpbyBmcm9tLgoKICAgICAgICA6cmV0dXJuczogQSB0dXBsZSBvZjoKICAgICAgICAgICAgLSB0aGUgYXVkaW8gZGF0YQogICAgICAgICAgICAtIHRoZSBzYW1wbGUgcmF0ZQogICAgICAgICIiIgogICAgICAgIHBhc3MKCiAgICBkZWYgdXBkYXRlX3RvX3dhdl9zdWZmaXgoc2VsZiwgYXVkaW9fZmlsZTogUGF0aCk6CiAgICAgICAgdGFyZ2V0X3BhdGggPSBzZWxmLnRhcmdldF9kaXJlY3RvcnkgLyBhdWRpb19maWxlLm5hbWUKICAgICAgICBpZiB0YXJnZXRfcGF0aC5zdWZmaXggIT0gIi53YXYiOgogICAgICAgICAgICBvbGRfc3VmZml4ID0gdGFyZ2V0X3BhdGguc3VmZml4WzE6XQogICAgICAgICAgICB0YXJnZXRfcGF0aCA9IHRhcmdldF9wYXRoLndpdGhfc3RlbSh0YXJnZXRfcGF0aC5zdGVtICsgZiJfe29sZF9zdWZmaXh9IikKICAgICAgICAgICAgcmV0dXJuIHRhcmdldF9wYXRoLndpdGhfc3VmZml4KCIud2F2IikKICAgICAgICBlbHNlOgogICAgICAgICAgICByZXR1cm4gdGFyZ2V0X3BhdGgKCiAgICBkZWYgcmVtb3ZlX3NpbGVuY2UoCiAgICAgICAgc2VsZiwKICAgICAgICBhdWRpbzogbnAubmRhcnJheSwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgUmVtb3ZlIHNpbGVuY2Ugc2VjdGlvbnMgZnJvbSB0aGUgYXVkaW8uCgogICAgICAgIDpwYXJhbSBhdWRpbzogICBUaGUgYXVkaW8gdG8gcmVtb3ZlIHNpbGVuY2UgZnJvbS4KCiAgICAgICAgOnJldHVybnM6IFRoZSBhdWRpbyB3aXRob3V0IHNpbGVuY2UuCiAgICAgICAgIiIiCiAgICAgICAgaWYgc2VsZi5zaWxlbmNlX3RocmVzaG9sZCBpcyBOb25lOgogICAgICAgICAgICByZXR1cm4gYXVkaW8KCiAgICAgICAgIyBHZXQgdGhlIGluZGljZXMgb2YgdGhlIG5vbi1zaWxlbnQgZnJhbWVzOgogICAgICAgIG5vbl9zaWxlbnRfaW5kaWNlcyA9IGxpYnJvc2EuZWZmZWN0cy5zcGxpdCgKICAgICAgICAgICAgeT1hdWRpbywKICAgICAgICAgICAgdG9wX2RiPXNlbGYuc2lsZW5jZV90aHJlc2hvbGQsCiAgICAgICAgICAgIGZyYW1lX2xlbmd0aD0yMDQ4LAogICAgICAgICAgICBob3BfbGVuZ3RoPTI1NiwKICAgICAgICApCgogICAgICAgICMgR2V0IHRoZSBub24tc2lsZW50IGF1ZGlvOgogICAgICAgIG5vbl9zaWxlbnRfYXVkaW8gPSBucC5jb25jYXRlbmF0ZSgKICAgICAgICAgICAgW2F1ZGlvWzosIHN0YXJ0OmVuZF0gZm9yIHN0YXJ0LCBlbmQgaW4gbm9uX3NpbGVudF9pbmRpY2VzXSwgYXhpcz0xCiAgICAgICAgKQoKICAgICAgICByZXR1cm4gbm9uX3NpbGVudF9hdWRpbwoKCmNsYXNzIFJlZHVjZU5vaXNlKFJlZHVjZU5vaXNlQmFzZSk6CiAgICBkZWYgX19pbml0X18oCiAgICAgICAgc2VsZiwKICAgICAgICB0YXJnZXRfZGlyZWN0b3J5OiBQYXRoLAogICAgICAgIHZlcmJvc2U6IGJvb2wgPSBUcnVlLAogICAgICAgIHNpbGVuY2VfdGhyZXNob2xkOiBmbG9hdCA9IE5vbmUsCiAgICAgICAgc2FtcGxlX3JhdGU6IGludCA9IDE2MDAwLAogICAgICAgIGR1cmF0aW9uOiBpbnQgPSBOb25lLAogICAgICAgIGNoYW5uZWw6IGludCA9IE5vbmUsCiAgICApOgogICAgICAgIHN1cGVyKCkuX19pbml0X18odGFyZ2V0X2RpcmVjdG9yeSwgdmVyYm9zZSwgc2lsZW5jZV90aHJlc2hvbGQpCiAgICAgICAgc2VsZi5zYW1wbGVfcmF0ZSA9IHNhbXBsZV9yYXRlCiAgICAgICAgc2VsZi5kdXJhdGlvbiA9IGR1cmF0aW9uCiAgICAgICAgc2VsZi5jaGFubmVsID0gY2hhbm5lbAoKICAgIGRlZiBzYXZlX2F1ZGlvKHNlbGYsIGF1ZGlvOiBucC5uZGFycmF5LCB0YXJnZXRfcGF0aDogUGF0aCk6CiAgICAgICAgIyBJZiB0aGUgYXVkaW8gaGFzIG1vcmUgdGhhbiBvbmUgY2hhbm5lbCwgdHJhbnNwb3NlIGl0IGluIG9yZGVyIHRvIHNhdmUgaXQ6CiAgICAgICAgaWYgbGVuKGF1ZGlvKSA+IDE6CiAgICAgICAgICAgIGF1ZGlvID0gYXVkaW8uVAoKICAgICAgICB3YXZmaWxlLndyaXRlKAogICAgICAgICAgICBmaWxlbmFtZT10YXJnZXRfcGF0aCwKICAgICAgICAgICAgcmF0ZT1zZWxmLnNhbXBsZV9yYXRlLAogICAgICAgICAgICBkYXRhPWF1ZGlvLAogICAgICAgICkKCiAgICBkZWYgbG9hZF9hdWRpbyhzZWxmLCBmaWxlOiBzdHIpIC0+IG5wLm5kYXJyYXk6CiAgICAgICAgZGF0YSwgc3IgPSBsaWJyb3NhLmxvYWQoCiAgICAgICAgICAgIHBhdGg9ZmlsZSwKICAgICAgICAgICAgc3I9c2VsZi5zYW1wbGVfcmF0ZSwKICAgICAgICAgICAgbW9ubz1GYWxzZSwgICMga2VlcCBjaGFubmVscyBzZXBhcmF0ZQogICAgICAgICAgICBkdXJhdGlvbj1zZWxmLmR1cmF0aW9uLAogICAgICAgICkKICAgICAgICAjIHNldCBzYW1wbGUgcmF0ZToKICAgICAgICBzZWxmLnNhbXBsZV9yYXRlID0gaW50KHNyKQoKICAgICAgICAjIGNvbnZlcnQgdG8gaW50IHdpdGggc2NhbGluZyBmb3IgMTYtYml0IGludGVnZXIKICAgICAgICBkYXRhICo9IDMyNzY3IC8gbnAubWF4KG5wLmFicyhkYXRhKSkgICMgcmUtc2NhbGluZwogICAgICAgIGRhdGEgPSBkYXRhLmFzdHlwZShucC5pbnQxNikgICMgY2hhbmdlIGRhdGEgdHlwZQoKICAgICAgICAjIHNlbGVjdCBjaGFubmVsCiAgICAgICAgZGF0YV90b19yZWR1Y2UgPSBkYXRhW3NlbGYuY2hhbm5lbF0gaWYgc2VsZi5jaGFubmVsIGlzIG5vdCBOb25lIGVsc2UgZGF0YQogICAgICAgIHJldHVybiBkYXRhX3RvX3JlZHVjZQoKICAgIGRlZiBjbGVhbl9hdWRpbyhzZWxmLCBkYXRhOiBucC5uZGFycmF5KSAtPiBucC5uZGFycmF5OgogICAgICAgIHRyeToKICAgICAgICAgICAgaW1wb3J0IG5vaXNlcmVkdWNlCiAgICAgICAgZXhjZXB0IEltcG9ydEVycm9yIGFzIGU6CiAgICAgICAgICAgIHJhaXNlIEltcG9ydEVycm9yKCJQbGVhc2UgaW5zdGFsbCBub2lzZXJlZHVjZSBwYWNrYWdlIikgZnJvbSBlCgogICAgICAgIHJlZHVjZWRfbm9pc2UgPSBub2lzZXJlZHVjZS5yZWR1Y2Vfbm9pc2UoeT1kYXRhLCBzcj1zZWxmLnNhbXBsZV9yYXRlKQoKICAgICAgICAjIGFkZCBjaGFubmVsIGJhY2sgYWZ0ZXIgbm9pc2UgcmVkdWN0aW9uCiAgICAgICAgaWYgc2VsZi5jaGFubmVsIGlzIG5vdCBOb25lOgogICAgICAgICAgICAjIHB1dHRpbmcgdGhlIGNoYW5uZWwgYmFjayBpbiB0aGUgZGF0YQogICAgICAgICAgICBkYXRhW3NlbGYuY2hhbm5lbF0gPSByZWR1Y2VkX25vaXNlCiAgICAgICAgICAgICMgdXBkYXRpbmcgdGhlIGRhdGEgdG8gc2F2ZQogICAgICAgICAgICByZWR1Y2VkX25vaXNlID0gZGF0YQoKICAgICAgICByZXR1cm4gcmVkdWNlZF9ub2lzZQoKCmNsYXNzIERGTihSZWR1Y2VOb2lzZUJhc2UpOgogICAgZGVmIF9faW5pdF9fKAogICAgICAgIHNlbGYsCiAgICAgICAgdGFyZ2V0X2RpcmVjdG9yeTogUGF0aCwKICAgICAgICB2ZXJib3NlOiBib29sID0gVHJ1ZSwKICAgICAgICBzaWxlbmNlX3RocmVzaG9sZDogZmxvYXQgPSBOb25lLAogICAgICAgIHBhZDogYm9vbCA9IFRydWUsCiAgICAgICAgYXR0ZW5fbGltX2RiOiBpbnQgPSBOb25lLAogICAgICAgICoqa3dhcmdzLAogICAgKToKICAgICAgICBzdXBlcigpLl9faW5pdF9fKHRhcmdldF9kaXJlY3RvcnksIHZlcmJvc2UsIHNpbGVuY2VfdGhyZXNob2xkKQogICAgICAgIHNlbGYucGFkID0gcGFkCiAgICAgICAgc2VsZi5hdHRlbl9saW1fZGIgPSBhdHRlbl9saW1fZGIKICAgICAgICBzZWxmLmt3YXJncyA9IGt3YXJncwoKICAgICAgICAjIGltcG9ydCByZXF1aXJlZCBwYWNrYWdlcwogICAgICAgIHRyeToKICAgICAgICAgICAgZnJvbSBkZi5lbmhhbmNlIGltcG9ydCBpbml0X2RmCiAgICAgICAgZXhjZXB0IEltcG9ydEVycm9yIGFzIGU6CiAgICAgICAgICAgIHJhaXNlIEltcG9ydEVycm9yKCJQbGVhc2UgaW5zdGFsbCBkZWVwZmlsdGVybmV0IHBhY2thZ2VzIikgZnJvbSBlCgogICAgICAgIGlmIHNlbGYudmVyYm9zZToKICAgICAgICAgICAgX0xPR0dFUi5pbmZvKCJMb2FkaW5nIERlZXBGaWx0ZXJOZXQyIG1vZGVsLiIpCgogICAgICAgICMgTG9hZCB0aGUgbW9kZWw6CiAgICAgICAgbW9kZWwsIGRmX3N0YXRlLCBfID0gaW5pdF9kZigpCiAgICAgICAgc2VsZi5tb2RlbCA9IG1vZGVsCiAgICAgICAgc2VsZi5kZl9zdGF0ZSA9IGRmX3N0YXRlCiAgICAgICAgc2VsZi5zYW1wbGVfcmF0ZSA9IHNlbGYuZGZfc3RhdGUuc3IoKQoKICAgIGRlZiBzYXZlX2F1ZGlvKHNlbGYsIGF1ZGlvOiBucC5uZGFycmF5LCB0YXJnZXRfcGF0aDogUGF0aCk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICBmcm9tIGRmLmVuaGFuY2UgaW1wb3J0IHNhdmVfYXVkaW8KICAgICAgICBleGNlcHQgSW1wb3J0RXJyb3IgYXMgZToKICAgICAgICAgICAgcmFpc2UgSW1wb3J0RXJyb3IoIlBsZWFzZSBpbnN0YWxsIGRlZXBmaWx0ZXJuZXQgcGFja2FnZSIpIGZyb20gZQogICAgICAgIHNhdmVfYXVkaW8oCiAgICAgICAgICAgIGZpbGU9dGFyZ2V0X3BhdGgubmFtZSwKICAgICAgICAgICAgYXVkaW89YXVkaW8sCiAgICAgICAgICAgIHNyPXNlbGYuc2FtcGxlX3JhdGUsCiAgICAgICAgICAgIG91dHB1dF9kaXI9c3RyKHNlbGYudGFyZ2V0X2RpcmVjdG9yeSksCiAgICAgICAgKQoKICAgIGRlZiBsb2FkX2F1ZGlvKHNlbGYsIGZpbGU6IHN0cikgLT4gdG9yY2guVGVuc29yOgogICAgICAgIHRyeToKICAgICAgICAgICAgZnJvbSBkZi5lbmhhbmNlIGltcG9ydCBsb2FkX2F1ZGlvCiAgICAgICAgZXhjZXB0IEltcG9ydEVycm9yIGFzIGU6CiAgICAgICAgICAgIHJhaXNlIEltcG9ydEVycm9yKCJQbGVhc2UgaW5zdGFsbCBkZWVwZmlsdGVybmV0IHBhY2thZ2UiKSBmcm9tIGUKICAgICAgICBhdWRpbywgXyA9IGxvYWRfYXVkaW8oZmlsZT1maWxlLCBzcj1zZWxmLnNhbXBsZV9yYXRlLCAqKnNlbGYua3dhcmdzKQogICAgICAgIHJldHVybiBhdWRpbwoKICAgIGRlZiBjbGVhbl9hdWRpbyhzZWxmLCBkYXRhOiB0b3JjaC5UZW5zb3IpIC0+IHRvcmNoLlRlbnNvcjoKICAgICAgICB0cnk6CiAgICAgICAgICAgIGZyb20gZGYuZW5oYW5jZSBpbXBvcnQgZW5oYW5jZQogICAgICAgIGV4Y2VwdCBJbXBvcnRFcnJvciBhcyBlOgogICAgICAgICAgICByYWlzZSBJbXBvcnRFcnJvcigiUGxlYXNlIGluc3RhbGwgZGVlcGZpbHRlcm5ldCBwYWNrYWdlIikgZnJvbSBlCiAgICAgICAgcmV0dXJuIGVuaGFuY2UoCiAgICAgICAgICAgIG1vZGVsPXNlbGYubW9kZWwsCiAgICAgICAgICAgIGRmX3N0YXRlPXNlbGYuZGZfc3RhdGUsCiAgICAgICAgICAgIGF1ZGlvPWRhdGEsCiAgICAgICAgICAgIHBhZD1zZWxmLnBhZCwKICAgICAgICAgICAgYXR0ZW5fbGltX2RiPXNlbGYuYXR0ZW5fbGltX2RiLAogICAgICAgICkKCgpkZWYgX211bHRpcHJvY2Vzc2luZ19jb21wbGV0ZV90YXNrcygKICAgIG5vaXNlX3JlZHVjZV90eXBlOiB0eXBlW1JlZHVjZU5vaXNlQmFzZV0sCiAgICBub2lzZV9yZWR1Y2VfYXJndW1lbnRzOiBkaWN0LAogICAgdGFza3NfcXVldWU6IFF1ZXVlLAogICAgcmVzdWx0c19xdWV1ZTogUXVldWUsCik6CiAgICAiIiIKICAgIENvbXBsZXRlIHRoZSB0YXNrcyBpbiB0aGUgZ2l2ZW4gcXVldWUgYW5kIHB1dCB0aGUgcmVzdWx0cyBpbiB0aGUgZ2l2ZW4gcmVzdWx0cyBxdWV1ZS4gVGhlIGZ1bmN0aW9uIHdpbGwgc3RvcCB3aGVuCiAgICB0aGUgZ2l2ZW4gdGFza3MgcXVldWUgd2lsbCByZWNlaXZlIHRoZSBzdG9wIG1hcmsuIEl0IGlzIGFpbWVkIHRvIGJlIHVzZWQgd2l0aCBtdWx0aXByb2Nlc3NpbmcgYXMgYSBwcm9jZXNzLgoKICAgIDpwYXJhbSBub2lzZV9yZWR1Y2VfdHlwZTogICAgICAgVGhlIG5vaXNlIHJlZHVjZSB0eXBlIHRvIHVzZS4KICAgIDpwYXJhbSBub2lzZV9yZWR1Y2VfYXJndW1lbnRzOiAgVGhlIG5vaXNlcmVkdWNlIGluaXRpYWxpemF0aW9uIGt3YXJncy4KICAgIDpwYXJhbSB0YXNrc19xdWV1ZTogICAgICAgICAgICAgQSBxdWV1ZSB0byBnZXQgdGhlIHRhc2tzIGZyb20uCiAgICA6cGFyYW0gcmVzdWx0c19xdWV1ZTogICAgICAgICAgIEEgcXVldWUgdG8gcHV0IHRoZSByZXN1bHRzIGluLgogICAgIiIiCiAgICAjIEluaXRpYWxpemUgdGhlIHJlZHVjZSBub2lzZSBvYmplY3QKICAgIG5vaXNlX3JlZHVjZXIgPSBub2lzZV9yZWR1Y2VfdHlwZSgqKm5vaXNlX3JlZHVjZV9hcmd1bWVudHMpCgogICAgIyBTdGFydCBsaXN0ZW5pbmcgdG8gdGhlIHRhc2tzIHF1ZXVlOgogICAgd2hpbGUgVHJ1ZToKICAgICAgICAjIEdldCB0aGUgYXVkaW9fZmlsZToKICAgICAgICBhdWRpb19maWxlID0gdGFza3NfcXVldWUuZ2V0KCkKICAgICAgICBpZiBhdWRpb19maWxlID09IF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLOgogICAgICAgICAgICBicmVhawogICAgICAgIGF1ZGlvX2ZpbGUgPSBQYXRoKGF1ZGlvX2ZpbGUpCiAgICAgICAgIyBBcHBseSBub2lzZSByZWR1Y3Rpb24gYW5kIGNvbGxlY3QgdGhlIHJlc3VsdDoKICAgICAgICByZXN1bHRzX3F1ZXVlLnB1dChub2lzZV9yZWR1Y2VyLnJlZHVjZV9ub2lzZShhdWRpb19maWxlPWF1ZGlvX2ZpbGUpKQoKICAgICMgTWFyayB0aGUgZW5kIG9mIHRoZSB0YXNrczoKICAgIHJlc3VsdHNfcXVldWUucHV0KF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLKQoKCmRlZiByZWR1Y2Vfbm9pc2VfZGZuKAogICAgYXVkaW9fc291cmNlOiBzdHIsCiAgICB0YXJnZXRfZGlyZWN0b3J5OiBzdHIsCiAgICBwYWQ6IGJvb2wgPSBUcnVlLAogICAgYXR0ZW5fbGltX2RiOiBpbnQgPSBOb25lLAogICAgc2lsZW5jZV90aHJlc2hvbGQ6IGZsb2F0ID0gTm9uZSwKICAgIHVzZV9tdWx0aXByb2Nlc3Npbmc6IGludCA9IDAsCiAgICB2ZXJib3NlOiBib29sID0gVHJ1ZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBSZWR1Y2Ugbm9pc2UgZnJvbSBhdWRpbyBmaWxlcyB1c2luZyBEZWVwRmlsdGVyTmV0LgogICAgRm9yIG1vcmUgaW5mb3JtYXRpb24gYWJvdXQgdGhlIG5vaXNlIHJlZHVjdGlvbiBhbGdvcml0aG0gc2VlOgogICAgaHR0cHM6Ly9naXRodWIuY29tL1Jpa29yb3NlL0RlZXBGaWx0ZXJOZXQKICAgIE5vdGljZSB0aGF0IHRoZSBzYXZlZCBmaWxlcyBhcmUgaW4gd2F2IGZvcm1hdCwgZXZlbiBpZiB0aGUgb3JpZ2luYWwgZmlsZXMgYXJlIGluIG90aGVyIGZvcm1hdC4KCiAgICA6cGFyYW0gYXVkaW9fc291cmNlOiAgICAgICAgcGF0aCB0byBhdWRpbyBmaWxlIG9yIGRpcmVjdG9yeSBvZiBhdWRpbyBmaWxlcwogICAgOnBhcmFtIHRhcmdldF9kaXJlY3Rvcnk6ICAgIHBhdGggdG8gdGFyZ2V0IGRpcmVjdG9yeSB0byBzYXZlIGNsZWFuZWQgYXVkaW8gZmlsZXMKICAgIDpwYXJhbSBwYWQ6ICAgICAgICAgICAgICAgICB3aGV0aGVyIHRvIHBhZCB0aGUgYXVkaW8gZmlsZSB3aXRoIHplcm9zIGJlZm9yZSBjbGVhbmluZwogICAgOnBhcmFtIGF0dGVuX2xpbV9kYjogICAgICAgIG1heGltdW0gYXR0ZW51YXRpb24gaW4gZEIKICAgIDpwYXJhbSBzaWxlbmNlX3RocmVzaG9sZDogICB0aGUgdGhyZXNob2xkIHRvIHJlbW92ZSBzaWxlbmNlIGZyb20gdGhlIGF1ZGlvLCBpbiBkQi4gSWYgTm9uZSwgbm8gc2lsZW5jZSByZW1vdmFsIGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcGVyZm9ybWVkLgogICAgOnBhcmFtIHVzZV9tdWx0aXByb2Nlc3Npbmc6IE51bWJlciBvZiBwcm9jZXNzZXMgdG8gdXNlIGZvciBjbGVhbmluZyB0aGUgYXVkaW8gZmlsZXMuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgMCwgbm8gbXVsdGlwcm9jZXNzaW5nIGlzIHVzZWQuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICAgdmVyYm9zaXR5IGxldmVsLiBJZiBUcnVlLCBkaXNwbGF5IHByb2dyZXNzIGJhciBhbmQgbG9ncy4KICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICBhZGRpdGlvbmFsIGFyZ3VtZW50cyB0byBwYXNzIHRvIHRvcmNoYXVkaW8ubG9hZCgpLiBGb3IgbW9yZSBpbmZvcm1hdGlvbiBzZWU6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaHR0cHM6Ly9weXRvcmNoLm9yZy9hdWRpby9zdGFibGUvZ2VuZXJhdGVkL3RvcmNoYXVkaW8ubG9hZC5odG1sCiAgICAiIiIKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJSZWR1Y2luZyBub2lzZSBmcm9tIGF1ZGlvIGZpbGVzLiIpCgogICAgIyBjcmVhdGUgdGFyZ2V0IGRpcmVjdG9yeToKICAgIHRhcmdldF9kaXJlY3RvcnkgPSBfY3JlYXRlX3RhcmdldF9kaXJlY3RvcnkodGFyZ2V0X2RpcmVjdG9yeSkKCiAgICAjIGdldCBhdWRpbyBmaWxlczoKICAgIGF1ZGlvX2ZpbGVzID0gX2dldF9hdWRpb19maWxlcyhhdWRpb19zb3VyY2UpCgogICAgbm9pc2VfcmVkdWNlX2FyZ3VtZW50cyA9IHsKICAgICAgICAidGFyZ2V0X2RpcmVjdG9yeSI6IHRhcmdldF9kaXJlY3RvcnksCiAgICAgICAgInBhZCI6IHBhZCwKICAgICAgICAiYXR0ZW5fbGltX2RiIjogYXR0ZW5fbGltX2RiLAogICAgICAgICJzaWxlbmNlX3RocmVzaG9sZCI6IHNpbGVuY2VfdGhyZXNob2xkLAogICAgICAgICoqa3dhcmdzLAogICAgfQoKICAgIGlmIHVzZV9tdWx0aXByb2Nlc3Npbmc6CiAgICAgICAgcmVzdWx0cyA9IF9wYXJhbGxlbF9ydW4oCiAgICAgICAgICAgIG5vaXNlX3JlZHVjZV90eXBlPURGTiwKICAgICAgICAgICAgbm9pc2VfcmVkdWNlX2FyZ3VtZW50cz1ub2lzZV9yZWR1Y2VfYXJndW1lbnRzLAogICAgICAgICAgICBuX3dvcmtlcnM9dXNlX211bHRpcHJvY2Vzc2luZywKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJOb2lzZS1yZWR1Y3Rpb24iLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXN1bHRzID0gX3J1bigKICAgICAgICAgICAgbm9pc2VfcmVkdWNlX3R5cGU9REZOLAogICAgICAgICAgICBub2lzZV9yZWR1Y2VfYXJndW1lbnRzPW5vaXNlX3JlZHVjZV9hcmd1bWVudHMsCiAgICAgICAgICAgIGF1ZGlvX2ZpbGVzPWF1ZGlvX2ZpbGVzLAogICAgICAgICAgICBkZXNjcmlwdGlvbj0iTm9pc2UtcmVkdWN0aW9uIiwKICAgICAgICAgICAgdmVyYm9zZT12ZXJib3NlLAogICAgICAgICkKCiAgICByZXR1cm4gX3Byb2Nlc3NfcmVzdWx0cyhyZXN1bHRzLCB2ZXJib3NlKQoKCmRlZiByZWR1Y2Vfbm9pc2UoCiAgICBhdWRpb19zb3VyY2U6IHN0ciwKICAgIHRhcmdldF9kaXJlY3Rvcnk6IHN0ciwKICAgIHNhbXBsZV9yYXRlOiBpbnQgPSAxNjAwMCwKICAgIGR1cmF0aW9uOiBpbnQgPSBOb25lLAogICAgY2hhbm5lbDogaW50ID0gTm9uZSwKICAgIHNpbGVuY2VfdGhyZXNob2xkOiBmbG9hdCA9IE5vbmUsCiAgICB1c2VfbXVsdGlwcm9jZXNzaW5nOiBpbnQgPSAwLAogICAgdmVyYm9zZTogYm9vbCA9IFRydWUsCik6CiAgICAiIiIKICAgIFJlZHVjZSBub2lzZSBmcm9tIGF1ZGlvIGZpbGUgb3IgZGlyZWN0b3J5IGNvbnRhaW5pbmcgYXVkaW8gZmlsZXMuCiAgICBUaGUgYXVkaW8gZmlsZXMgbXVzdCBiZSBpbiAud2F2IGZvcm1hdC4KICAgIFRoZSBjbGVhbmVkIGF1ZGlvIGZpbGVzIHdpbGwgYmUgc2F2ZWQgaW4gdGhlIHRhcmdldF9kaXJlY3RvcnkuCiAgICBGb3IgaW5mb3JtYXRpb24gYWJvdXQgdGhlIG5vaXNlIHJlZHVjdGlvbiBhbGdvcml0aG0gc2VlOgogICAgaHR0cHM6Ly9naXRodWIuY29tL3RpbXNhaW5iL25vaXNlcmVkdWNlCiAgICBOb3RpY2UgdGhhdCB0aGUgc2F2ZWQgZmlsZXMgYXJlIGluIHdhdiBmb3JtYXQsIGV2ZW4gaWYgdGhlIG9yaWdpbmFsIGZpbGVzIGFyZSBpbiBvdGhlciBmb3JtYXQuCgogICAgOnBhcmFtIGF1ZGlvX3NvdXJjZTogICAgICAgIHBhdGggdG8gYXVkaW8gZmlsZSBvciBkaXJlY3RvcnkgY29udGFpbmluZyBhdWRpbyBmaWxlcwogICAgOnBhcmFtIHRhcmdldF9kaXJlY3Rvcnk6ICAgIHBhdGggdG8gZGlyZWN0b3J5IHRvIHNhdmUgdGhlIGNsZWFuZWQgYXVkaW8gZmlsZXMuCiAgICA6cGFyYW0gc2FtcGxlX3JhdGU6ICAgICAgICAgTnVtYmVyIG9mIHNhbXBsZXMgaW4gb25lIHNlY29uZCBpbiB0aGUgYXVkaW8gZmlsZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBQYXNzIGBOb25lYCB0byBrZWVwIHRoZSBvcmlnaW5hbCBzYW1wbGUgcmF0ZS4KICAgIDpwYXJhbSBkdXJhdGlvbjogICAgICAgICAgICBEdXJhdGlvbiBvZiB0aGUgYXVkaW8gZmlsZSB0byBjbGVhbiBpbiBzZWNvbmRzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFBhc3MgYE5vbmVgIHRvIGtlZXAgdGhlIG9yaWdpbmFsIGR1cmF0aW9uLgogICAgOnBhcmFtIGNoYW5uZWw6ICAgICAgICAgICAgIENoYW5uZWwgdG8gY2xlYW4uIFBhc3MgdGhlIG51bWJlciBvZiB0aGUgY2hhbm5lbCB0byBjbGVhbi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBUbyBjbGVhbiBhbGwgY2hhbm5lbHMgcGFzcyBOb25lLgogICAgOnBhcmFtIHNpbGVuY2VfdGhyZXNob2xkOiAgIFRoZSB0aHJlc2hvbGQgdG8gcmVtb3ZlIHNpbGVuY2UgZnJvbSB0aGUgYXVkaW8sIGluIGRCLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIElmIE5vbmUsIG5vIHNpbGVuY2UgcmVtb3ZhbCBpcyBwZXJmb3JtZWQuCiAgICA6cGFyYW0gdXNlX211bHRpcHJvY2Vzc2luZzogTnVtYmVyIG9mIHByb2Nlc3NlcyB0byB1c2UgZm9yIGNsZWFuaW5nIHRoZSBhdWRpbyBmaWxlcy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiAwLCBubyBtdWx0aXByb2Nlc3NpbmcgaXMgdXNlZC4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICBWZXJib3NpdHkgbGV2ZWwuIElmIFRydWUsIGRpc3BsYXkgcHJvZ3Jlc3MgYmFyLgogICAgIiIiCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiUmVkdWNpbmcgbm9pc2UgZnJvbSBhdWRpbyBmaWxlcy4iKQoKICAgICMgY3JlYXRlIHRhcmdldCBkaXJlY3Rvcnk6CiAgICB0YXJnZXRfZGlyZWN0b3J5ID0gX2NyZWF0ZV90YXJnZXRfZGlyZWN0b3J5KHRhcmdldF9kaXJlY3RvcnkpCgogICAgIyBnZXQgYXVkaW8gZmlsZXM6CiAgICBhdWRpb19maWxlcyA9IF9nZXRfYXVkaW9fZmlsZXMoYXVkaW9fc291cmNlKQoKICAgICMgQ3JlYXRlIHRoZSByZWR1Y2Ugbm9pc2Ugb2JqZWN0OgogICAgbm9pc2VfcmVkdWNlX2FyZ3VtZW50cyA9IHsKICAgICAgICAidGFyZ2V0X2RpcmVjdG9yeSI6IHRhcmdldF9kaXJlY3RvcnksCiAgICAgICAgInNhbXBsZV9yYXRlIjogc2FtcGxlX3JhdGUsCiAgICAgICAgImR1cmF0aW9uIjogZHVyYXRpb24sCiAgICAgICAgImNoYW5uZWwiOiBjaGFubmVsLAogICAgICAgICJzaWxlbmNlX3RocmVzaG9sZCI6IHNpbGVuY2VfdGhyZXNob2xkLAogICAgfQoKICAgIGlmIHVzZV9tdWx0aXByb2Nlc3Npbmc6CiAgICAgICAgcmVzdWx0cyA9IF9wYXJhbGxlbF9ydW4oCiAgICAgICAgICAgIG5vaXNlX3JlZHVjZV90eXBlPVJlZHVjZU5vaXNlLAogICAgICAgICAgICBub2lzZV9yZWR1Y2VfYXJndW1lbnRzPW5vaXNlX3JlZHVjZV9hcmd1bWVudHMsCiAgICAgICAgICAgIG5fd29ya2Vycz11c2VfbXVsdGlwcm9jZXNzaW5nLAogICAgICAgICAgICBhdWRpb19maWxlcz1hdWRpb19maWxlcywKICAgICAgICAgICAgZGVzY3JpcHRpb249Ik5vaXNlLXJlZHVjdGlvbiIsCiAgICAgICAgICAgIHZlcmJvc2U9dmVyYm9zZSwKICAgICAgICApCiAgICBlbHNlOgogICAgICAgIHJlc3VsdHMgPSBfcnVuKAogICAgICAgICAgICBub2lzZV9yZWR1Y2VfdHlwZT1SZWR1Y2VOb2lzZSwKICAgICAgICAgICAgbm9pc2VfcmVkdWNlX2FyZ3VtZW50cz1ub2lzZV9yZWR1Y2VfYXJndW1lbnRzLAogICAgICAgICAgICBhdWRpb19maWxlcz1hdWRpb19maWxlcywKICAgICAgICAgICAgZGVzY3JpcHRpb249Ik5vaXNlLXJlZHVjdGlvbiIsCiAgICAgICAgICAgIHZlcmJvc2U9dmVyYm9zZSwKICAgICAgICApCgogICAgcmV0dXJuIF9wcm9jZXNzX3Jlc3VsdHMocmVzdWx0cywgdmVyYm9zZSkKCgpkZWYgX2NyZWF0ZV90YXJnZXRfZGlyZWN0b3J5KHRhcmdldF9kaXJlY3Rvcnk6IHN0cikgLT4gc3RyOgogICAgdGFyZ2V0X2RpcmVjdG9yeSA9IFBhdGgodGFyZ2V0X2RpcmVjdG9yeSkKICAgIGlmIG5vdCB0YXJnZXRfZGlyZWN0b3J5LmV4aXN0cygpOgogICAgICAgIHRhcmdldF9kaXJlY3RvcnkubWtkaXIocGFyZW50cz1UcnVlLCBleGlzdF9vaz1UcnVlKQogICAgcmV0dXJuIHN0cih0YXJnZXRfZGlyZWN0b3J5KQoKCmRlZiBfZ2V0X2F1ZGlvX2ZpbGVzKGF1ZGlvX3NvdXJjZTogc3RyKToKICAgIGF1ZGlvX3NvdXJjZSA9IFBhdGgoYXVkaW9fc291cmNlKQogICAgYXVkaW9fZmlsZXMgPSBbXQogICAgaWYgYXVkaW9fc291cmNlLmlzX2RpcigpOgogICAgICAgIGF1ZGlvX2ZpbGVzID0gbGlzdChhdWRpb19zb3VyY2UuZ2xvYigiKi4qIikpCiAgICBlbGlmIGF1ZGlvX3NvdXJjZS5pc19maWxlKCk6CiAgICAgICAgYXVkaW9fZmlsZXMuYXBwZW5kKGF1ZGlvX3NvdXJjZSkKICAgIGVsc2U6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJhdWRpb19zb3VyY2UgbXVzdCBiZSBhIGZpbGUgb3IgYSBkaXJlY3RvcnksIGdvdCB7YXVkaW9fc291cmNlfSIKICAgICAgICApCiAgICByZXR1cm4gYXVkaW9fZmlsZXMKCgpkZWYgX3BhcmFsbGVsX3J1bigKICAgIG5vaXNlX3JlZHVjZV90eXBlOiB0eXBlW1JlZHVjZU5vaXNlQmFzZV0sCiAgICBub2lzZV9yZWR1Y2VfYXJndW1lbnRzOiBkaWN0LAogICAgbl93b3JrZXJzOiBpbnQsCiAgICBhdWRpb19maWxlczogbGlzdFtQYXRoXSwKICAgIGRlc2NyaXB0aW9uOiBzdHIsCiAgICB2ZXJib3NlOiBib29sLAopIC0+IGxpc3RbdHVwbGVbYm9vbCwgdHVwbGVbc3RyLCBzdHJdXV06CiAgICAiIiIKICAgIFJ1biBtdWx0aXBsZSBub2lzZSByZWR1Y2Ugd29ya2VycyB3aXRoIG11bHRpcHJvY2Vzc2luZyB0byBjb21wbGV0ZSB0aGUgdGFza3MgdGhhdCB3aWxsIGJlIGNyZWF0ZWQgb24gdGhlIHByb3ZpZGVkCiAgICBmaWxlcyB1c2luZyB0aGUgZ2l2ZW4gdGFzayBjcmVhdG9yLgoKICAgIDpwYXJhbSBub2lzZV9yZWR1Y2VfdHlwZTogICBUaGUgbm9pc2UgcmVkdWNlIHR5cGUgdG8gdXNlLgogICAgOnBhcmFtIG5fd29ya2VyczogICAgICAgICAgIFRoZSBudW1iZXIgb2Ygd29ya2VycyB0byB1c2UuCiAgICA6cGFyYW0gYXVkaW9fZmlsZXM6ICAgICAgICAgVGhlIGF1ZGlvIGZpbGVzIHRvIHVzZS4KICAgIDpwYXJhbSBkZXNjcmlwdGlvbjogICAgICAgICBUaGUgZGVzY3JpcHRpb24gdG8gdXNlIGZvciB0aGUgcHJvZ3Jlc3MgYmFyLgogICAgOnBhcmFtIHZlcmJvc2U6ICAgICAgICAgICAgIFZlcmJvc2l0eS4KCiAgICA6cmV0dXJuczogVGhlIGNvbGxlY3RlZCByZXN1bHRzLgogICAgIiIiCiAgICAjIENoZWNrIHRoZSBudW1iZXIgb2Ygd29ya2VyczoKICAgIGlmIG5fd29ya2VycyA+IGxlbihhdWRpb19maWxlcyk6CiAgICAgICAgX0xPR0dFUi53YXJuaW5nKAogICAgICAgICAgICBmIlRoZSBudW1iZXIgb2Ygd29ya2VycyAoe25fd29ya2Vyc30pIGlzIGxhcmdlciB0aGFuIHRoZSBudW1iZXIgb2YgYXVkaW8gZmlsZXMgKHtsZW4oYXVkaW9fZmlsZXMpfSkuICIKICAgICAgICAgICAgZiJTZXR0aW5nIHRoZSBudW1iZXIgb2Ygd29ya2VycyB0byB7bGVuKGF1ZGlvX2ZpbGVzKX0uIgogICAgICAgICkKICAgICAgICBuX3dvcmtlcnMgPSBsZW4oYXVkaW9fZmlsZXMpCgogICAgIyBJbml0aWFsaXplIHRoZSBtdWx0aXByb2Nlc3NpbmcgcXVldWVzOgogICAgdGFza3NfcXVldWUgPSBRdWV1ZSgpCiAgICByZXN1bHRzX3F1ZXVlID0gUXVldWUoKQoKICAgICMgSW5pdGlhbGl6ZSB0aGUgbXVsdGlwcm9jZXNzaW5nIHByb2Nlc3NlczoKICAgIHRhc2tfY29tcGxldGlvbl9wcm9jZXNzZXMgPSBbCiAgICAgICAgUHJvY2VzcygKICAgICAgICAgICAgdGFyZ2V0PV9tdWx0aXByb2Nlc3NpbmdfY29tcGxldGVfdGFza3MsCiAgICAgICAgICAgIGt3YXJncz17CiAgICAgICAgICAgICAgICAibm9pc2VfcmVkdWNlX3R5cGUiOiBub2lzZV9yZWR1Y2VfdHlwZSwKICAgICAgICAgICAgICAgICJub2lzZV9yZWR1Y2VfYXJndW1lbnRzIjogbm9pc2VfcmVkdWNlX2FyZ3VtZW50cywKICAgICAgICAgICAgICAgICJ0YXNrc19xdWV1ZSI6IHRhc2tzX3F1ZXVlLAogICAgICAgICAgICAgICAgInJlc3VsdHNfcXVldWUiOiByZXN1bHRzX3F1ZXVlLAogICAgICAgICAgICB9LAogICAgICAgICkKICAgICAgICBmb3IgXyBpbiByYW5nZShuX3dvcmtlcnMpCiAgICBdCgogICAgIyBTdGFydCB0aGUgbXVsdGlwcm9jZXNzaW5nIHByb2Nlc3NlczoKICAgIGZvciBwIGluIHRhc2tfY29tcGxldGlvbl9wcm9jZXNzZXM6CiAgICAgICAgcC5zdGFydCgpCgogICAgIyBQdXQgdGhlIHRhc2tzIGluIHRoZSBxdWV1ZToKICAgIGZvciBhdWRpb19maWxlIGluIGF1ZGlvX2ZpbGVzOgogICAgICAgICMgdGFza3NfcXVldWUucHV0KHRhc2tfY3JlYXRvci5jcmVhdGVfdGFzayhhdWRpb19maWxlPWF1ZGlvX2ZpbGUpLnRvX3R1cGxlKCkpCiAgICAgICAgdGFza3NfcXVldWUucHV0KGF1ZGlvX2ZpbGUpCgogICAgIyBQdXQgdGhlIHN0b3AgbWFya3MgaW4gdGhlIHF1ZXVlOgogICAgZm9yIF8gaW4gcmFuZ2Uobl93b3JrZXJzKToKICAgICAgICB0YXNrc19xdWV1ZS5wdXQoX01VTFRJUFJPQ0VTU0lOR19TVE9QX01BUkspCgogICAgIyBDb2xsZWN0IHRoZSByZXN1bHRzOgogICAgcmVzdWx0cyA9IFtdCiAgICBzdG9wX21hcmtzX2NvdW50ZXIgPSAwCiAgICB3aXRoIHRxZG0oCiAgICAgICAgZGVzYz1kZXNjcmlwdGlvbiwKICAgICAgICB1bml0PSJmaWxlIiwKICAgICAgICB0b3RhbD1sZW4oYXVkaW9fZmlsZXMpLAogICAgICAgIGRpc2FibGU9bm90IHZlcmJvc2UsCiAgICApIGFzIHByb2dyZXNzYmFyOgogICAgICAgIHdoaWxlIFRydWU6CiAgICAgICAgICAgICMgR2V0IGEgcmVzdWx0IGZyb20gdGhlIHF1ZXVlOgogICAgICAgICAgICByZXN1bHQ6IHR1cGxlW2Jvb2wsIHR1cGxlW3N0ciwgc3RyXV0gPSByZXN1bHRzX3F1ZXVlLmdldCgpCiAgICAgICAgICAgIGlmIHJlc3VsdCA9PSBfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSzoKICAgICAgICAgICAgICAgIHN0b3BfbWFya3NfY291bnRlciArPSAxCiAgICAgICAgICAgICAgICBpZiBzdG9wX21hcmtzX2NvdW50ZXIgPT0gbl93b3JrZXJzOgogICAgICAgICAgICAgICAgICAgIGJyZWFrCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAjIENvbGxlY3QgdGhlIHJlc3VsdDoKICAgICAgICAgICAgICAgIHJlc3VsdHMuYXBwZW5kKHJlc3VsdCkKICAgICAgICAgICAgICAgIHByb2dyZXNzYmFyLnVwZGF0ZSgxKQoKICAgICMgV2FpdCBmb3IgdGhlIHByb2Nlc3NlcyB0byBmaW5pc2g6CiAgICBmb3IgcCBpbiB0YXNrX2NvbXBsZXRpb25fcHJvY2Vzc2VzOgogICAgICAgIHAuam9pbigpCgogICAgcmV0dXJuIHJlc3VsdHMKCgpkZWYgX3J1bigKICAgIG5vaXNlX3JlZHVjZV90eXBlOiB0eXBlW1JlZHVjZU5vaXNlQmFzZV0sCiAgICBub2lzZV9yZWR1Y2VfYXJndW1lbnRzOiBkaWN0LAogICAgYXVkaW9fZmlsZXM6IGxpc3RbUGF0aF0sCiAgICBkZXNjcmlwdGlvbjogc3RyLAogICAgdmVyYm9zZTogYm9vbCwKKSAtPiBsaXN0W3R1cGxlW2Jvb2wsIHR1cGxlW3N0ciwgc3RyXV1dOgogICAgIiIiCiAgICBSdW4gdGhlIG5vaXNlIHJlZHVjZSBhbGdvcml0aG0gb24gdGhlIGdpdmVuIGF1ZGlvIGZpbGVzIGFuZCBjb2xsZWN0IHRoZSByZXN1bHRzLgoKICAgIDpwYXJhbSBub2lzZV9yZWR1Y2VfdHlwZTogICAgICAgVGhlIG5vaXNlIHJlZHVjZSB0eXBlIHRvIHVzZS4KICAgIDpwYXJhbSBub2lzZV9yZWR1Y2VfYXJndW1lbnRzOiAgVGhlIG5vaXNlcmVkdWNlIGluaXRpYWxpemF0aW9uIGt3YXJncy4KICAgIDpwYXJhbSBhdWRpb19maWxlczogICAgICAgICAgICAgVGhlIGF1ZGlvIGZpbGVzIHRvIHVzZS4KICAgIDpwYXJhbSBkZXNjcmlwdGlvbjogICAgICAgICAgICAgVGhlIGRlc2NyaXB0aW9uIHRvIHVzZSBmb3IgdGhlIHByb2dyZXNzIGJhci4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICAgICAgVmVyYm9zaXR5LgoKICAgIDpyZXR1cm5zOiBUaGUgY29sbGVjdGVkIHJlc3VsdHMuCiAgICAiIiIKICAgICMgQ3JlYXRlIHRoZSByZWR1Y2Ugbm9pc2Ugb2JqZWN0OgogICAgbm9pc2VfcmVkdWNlciA9IG5vaXNlX3JlZHVjZV90eXBlKCoqbm9pc2VfcmVkdWNlX2FyZ3VtZW50cykKCiAgICAjIFJ1biB0aGUgbm9pc2UgcmVkdWNlIGFsZ29yaXRobSBvbiB0aGUgYXVkaW8gZmlsZXMgYW5kIGNvbGxlY3QgdGhlIHJlc3VsdHM6CiAgICByZXN1bHRzID0gW10KICAgIGZvciBhdWRpb19maWxlIGluIHRxZG0oCiAgICAgICAgYXVkaW9fZmlsZXMsCiAgICAgICAgZGVzYz1kZXNjcmlwdGlvbiwKICAgICAgICB1bml0PSJmaWxlIiwKICAgICAgICB0b3RhbD1sZW4oYXVkaW9fZmlsZXMpLAogICAgICAgIGRpc2FibGU9bm90IHZlcmJvc2UsCiAgICApOgogICAgICAgIHJlc3VsdHMuYXBwZW5kKG5vaXNlX3JlZHVjZXIucmVkdWNlX25vaXNlKGF1ZGlvX2ZpbGU9YXVkaW9fZmlsZSkpCgogICAgcmV0dXJuIHJlc3VsdHMKCgpkZWYgX3Byb2Nlc3NfcmVzdWx0cygKICAgIHJlc3VsdHM6IGxpc3RbdHVwbGVbYm9vbCwgdHVwbGVbc3RyLCBzdHJdXV0sIHZlcmJvc2U6IGJvb2wKKSAtPiB0dXBsZVtkaWN0LCBkaWN0XToKICAgICIiIgogICAgUHJvY2VzcyB0aGUgcmVzdWx0cyBvZiB0aGUgdGFza3MuCgogICAgOnBhcmFtIHJlc3VsdHM6IFRoZSByZXN1bHRzIHRvIHByb2Nlc3MuCiAgICA6cGFyYW0gdmVyYm9zZTogVmVyYm9zaXR5LgoKICAgIDpyZXR1cm5zOiBUaGUgcHJvY2Vzc2VkIHJlc3VsdHMgYXMgYSB0dXBsZSBvZiBzdWNjZXNzZXMgYW5kIGVycm9ycy4KICAgICIiIgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIlN1bW1hcml6aW5nIHRoZSByZXN1bHRzLiIpCiAgICBzdWNjZXNzZXMgPSB7fQogICAgZXJyb3JzID0ge30KICAgIGZvciBpc19lcnJvciwgcmVzdWx0IGluIHJlc3VsdHM6CiAgICAgICAgaWYgaXNfZXJyb3I6CiAgICAgICAgICAgIGVycm9yc1tyZXN1bHRbMF1dID0gcmVzdWx0WzFdCiAgICAgICAgZWxzZToKICAgICAgICAgICAgc3VjY2Vzc2VzW3Jlc3VsdFswXV0gPSByZXN1bHRbMV0KICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiRG9uZSAoe2xlbihzdWNjZXNzZXMpfS97bGVuKHN1Y2Nlc3NlcykgKyBsZW4oZXJyb3JzKX0pXG4iKQoKICAgIHJldHVybiBzdWNjZXNzZXMsIGVycm9ycwo= + requirements: + - librosa + - noisereduce + - deepfilternet + - torchaudio>=2.1.2 + code_origin: '' + base_image: mlrun/mlrun + filename: noise_reduction.py entry_points: reduce_noise: - has_kwargs: false - name: reduce_noise - has_varargs: false - doc: 'Reduce noise from audio file or directory containing audio files. - - The audio files must be in .wav format. - - The cleaned audio files will be saved in the target_directory. - - For information about the noise reduction algorithm see: - - https://github.com/timsainb/noisereduce - - Notice that the saved files are in wav format, even if the original files - are in other format.' parameters: - name: audio_source type: str @@ -52,78 +58,82 @@ spec: type: bool doc: Verbosity level. If True, display progress bar. default: true - lineno: 388 - clean_audio: + name: reduce_noise + doc: 'Reduce noise from audio file or directory containing audio files. + + The audio files must be in .wav format. + + The cleaned audio files will be saved in the target_directory. + + For information about the noise reduction algorithm see: + + https://github.com/timsainb/noisereduce + + Notice that the saved files are in wav format, even if the original files + are in other format.' has_kwargs: false - name: clean_audio has_varargs: false + lineno: 388 + clean_audio: outputs: - type: torch.Tensor - doc: '' parameters: - name: self - name: data type: Tensor - lineno: 276 - save_audio: + name: clean_audio + doc: '' has_kwargs: false - name: save_audio has_varargs: false - doc: '' + lineno: 276 + save_audio: parameters: - name: self - name: audio type: ndarray - name: target_path type: Path - lineno: 256 - load_audio: + name: save_audio + doc: '' has_kwargs: false - name: load_audio has_varargs: false + lineno: 256 + load_audio: outputs: - type: torch.Tensor - doc: '' parameters: - name: self - name: file type: str - lineno: 268 - update_to_wav_suffix: + name: load_audio + doc: '' has_kwargs: false - name: update_to_wav_suffix has_varargs: false - doc: '' + lineno: 268 + update_to_wav_suffix: parameters: - name: self - name: audio_file type: Path - lineno: 125 - remove_silence: + name: update_to_wav_suffix + doc: '' has_kwargs: false - name: remove_silence has_varargs: false + lineno: 125 + remove_silence: outputs: - doc: The audio without silence. - doc: Remove silence sections from the audio. parameters: - name: self - name: audio type: ndarray doc: The audio to remove silence from. + name: remove_silence + doc: Remove silence sections from the audio. + has_kwargs: false + has_varargs: false lineno: 134 reduce_noise_dfn: - has_kwargs: true - name: reduce_noise_dfn - has_varargs: false - doc: 'Reduce noise from audio files using DeepFilterNet. - - For more information about the noise reduction algorithm see: - - https://github.com/Rikorose/DeepFilterNet - - Notice that the saved files are in wav format, even if the original files - are in other format.' parameters: - name: audio_source type: str @@ -153,27 +163,18 @@ spec: type: bool doc: verbosity level. If True, display progress bar and logs. default: true + name: reduce_noise_dfn + doc: 'Reduce noise from audio files using DeepFilterNet. + + For more information about the noise reduction algorithm see: + + https://github.com/Rikorose/DeepFilterNet + + Notice that the saved files are in wav format, even if the original files + are in other format.' + has_kwargs: true + has_varargs: false lineno: 322 - build: - code_origin: '' - base_image: mlrun/mlrun - requirements: - - librosa - - noisereduce - - deepfilternet - - torchaudio>=2.1.2 - functionSourceCode: aW1wb3J0IGxvZ2dpbmcKZnJvbSBhYmMgaW1wb3J0IEFCQ01ldGEsIGFic3RyYWN0bWV0aG9kCmZyb20gbXVsdGlwcm9jZXNzaW5nIGltcG9ydCBQcm9jZXNzLCBRdWV1ZQpmcm9tIHBhdGhsaWIgaW1wb3J0IFBhdGgKZnJvbSB0eXBpbmcgaW1wb3J0IExpc3QsIFR1cGxlLCBUeXBlLCBVbmlvbgoKaW1wb3J0IGxpYnJvc2EKaW1wb3J0IG51bXB5IGFzIG5wCmltcG9ydCB0b3JjaApmcm9tIHNjaXB5LmlvIGltcG9ydCB3YXZmaWxlCmZyb20gdHFkbSBpbXBvcnQgdHFkbQoKIzogVGhlIHZhbHVlIHRvIHNlbmQgaW50byBtdWx0aXByb2Nlc3NpbmcgcXVldWVzIHRvIHN0b3AgdGhlIHByb2Nlc3M6Cl9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLID0gIlNUT1AiCgojIEdldCB0aGUgZ2xvYmFsIGxvZ2dlcjoKdHJ5OgogICAgaW1wb3J0IG1scnVuCgogICAgX0xPR0dFUiA9IG1scnVuLmdldF9vcl9jcmVhdGVfY3R4KCJub2lzZV9yZWR1Y2UiKS5sb2dnZXIKZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3I6CiAgICBfTE9HR0VSID0gbG9nZ2luZy5nZXRMb2dnZXIoKQoKCmNsYXNzIFJlZHVjZU5vaXNlQmFzZShtZXRhY2xhc3M9QUJDTWV0YSk6CiAgICAiIiIKICAgIEJhc2UgY2xhc3MgZm9yIG5vaXNlIHJlZHVjdGlvbi4KICAgIFRoaXMgY2xhc3MgaXMgYWltZWQgdG8gYmUgaW5oZXJpdGVkIGJ5IHNwZWNpZmljIG5vaXNlIHJlZHVjdGlvbiBhbGdvcml0aG1zLgogICAgWW91IG11c3QgaW1wbGVtZW50IHRoZSBmb2xsb3dpbmcgbWV0aG9kczoKICAgIC0gY2xlYW5fYXVkaW86ICBUaGUgbWV0aG9kIHRvIGNsZWFuIHRoZSBhdWRpbywgd2hlcmUgdGhlIG5vaXNlIHJlZHVjdGlvbiBhbGdvcml0aG0gaXMgaW1wbGVtZW50ZWQuCiAgICAtIHNhdmVfYXVkaW86ICAgVGhlIG1ldGhvZCB0byBzYXZlIHRoZSBhdWRpbyB0byBhIGZpbGUuCiAgICAtIGxvYWRfYXVkaW86ICAgVGhlIG1ldGhvZCB0byBsb2FkIHRoZSBhdWRpbyBmcm9tIGEgZmlsZS4KCiAgICBBZnRlciBpbXBsZW1lbnRpbmcgdGhlIGFib3ZlIG1ldGhvZHMsIHlvdSBjYW4gdXNlIHRoZSByZWR1Y2Vfbm9pc2UgbWV0aG9kIHRvIHJlZHVjZSBub2lzZSBmcm9tIGF1ZGlvIGZpbGVzLgogICAgIiIiCiAgICBkZWYgX19pbml0X18oCiAgICAgICAgc2VsZiwKICAgICAgICB0YXJnZXRfZGlyZWN0b3J5OiBQYXRoLAogICAgICAgIHZlcmJvc2U6IGJvb2wgPSBUcnVlLAogICAgICAgIHNpbGVuY2VfdGhyZXNob2xkOiBmbG9hdCA9IE5vbmUsCiAgICApOgogICAgICAgIHNlbGYudGFyZ2V0X2RpcmVjdG9yeSA9IFBhdGgodGFyZ2V0X2RpcmVjdG9yeSkKICAgICAgICBzZWxmLnZlcmJvc2UgPSB2ZXJib3NlCiAgICAgICAgc2VsZi5zaWxlbmNlX3RocmVzaG9sZCA9IHNpbGVuY2VfdGhyZXNob2xkCgogICAgZGVmIHJlZHVjZV9ub2lzZShzZWxmLCBhdWRpb19maWxlOiBQYXRoKSAtPiBUdXBsZVtib29sLCBUdXBsZVtzdHIsIHN0cl1dOgogICAgICAgICIiIgogICAgICAgIFJlZHVjZSBub2lzZSBmcm9tIHRoZSBnaXZlbiBhdWRpbyBmaWxlLgoKICAgICAgICA6cGFyYW0gYXVkaW9fZmlsZTogIFRoZSBhdWRpbyBmaWxlIHRvIHJlZHVjZSBub2lzZSBmcm9tLgoKICAgICAgICA6cmV0dXJuczogQSB0dXBsZSBvZjoKICAgICAgICAgLSBhIGJvb2xlYW4gaW5kaWNhdGluZyB3aGV0aGVyIGFuIGVycm9yIG9jY3VycmVkCiAgICAgICAgIC0gYSB0dXBsZSBvZjoKICAgICAgICAgICAgLSBhdWRpbyBmaWxlIG5hbWUKICAgICAgICAgICAgLSB0YXJnZXQgcGF0aCBpbiBjYXNlIG9mIHN1Y2Nlc3MgLyBlcnJvciBtZXNzYWdlIGluIGNhc2Ugb2YgZmFpbHVyZS4KICAgICAgICAiIiIKICAgICAgICB0cnk6CiAgICAgICAgICAgIGlmIHNlbGYudmVyYm9zZToKICAgICAgICAgICAgICAgIF9MT0dHRVIuaW5mbyhmIlJlZHVjaW5nIG5vaXNlIGZyb20ge2F1ZGlvX2ZpbGUubmFtZX0uIikKCiAgICAgICAgICAgICMgTG9hZCBhdWRpbyBkYXRhOgogICAgICAgICAgICBhdWRpbyA9IHNlbGYubG9hZF9hdWRpbyhmaWxlPXN0cihhdWRpb19maWxlKSkKCiAgICAgICAgICAgICMgUGVyZm9ybSBub2lzZSByZWR1Y3Rpb246CiAgICAgICAgICAgIHJlZHVjZWRfbm9pc2UgPSBzZWxmLmNsZWFuX2F1ZGlvKGRhdGE9YXVkaW8pCgogICAgICAgICAgICAjIFJlbW92ZSBzaWxlbmNlIGZyb20gdGhlIGF1ZGlvIGlmIG5lY2Vzc2FyeToKICAgICAgICAgICAgcmVkdWNlZF9ub2lzZSA9IHNlbGYucmVtb3ZlX3NpbGVuY2UoYXVkaW89cmVkdWNlZF9ub2lzZSkKCiAgICAgICAgICAgICMgUHJlcGFyZSB0YXJnZXQgcGF0aDoKICAgICAgICAgICAgdGFyZ2V0X3BhdGggPSBzZWxmLnVwZGF0ZV90b193YXZfc3VmZml4KGF1ZGlvX2ZpbGU9YXVkaW9fZmlsZSkKCiAgICAgICAgICAgICMgU2F2ZSBmaWxlOgogICAgICAgICAgICBzZWxmLnNhdmVfYXVkaW8oCiAgICAgICAgICAgICAgICBhdWRpbz1yZWR1Y2VkX25vaXNlLAogICAgICAgICAgICAgICAgdGFyZ2V0X3BhdGg9dGFyZ2V0X3BhdGgsCiAgICAgICAgICAgICkKCiAgICAgICAgICAgIGlmIHNlbGYudmVyYm9zZToKICAgICAgICAgICAgICAgIF9MT0dHRVIuaW5mbyhmIlNhdmVkIGNsZWFuZWQgYXVkaW8gZmlsZSB0byB7dGFyZ2V0X3BhdGh9LiIpCgogICAgICAgICAgICByZXR1cm4gRmFsc2UsIChhdWRpb19maWxlLm5hbWUsIHN0cih0YXJnZXRfcGF0aCkpCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBleGNlcHRpb246CiAgICAgICAgICAgIGlmIHNlbGYudmVyYm9zZToKICAgICAgICAgICAgICAgIF9MT0dHRVIuZXJyb3IoZiJGYWlsZWQgdG8gcmVkdWNlIG5vaXNlIGZyb20ge2F1ZGlvX2ZpbGUubmFtZX0uIikKICAgICAgICAgICAgICAgIF9MT0dHRVIuZXJyb3IoZiJFcnJvcjoge2V4Y2VwdGlvbn0iKQogICAgICAgICAgICAjIENvbGxlY3QgdGhlIGVycm9yOgogICAgICAgICAgICByZXR1cm4gVHJ1ZSwgKGF1ZGlvX2ZpbGUubmFtZSwgc3RyKGV4Y2VwdGlvbikpCgogICAgQGFic3RyYWN0bWV0aG9kCiAgICBkZWYgY2xlYW5fYXVkaW8oc2VsZiwgZGF0YSkgLT4gVW5pb25bbnAubmRhcnJheSwgdG9yY2guVGVuc29yXToKICAgICAgICAiIiIKICAgICAgICBDbGVhbiB0aGUgYXVkaW8gZnJvbSBub2lzZS4gSGVyZSB5b3Ugc2hvdWxkIGltcGxlbWVudCB0aGUgbm9pc2UgcmVkdWN0aW9uIGFsZ29yaXRobS4KCiAgICAgICAgOnBhcmFtIGRhdGE6ICAgIFRoZSBhdWRpbyBkYXRhIHRvIGNsZWFuLgoKICAgICAgICA6cmV0dXJuczogVGhlIGNsZWFuZWQgYXVkaW8uCiAgICAgICAgIiIiCiAgICAgICAgcGFzcwoKICAgIEBhYnN0cmFjdG1ldGhvZAogICAgZGVmIHNhdmVfYXVkaW8oc2VsZiwgYXVkaW86IG5wLm5kYXJyYXksIHRhcmdldF9wYXRoOiBQYXRoKToKICAgICAgICAiIiIKICAgICAgICBTYXZlIHRoZSBhdWRpbyB0byBhIGZpbGUuCgogICAgICAgIDpwYXJhbSBhdWRpbzogICAgICAgVGhlIGF1ZGlvIHRvIHNhdmUuCiAgICAgICAgOnBhcmFtIHRhcmdldF9wYXRoOiBUaGUgdGFyZ2V0IHBhdGggdG8gc2F2ZSB0aGUgYXVkaW8gdG8uCiAgICAgICAgIiIiCiAgICAgICAgcGFzcwoKICAgIEBhYnN0cmFjdG1ldGhvZAogICAgZGVmIGxvYWRfYXVkaW8oc2VsZiwgZmlsZTogc3RyKSAtPiBUdXBsZVtVbmlvbltucC5uZGFycmF5LCB0b3JjaC5UZW5zb3JdLCBpbnRdOgogICAgICAgICIiIgogICAgICAgIExvYWQgdGhlIGF1ZGlvIGZyb20gYSBmaWxlLgoKICAgICAgICA6cGFyYW0gZmlsZTogICAgVGhlIGZpbGUgdG8gbG9hZCB0aGUgYXVkaW8gZnJvbS4KCiAgICAgICAgOnJldHVybnM6IEEgdHVwbGUgb2Y6CiAgICAgICAgICAgIC0gdGhlIGF1ZGlvIGRhdGEKICAgICAgICAgICAgLSB0aGUgc2FtcGxlIHJhdGUKICAgICAgICAiIiIKICAgICAgICBwYXNzCgogICAgZGVmIHVwZGF0ZV90b193YXZfc3VmZml4KHNlbGYsIGF1ZGlvX2ZpbGU6IFBhdGgpOgogICAgICAgIHRhcmdldF9wYXRoID0gc2VsZi50YXJnZXRfZGlyZWN0b3J5IC8gYXVkaW9fZmlsZS5uYW1lCiAgICAgICAgaWYgdGFyZ2V0X3BhdGguc3VmZml4ICE9ICIud2F2IjoKICAgICAgICAgICAgb2xkX3N1ZmZpeCA9IHRhcmdldF9wYXRoLnN1ZmZpeFsxOl0KICAgICAgICAgICAgdGFyZ2V0X3BhdGggPSB0YXJnZXRfcGF0aC53aXRoX3N0ZW0odGFyZ2V0X3BhdGguc3RlbSArIGYiX3tvbGRfc3VmZml4fSIpCiAgICAgICAgICAgIHJldHVybiB0YXJnZXRfcGF0aC53aXRoX3N1ZmZpeCgiLndhdiIpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmV0dXJuIHRhcmdldF9wYXRoCgogICAgZGVmIHJlbW92ZV9zaWxlbmNlKAogICAgICAgIHNlbGYsCiAgICAgICAgYXVkaW86IG5wLm5kYXJyYXksCiAgICApOgogICAgICAgICIiIgogICAgICAgIFJlbW92ZSBzaWxlbmNlIHNlY3Rpb25zIGZyb20gdGhlIGF1ZGlvLgoKICAgICAgICA6cGFyYW0gYXVkaW86ICAgVGhlIGF1ZGlvIHRvIHJlbW92ZSBzaWxlbmNlIGZyb20uCgogICAgICAgIDpyZXR1cm5zOiBUaGUgYXVkaW8gd2l0aG91dCBzaWxlbmNlLgogICAgICAgICIiIgogICAgICAgIGlmIHNlbGYuc2lsZW5jZV90aHJlc2hvbGQgaXMgTm9uZToKICAgICAgICAgICAgcmV0dXJuIGF1ZGlvCgogICAgICAgICMgR2V0IHRoZSBpbmRpY2VzIG9mIHRoZSBub24tc2lsZW50IGZyYW1lczoKICAgICAgICBub25fc2lsZW50X2luZGljZXMgPSBsaWJyb3NhLmVmZmVjdHMuc3BsaXQoCiAgICAgICAgICAgIHk9YXVkaW8sCiAgICAgICAgICAgIHRvcF9kYj1zZWxmLnNpbGVuY2VfdGhyZXNob2xkLAogICAgICAgICAgICBmcmFtZV9sZW5ndGg9MjA0OCwKICAgICAgICAgICAgaG9wX2xlbmd0aD0yNTYsCiAgICAgICAgKQoKICAgICAgICAjIEdldCB0aGUgbm9uLXNpbGVudCBhdWRpbzoKICAgICAgICBub25fc2lsZW50X2F1ZGlvID0gbnAuY29uY2F0ZW5hdGUoCiAgICAgICAgICAgIFthdWRpb1s6LCBzdGFydDplbmRdIGZvciBzdGFydCwgZW5kIGluIG5vbl9zaWxlbnRfaW5kaWNlc10sIGF4aXM9MQogICAgICAgICkKCiAgICAgICAgcmV0dXJuIG5vbl9zaWxlbnRfYXVkaW8KCgpjbGFzcyBSZWR1Y2VOb2lzZShSZWR1Y2VOb2lzZUJhc2UpOgogICAgZGVmIF9faW5pdF9fKAogICAgICAgIHNlbGYsCiAgICAgICAgdGFyZ2V0X2RpcmVjdG9yeTogUGF0aCwKICAgICAgICB2ZXJib3NlOiBib29sID0gVHJ1ZSwKICAgICAgICBzaWxlbmNlX3RocmVzaG9sZDogZmxvYXQgPSBOb25lLAogICAgICAgIHNhbXBsZV9yYXRlOiBpbnQgPSAxNjAwMCwKICAgICAgICBkdXJhdGlvbjogaW50ID0gTm9uZSwKICAgICAgICBjaGFubmVsOiBpbnQgPSBOb25lLAogICAgKToKICAgICAgICBzdXBlcigpLl9faW5pdF9fKHRhcmdldF9kaXJlY3RvcnksIHZlcmJvc2UsIHNpbGVuY2VfdGhyZXNob2xkKQogICAgICAgIHNlbGYuc2FtcGxlX3JhdGUgPSBzYW1wbGVfcmF0ZQogICAgICAgIHNlbGYuZHVyYXRpb24gPSBkdXJhdGlvbgogICAgICAgIHNlbGYuY2hhbm5lbCA9IGNoYW5uZWwKCiAgICBkZWYgc2F2ZV9hdWRpbyhzZWxmLCBhdWRpbzogbnAubmRhcnJheSwgdGFyZ2V0X3BhdGg6IFBhdGgpOgogICAgICAgICMgSWYgdGhlIGF1ZGlvIGhhcyBtb3JlIHRoYW4gb25lIGNoYW5uZWwsIHRyYW5zcG9zZSBpdCBpbiBvcmRlciB0byBzYXZlIGl0OgogICAgICAgIGlmIGxlbihhdWRpbykgPiAxOgogICAgICAgICAgICBhdWRpbyA9IGF1ZGlvLlQKCiAgICAgICAgd2F2ZmlsZS53cml0ZSgKICAgICAgICAgICAgZmlsZW5hbWU9dGFyZ2V0X3BhdGgsCiAgICAgICAgICAgIHJhdGU9c2VsZi5zYW1wbGVfcmF0ZSwKICAgICAgICAgICAgZGF0YT1hdWRpbywKICAgICAgICApCgogICAgZGVmIGxvYWRfYXVkaW8oc2VsZiwgZmlsZTogc3RyKSAtPiBucC5uZGFycmF5OgogICAgICAgIGRhdGEsIHNyID0gbGlicm9zYS5sb2FkKAogICAgICAgICAgICBwYXRoPWZpbGUsCiAgICAgICAgICAgIHNyPXNlbGYuc2FtcGxlX3JhdGUsCiAgICAgICAgICAgIG1vbm89RmFsc2UsICAjIGtlZXAgY2hhbm5lbHMgc2VwYXJhdGUKICAgICAgICAgICAgZHVyYXRpb249c2VsZi5kdXJhdGlvbiwKICAgICAgICApCiAgICAgICAgIyBzZXQgc2FtcGxlIHJhdGU6CiAgICAgICAgc2VsZi5zYW1wbGVfcmF0ZSA9IGludChzcikKCiAgICAgICAgIyBjb252ZXJ0IHRvIGludCB3aXRoIHNjYWxpbmcgZm9yIDE2LWJpdCBpbnRlZ2VyCiAgICAgICAgZGF0YSAqPSAzMjc2NyAvIG5wLm1heChucC5hYnMoZGF0YSkpICAjIHJlLXNjYWxpbmcKICAgICAgICBkYXRhID0gZGF0YS5hc3R5cGUobnAuaW50MTYpICAjIGNoYW5nZSBkYXRhIHR5cGUKCiAgICAgICAgIyBzZWxlY3QgY2hhbm5lbAogICAgICAgIGRhdGFfdG9fcmVkdWNlID0gZGF0YVtzZWxmLmNoYW5uZWxdIGlmIHNlbGYuY2hhbm5lbCBpcyBub3QgTm9uZSBlbHNlIGRhdGEKICAgICAgICByZXR1cm4gZGF0YV90b19yZWR1Y2UKCiAgICBkZWYgY2xlYW5fYXVkaW8oc2VsZiwgZGF0YTogbnAubmRhcnJheSkgLT4gbnAubmRhcnJheToKICAgICAgICB0cnk6CiAgICAgICAgICAgIGltcG9ydCBub2lzZXJlZHVjZQogICAgICAgIGV4Y2VwdCBJbXBvcnRFcnJvciBhcyBlOgogICAgICAgICAgICByYWlzZSBJbXBvcnRFcnJvcigiUGxlYXNlIGluc3RhbGwgbm9pc2VyZWR1Y2UgcGFja2FnZSIpIGZyb20gZQoKICAgICAgICByZWR1Y2VkX25vaXNlID0gbm9pc2VyZWR1Y2UucmVkdWNlX25vaXNlKHk9ZGF0YSwgc3I9c2VsZi5zYW1wbGVfcmF0ZSkKCiAgICAgICAgIyBhZGQgY2hhbm5lbCBiYWNrIGFmdGVyIG5vaXNlIHJlZHVjdGlvbgogICAgICAgIGlmIHNlbGYuY2hhbm5lbCBpcyBub3QgTm9uZToKICAgICAgICAgICAgIyBwdXR0aW5nIHRoZSBjaGFubmVsIGJhY2sgaW4gdGhlIGRhdGEKICAgICAgICAgICAgZGF0YVtzZWxmLmNoYW5uZWxdID0gcmVkdWNlZF9ub2lzZQogICAgICAgICAgICAjIHVwZGF0aW5nIHRoZSBkYXRhIHRvIHNhdmUKICAgICAgICAgICAgcmVkdWNlZF9ub2lzZSA9IGRhdGEKCiAgICAgICAgcmV0dXJuIHJlZHVjZWRfbm9pc2UKCgpjbGFzcyBERk4oUmVkdWNlTm9pc2VCYXNlKToKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgIHRhcmdldF9kaXJlY3Rvcnk6IFBhdGgsCiAgICAgICAgdmVyYm9zZTogYm9vbCA9IFRydWUsCiAgICAgICAgc2lsZW5jZV90aHJlc2hvbGQ6IGZsb2F0ID0gTm9uZSwKICAgICAgICBwYWQ6IGJvb2wgPSBUcnVlLAogICAgICAgIGF0dGVuX2xpbV9kYjogaW50ID0gTm9uZSwKICAgICAgICAqKmt3YXJncywKICAgICk6CiAgICAgICAgc3VwZXIoKS5fX2luaXRfXyh0YXJnZXRfZGlyZWN0b3J5LCB2ZXJib3NlLCBzaWxlbmNlX3RocmVzaG9sZCkKICAgICAgICBzZWxmLnBhZCA9IHBhZAogICAgICAgIHNlbGYuYXR0ZW5fbGltX2RiID0gYXR0ZW5fbGltX2RiCiAgICAgICAgc2VsZi5rd2FyZ3MgPSBrd2FyZ3MKCiAgICAgICAgIyBpbXBvcnQgcmVxdWlyZWQgcGFja2FnZXMKICAgICAgICB0cnk6CiAgICAgICAgICAgIGZyb20gZGYuZW5oYW5jZSBpbXBvcnQgaW5pdF9kZgogICAgICAgIGV4Y2VwdCBJbXBvcnRFcnJvciBhcyBlOgogICAgICAgICAgICByYWlzZSBJbXBvcnRFcnJvcigiUGxlYXNlIGluc3RhbGwgZGVlcGZpbHRlcm5ldCBwYWNrYWdlcyIpIGZyb20gZQoKICAgICAgICBpZiBzZWxmLnZlcmJvc2U6CiAgICAgICAgICAgIF9MT0dHRVIuaW5mbygiTG9hZGluZyBEZWVwRmlsdGVyTmV0MiBtb2RlbC4iKQoKICAgICAgICAjIExvYWQgdGhlIG1vZGVsOgogICAgICAgIG1vZGVsLCBkZl9zdGF0ZSwgXyA9IGluaXRfZGYoKQogICAgICAgIHNlbGYubW9kZWwgPSBtb2RlbAogICAgICAgIHNlbGYuZGZfc3RhdGUgPSBkZl9zdGF0ZQogICAgICAgIHNlbGYuc2FtcGxlX3JhdGUgPSBzZWxmLmRmX3N0YXRlLnNyKCkKCiAgICBkZWYgc2F2ZV9hdWRpbyhzZWxmLCBhdWRpbzogbnAubmRhcnJheSwgdGFyZ2V0X3BhdGg6IFBhdGgpOgogICAgICAgIHRyeToKICAgICAgICAgICAgZnJvbSBkZi5lbmhhbmNlIGltcG9ydCBzYXZlX2F1ZGlvCiAgICAgICAgZXhjZXB0IEltcG9ydEVycm9yIGFzIGU6CiAgICAgICAgICAgIHJhaXNlIEltcG9ydEVycm9yKCJQbGVhc2UgaW5zdGFsbCBkZWVwZmlsdGVybmV0IHBhY2thZ2UiKSBmcm9tIGUKICAgICAgICBzYXZlX2F1ZGlvKAogICAgICAgICAgICBmaWxlPXRhcmdldF9wYXRoLm5hbWUsCiAgICAgICAgICAgIGF1ZGlvPWF1ZGlvLAogICAgICAgICAgICBzcj1zZWxmLnNhbXBsZV9yYXRlLAogICAgICAgICAgICBvdXRwdXRfZGlyPXN0cihzZWxmLnRhcmdldF9kaXJlY3RvcnkpLAogICAgICAgICkKCiAgICBkZWYgbG9hZF9hdWRpbyhzZWxmLCBmaWxlOiBzdHIpIC0+IHRvcmNoLlRlbnNvcjoKICAgICAgICB0cnk6CiAgICAgICAgICAgIGZyb20gZGYuZW5oYW5jZSBpbXBvcnQgbG9hZF9hdWRpbwogICAgICAgIGV4Y2VwdCBJbXBvcnRFcnJvciBhcyBlOgogICAgICAgICAgICByYWlzZSBJbXBvcnRFcnJvcigiUGxlYXNlIGluc3RhbGwgZGVlcGZpbHRlcm5ldCBwYWNrYWdlIikgZnJvbSBlCiAgICAgICAgYXVkaW8sIF8gPSBsb2FkX2F1ZGlvKGZpbGU9ZmlsZSwgc3I9c2VsZi5zYW1wbGVfcmF0ZSwgKipzZWxmLmt3YXJncykKICAgICAgICByZXR1cm4gYXVkaW8KCiAgICBkZWYgY2xlYW5fYXVkaW8oc2VsZiwgZGF0YTogdG9yY2guVGVuc29yKSAtPiB0b3JjaC5UZW5zb3I6CiAgICAgICAgdHJ5OgogICAgICAgICAgICBmcm9tIGRmLmVuaGFuY2UgaW1wb3J0IGVuaGFuY2UKICAgICAgICBleGNlcHQgSW1wb3J0RXJyb3IgYXMgZToKICAgICAgICAgICAgcmFpc2UgSW1wb3J0RXJyb3IoIlBsZWFzZSBpbnN0YWxsIGRlZXBmaWx0ZXJuZXQgcGFja2FnZSIpIGZyb20gZQogICAgICAgIHJldHVybiBlbmhhbmNlKAogICAgICAgICAgICBtb2RlbD1zZWxmLm1vZGVsLAogICAgICAgICAgICBkZl9zdGF0ZT1zZWxmLmRmX3N0YXRlLAogICAgICAgICAgICBhdWRpbz1kYXRhLAogICAgICAgICAgICBwYWQ9c2VsZi5wYWQsCiAgICAgICAgICAgIGF0dGVuX2xpbV9kYj1zZWxmLmF0dGVuX2xpbV9kYiwKICAgICAgICApCgoKZGVmIF9tdWx0aXByb2Nlc3NpbmdfY29tcGxldGVfdGFza3MoCiAgICBub2lzZV9yZWR1Y2VfdHlwZTogVHlwZVtSZWR1Y2VOb2lzZUJhc2VdLAogICAgbm9pc2VfcmVkdWNlX2FyZ3VtZW50czogZGljdCwKICAgIHRhc2tzX3F1ZXVlOiBRdWV1ZSwKICAgIHJlc3VsdHNfcXVldWU6IFF1ZXVlLAopOgogICAgIiIiCiAgICBDb21wbGV0ZSB0aGUgdGFza3MgaW4gdGhlIGdpdmVuIHF1ZXVlIGFuZCBwdXQgdGhlIHJlc3VsdHMgaW4gdGhlIGdpdmVuIHJlc3VsdHMgcXVldWUuIFRoZSBmdW5jdGlvbiB3aWxsIHN0b3Agd2hlbgogICAgdGhlIGdpdmVuIHRhc2tzIHF1ZXVlIHdpbGwgcmVjZWl2ZSB0aGUgc3RvcCBtYXJrLiBJdCBpcyBhaW1lZCB0byBiZSB1c2VkIHdpdGggbXVsdGlwcm9jZXNzaW5nIGFzIGEgcHJvY2Vzcy4KCiAgICA6cGFyYW0gbm9pc2VfcmVkdWNlX3R5cGU6ICAgICAgIFRoZSBub2lzZSByZWR1Y2UgdHlwZSB0byB1c2UuCiAgICA6cGFyYW0gbm9pc2VfcmVkdWNlX2FyZ3VtZW50czogIFRoZSBub2lzZXJlZHVjZSBpbml0aWFsaXphdGlvbiBrd2FyZ3MuCiAgICA6cGFyYW0gdGFza3NfcXVldWU6ICAgICAgICAgICAgIEEgcXVldWUgdG8gZ2V0IHRoZSB0YXNrcyBmcm9tLgogICAgOnBhcmFtIHJlc3VsdHNfcXVldWU6ICAgICAgICAgICBBIHF1ZXVlIHRvIHB1dCB0aGUgcmVzdWx0cyBpbi4KICAgICIiIgogICAgIyBJbml0aWFsaXplIHRoZSByZWR1Y2Ugbm9pc2Ugb2JqZWN0CiAgICBub2lzZV9yZWR1Y2VyID0gbm9pc2VfcmVkdWNlX3R5cGUoKipub2lzZV9yZWR1Y2VfYXJndW1lbnRzKQoKICAgICMgU3RhcnQgbGlzdGVuaW5nIHRvIHRoZSB0YXNrcyBxdWV1ZToKICAgIHdoaWxlIFRydWU6CiAgICAgICAgIyBHZXQgdGhlIGF1ZGlvX2ZpbGU6CiAgICAgICAgYXVkaW9fZmlsZSA9IHRhc2tzX3F1ZXVlLmdldCgpCiAgICAgICAgaWYgYXVkaW9fZmlsZSA9PSBfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSzoKICAgICAgICAgICAgYnJlYWsKICAgICAgICBhdWRpb19maWxlID0gUGF0aChhdWRpb19maWxlKQogICAgICAgICMgQXBwbHkgbm9pc2UgcmVkdWN0aW9uIGFuZCBjb2xsZWN0IHRoZSByZXN1bHQ6CiAgICAgICAgcmVzdWx0c19xdWV1ZS5wdXQobm9pc2VfcmVkdWNlci5yZWR1Y2Vfbm9pc2UoYXVkaW9fZmlsZT1hdWRpb19maWxlKSkKCiAgICAjIE1hcmsgdGhlIGVuZCBvZiB0aGUgdGFza3M6CiAgICByZXN1bHRzX3F1ZXVlLnB1dChfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSykKCgpkZWYgcmVkdWNlX25vaXNlX2RmbigKICAgIGF1ZGlvX3NvdXJjZTogc3RyLAogICAgdGFyZ2V0X2RpcmVjdG9yeTogc3RyLAogICAgcGFkOiBib29sID0gVHJ1ZSwKICAgIGF0dGVuX2xpbV9kYjogaW50ID0gTm9uZSwKICAgIHNpbGVuY2VfdGhyZXNob2xkOiBmbG9hdCA9IE5vbmUsCiAgICB1c2VfbXVsdGlwcm9jZXNzaW5nOiBpbnQgPSAwLAogICAgdmVyYm9zZTogYm9vbCA9IFRydWUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgUmVkdWNlIG5vaXNlIGZyb20gYXVkaW8gZmlsZXMgdXNpbmcgRGVlcEZpbHRlck5ldC4KICAgIEZvciBtb3JlIGluZm9ybWF0aW9uIGFib3V0IHRoZSBub2lzZSByZWR1Y3Rpb24gYWxnb3JpdGhtIHNlZToKICAgIGh0dHBzOi8vZ2l0aHViLmNvbS9SaWtvcm9zZS9EZWVwRmlsdGVyTmV0CiAgICBOb3RpY2UgdGhhdCB0aGUgc2F2ZWQgZmlsZXMgYXJlIGluIHdhdiBmb3JtYXQsIGV2ZW4gaWYgdGhlIG9yaWdpbmFsIGZpbGVzIGFyZSBpbiBvdGhlciBmb3JtYXQuCgogICAgOnBhcmFtIGF1ZGlvX3NvdXJjZTogICAgICAgIHBhdGggdG8gYXVkaW8gZmlsZSBvciBkaXJlY3Rvcnkgb2YgYXVkaW8gZmlsZXMKICAgIDpwYXJhbSB0YXJnZXRfZGlyZWN0b3J5OiAgICBwYXRoIHRvIHRhcmdldCBkaXJlY3RvcnkgdG8gc2F2ZSBjbGVhbmVkIGF1ZGlvIGZpbGVzCiAgICA6cGFyYW0gcGFkOiAgICAgICAgICAgICAgICAgd2hldGhlciB0byBwYWQgdGhlIGF1ZGlvIGZpbGUgd2l0aCB6ZXJvcyBiZWZvcmUgY2xlYW5pbmcKICAgIDpwYXJhbSBhdHRlbl9saW1fZGI6ICAgICAgICBtYXhpbXVtIGF0dGVudWF0aW9uIGluIGRCCiAgICA6cGFyYW0gc2lsZW5jZV90aHJlc2hvbGQ6ICAgdGhlIHRocmVzaG9sZCB0byByZW1vdmUgc2lsZW5jZSBmcm9tIHRoZSBhdWRpbywgaW4gZEIuIElmIE5vbmUsIG5vIHNpbGVuY2UgcmVtb3ZhbCBpcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHBlcmZvcm1lZC4KICAgIDpwYXJhbSB1c2VfbXVsdGlwcm9jZXNzaW5nOiBOdW1iZXIgb2YgcHJvY2Vzc2VzIHRvIHVzZSBmb3IgY2xlYW5pbmcgdGhlIGF1ZGlvIGZpbGVzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIElmIDAsIG5vIG11bHRpcHJvY2Vzc2luZyBpcyB1c2VkLgogICAgOnBhcmFtIHZlcmJvc2U6ICAgICAgICAgICAgIHZlcmJvc2l0eSBsZXZlbC4gSWYgVHJ1ZSwgZGlzcGxheSBwcm9ncmVzcyBiYXIgYW5kIGxvZ3MuCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgYWRkaXRpb25hbCBhcmd1bWVudHMgdG8gcGFzcyB0byB0b3JjaGF1ZGlvLmxvYWQoKS4gRm9yIG1vcmUgaW5mb3JtYXRpb24gc2VlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGh0dHBzOi8vcHl0b3JjaC5vcmcvYXVkaW8vc3RhYmxlL2dlbmVyYXRlZC90b3JjaGF1ZGlvLmxvYWQuaHRtbAogICAgIiIiCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiUmVkdWNpbmcgbm9pc2UgZnJvbSBhdWRpbyBmaWxlcy4iKQoKICAgICMgY3JlYXRlIHRhcmdldCBkaXJlY3Rvcnk6CiAgICB0YXJnZXRfZGlyZWN0b3J5ID0gX2NyZWF0ZV90YXJnZXRfZGlyZWN0b3J5KHRhcmdldF9kaXJlY3RvcnkpCgogICAgIyBnZXQgYXVkaW8gZmlsZXM6CiAgICBhdWRpb19maWxlcyA9IF9nZXRfYXVkaW9fZmlsZXMoYXVkaW9fc291cmNlKQoKICAgIG5vaXNlX3JlZHVjZV9hcmd1bWVudHMgPSB7CiAgICAgICAgInRhcmdldF9kaXJlY3RvcnkiOiB0YXJnZXRfZGlyZWN0b3J5LAogICAgICAgICJwYWQiOiBwYWQsCiAgICAgICAgImF0dGVuX2xpbV9kYiI6IGF0dGVuX2xpbV9kYiwKICAgICAgICAic2lsZW5jZV90aHJlc2hvbGQiOiBzaWxlbmNlX3RocmVzaG9sZCwKICAgICAgICAqKmt3YXJncywKICAgIH0KCiAgICBpZiB1c2VfbXVsdGlwcm9jZXNzaW5nOgogICAgICAgIHJlc3VsdHMgPSBfcGFyYWxsZWxfcnVuKAogICAgICAgICAgICBub2lzZV9yZWR1Y2VfdHlwZT1ERk4sCiAgICAgICAgICAgIG5vaXNlX3JlZHVjZV9hcmd1bWVudHM9bm9pc2VfcmVkdWNlX2FyZ3VtZW50cywKICAgICAgICAgICAgbl93b3JrZXJzPXVzZV9tdWx0aXByb2Nlc3NpbmcsCiAgICAgICAgICAgIGF1ZGlvX2ZpbGVzPWF1ZGlvX2ZpbGVzLAogICAgICAgICAgICBkZXNjcmlwdGlvbj0iTm9pc2UtcmVkdWN0aW9uIiwKICAgICAgICAgICAgdmVyYm9zZT12ZXJib3NlLAogICAgICAgICkKICAgIGVsc2U6CiAgICAgICAgcmVzdWx0cyA9IF9ydW4oCiAgICAgICAgICAgIG5vaXNlX3JlZHVjZV90eXBlPURGTiwKICAgICAgICAgICAgbm9pc2VfcmVkdWNlX2FyZ3VtZW50cz1ub2lzZV9yZWR1Y2VfYXJndW1lbnRzLAogICAgICAgICAgICBhdWRpb19maWxlcz1hdWRpb19maWxlcywKICAgICAgICAgICAgZGVzY3JpcHRpb249Ik5vaXNlLXJlZHVjdGlvbiIsCiAgICAgICAgICAgIHZlcmJvc2U9dmVyYm9zZSwKICAgICAgICApCgogICAgcmV0dXJuIF9wcm9jZXNzX3Jlc3VsdHMocmVzdWx0cywgdmVyYm9zZSkKCgpkZWYgcmVkdWNlX25vaXNlKAogICAgYXVkaW9fc291cmNlOiBzdHIsCiAgICB0YXJnZXRfZGlyZWN0b3J5OiBzdHIsCiAgICBzYW1wbGVfcmF0ZTogaW50ID0gMTYwMDAsCiAgICBkdXJhdGlvbjogaW50ID0gTm9uZSwKICAgIGNoYW5uZWw6IGludCA9IE5vbmUsCiAgICBzaWxlbmNlX3RocmVzaG9sZDogZmxvYXQgPSBOb25lLAogICAgdXNlX211bHRpcHJvY2Vzc2luZzogaW50ID0gMCwKICAgIHZlcmJvc2U6IGJvb2wgPSBUcnVlLAopOgogICAgIiIiCiAgICBSZWR1Y2Ugbm9pc2UgZnJvbSBhdWRpbyBmaWxlIG9yIGRpcmVjdG9yeSBjb250YWluaW5nIGF1ZGlvIGZpbGVzLgogICAgVGhlIGF1ZGlvIGZpbGVzIG11c3QgYmUgaW4gLndhdiBmb3JtYXQuCiAgICBUaGUgY2xlYW5lZCBhdWRpbyBmaWxlcyB3aWxsIGJlIHNhdmVkIGluIHRoZSB0YXJnZXRfZGlyZWN0b3J5LgogICAgRm9yIGluZm9ybWF0aW9uIGFib3V0IHRoZSBub2lzZSByZWR1Y3Rpb24gYWxnb3JpdGhtIHNlZToKICAgIGh0dHBzOi8vZ2l0aHViLmNvbS90aW1zYWluYi9ub2lzZXJlZHVjZQogICAgTm90aWNlIHRoYXQgdGhlIHNhdmVkIGZpbGVzIGFyZSBpbiB3YXYgZm9ybWF0LCBldmVuIGlmIHRoZSBvcmlnaW5hbCBmaWxlcyBhcmUgaW4gb3RoZXIgZm9ybWF0LgoKICAgIDpwYXJhbSBhdWRpb19zb3VyY2U6ICAgICAgICBwYXRoIHRvIGF1ZGlvIGZpbGUgb3IgZGlyZWN0b3J5IGNvbnRhaW5pbmcgYXVkaW8gZmlsZXMKICAgIDpwYXJhbSB0YXJnZXRfZGlyZWN0b3J5OiAgICBwYXRoIHRvIGRpcmVjdG9yeSB0byBzYXZlIHRoZSBjbGVhbmVkIGF1ZGlvIGZpbGVzLgogICAgOnBhcmFtIHNhbXBsZV9yYXRlOiAgICAgICAgIE51bWJlciBvZiBzYW1wbGVzIGluIG9uZSBzZWNvbmQgaW4gdGhlIGF1ZGlvIGZpbGUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgUGFzcyBgTm9uZWAgdG8ga2VlcCB0aGUgb3JpZ2luYWwgc2FtcGxlIHJhdGUuCiAgICA6cGFyYW0gZHVyYXRpb246ICAgICAgICAgICAgRHVyYXRpb24gb2YgdGhlIGF1ZGlvIGZpbGUgdG8gY2xlYW4gaW4gc2Vjb25kcy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBQYXNzIGBOb25lYCB0byBrZWVwIHRoZSBvcmlnaW5hbCBkdXJhdGlvbi4KICAgIDpwYXJhbSBjaGFubmVsOiAgICAgICAgICAgICBDaGFubmVsIHRvIGNsZWFuLiBQYXNzIHRoZSBudW1iZXIgb2YgdGhlIGNoYW5uZWwgdG8gY2xlYW4uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgVG8gY2xlYW4gYWxsIGNoYW5uZWxzIHBhc3MgTm9uZS4KICAgIDpwYXJhbSBzaWxlbmNlX3RocmVzaG9sZDogICBUaGUgdGhyZXNob2xkIHRvIHJlbW92ZSBzaWxlbmNlIGZyb20gdGhlIGF1ZGlvLCBpbiBkQi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBOb25lLCBubyBzaWxlbmNlIHJlbW92YWwgaXMgcGVyZm9ybWVkLgogICAgOnBhcmFtIHVzZV9tdWx0aXByb2Nlc3Npbmc6IE51bWJlciBvZiBwcm9jZXNzZXMgdG8gdXNlIGZvciBjbGVhbmluZyB0aGUgYXVkaW8gZmlsZXMuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgMCwgbm8gbXVsdGlwcm9jZXNzaW5nIGlzIHVzZWQuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICAgVmVyYm9zaXR5IGxldmVsLiBJZiBUcnVlLCBkaXNwbGF5IHByb2dyZXNzIGJhci4KICAgICIiIgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIlJlZHVjaW5nIG5vaXNlIGZyb20gYXVkaW8gZmlsZXMuIikKCiAgICAjIGNyZWF0ZSB0YXJnZXQgZGlyZWN0b3J5OgogICAgdGFyZ2V0X2RpcmVjdG9yeSA9IF9jcmVhdGVfdGFyZ2V0X2RpcmVjdG9yeSh0YXJnZXRfZGlyZWN0b3J5KQoKICAgICMgZ2V0IGF1ZGlvIGZpbGVzOgogICAgYXVkaW9fZmlsZXMgPSBfZ2V0X2F1ZGlvX2ZpbGVzKGF1ZGlvX3NvdXJjZSkKCiAgICAjIENyZWF0ZSB0aGUgcmVkdWNlIG5vaXNlIG9iamVjdDoKICAgIG5vaXNlX3JlZHVjZV9hcmd1bWVudHMgPSB7CiAgICAgICAgInRhcmdldF9kaXJlY3RvcnkiOiB0YXJnZXRfZGlyZWN0b3J5LAogICAgICAgICJzYW1wbGVfcmF0ZSI6IHNhbXBsZV9yYXRlLAogICAgICAgICJkdXJhdGlvbiI6IGR1cmF0aW9uLAogICAgICAgICJjaGFubmVsIjogY2hhbm5lbCwKICAgICAgICAic2lsZW5jZV90aHJlc2hvbGQiOiBzaWxlbmNlX3RocmVzaG9sZCwKICAgIH0KCiAgICBpZiB1c2VfbXVsdGlwcm9jZXNzaW5nOgogICAgICAgIHJlc3VsdHMgPSBfcGFyYWxsZWxfcnVuKAogICAgICAgICAgICBub2lzZV9yZWR1Y2VfdHlwZT1SZWR1Y2VOb2lzZSwKICAgICAgICAgICAgbm9pc2VfcmVkdWNlX2FyZ3VtZW50cz1ub2lzZV9yZWR1Y2VfYXJndW1lbnRzLAogICAgICAgICAgICBuX3dvcmtlcnM9dXNlX211bHRpcHJvY2Vzc2luZywKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJOb2lzZS1yZWR1Y3Rpb24iLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXN1bHRzID0gX3J1bigKICAgICAgICAgICAgbm9pc2VfcmVkdWNlX3R5cGU9UmVkdWNlTm9pc2UsCiAgICAgICAgICAgIG5vaXNlX3JlZHVjZV9hcmd1bWVudHM9bm9pc2VfcmVkdWNlX2FyZ3VtZW50cywKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJOb2lzZS1yZWR1Y3Rpb24iLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQoKICAgIHJldHVybiBfcHJvY2Vzc19yZXN1bHRzKHJlc3VsdHMsIHZlcmJvc2UpCgoKZGVmIF9jcmVhdGVfdGFyZ2V0X2RpcmVjdG9yeSh0YXJnZXRfZGlyZWN0b3J5OiBzdHIpIC0+IHN0cjoKICAgIHRhcmdldF9kaXJlY3RvcnkgPSBQYXRoKHRhcmdldF9kaXJlY3RvcnkpCiAgICBpZiBub3QgdGFyZ2V0X2RpcmVjdG9yeS5leGlzdHMoKToKICAgICAgICB0YXJnZXRfZGlyZWN0b3J5Lm1rZGlyKHBhcmVudHM9VHJ1ZSwgZXhpc3Rfb2s9VHJ1ZSkKICAgIHJldHVybiBzdHIodGFyZ2V0X2RpcmVjdG9yeSkKCgpkZWYgX2dldF9hdWRpb19maWxlcyhhdWRpb19zb3VyY2U6IHN0cik6CiAgICBhdWRpb19zb3VyY2UgPSBQYXRoKGF1ZGlvX3NvdXJjZSkKICAgIGF1ZGlvX2ZpbGVzID0gW10KICAgIGlmIGF1ZGlvX3NvdXJjZS5pc19kaXIoKToKICAgICAgICBhdWRpb19maWxlcyA9IGxpc3QoYXVkaW9fc291cmNlLmdsb2IoIiouKiIpKQogICAgZWxpZiBhdWRpb19zb3VyY2UuaXNfZmlsZSgpOgogICAgICAgIGF1ZGlvX2ZpbGVzLmFwcGVuZChhdWRpb19zb3VyY2UpCiAgICBlbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgIGYiYXVkaW9fc291cmNlIG11c3QgYmUgYSBmaWxlIG9yIGEgZGlyZWN0b3J5LCBnb3Qge2F1ZGlvX3NvdXJjZX0iCiAgICAgICAgKQogICAgcmV0dXJuIGF1ZGlvX2ZpbGVzCgoKZGVmIF9wYXJhbGxlbF9ydW4oCiAgICBub2lzZV9yZWR1Y2VfdHlwZTogVHlwZVtSZWR1Y2VOb2lzZUJhc2VdLAogICAgbm9pc2VfcmVkdWNlX2FyZ3VtZW50czogZGljdCwKICAgIG5fd29ya2VyczogaW50LAogICAgYXVkaW9fZmlsZXM6IExpc3RbUGF0aF0sCiAgICBkZXNjcmlwdGlvbjogc3RyLAogICAgdmVyYm9zZTogYm9vbCwKKSAtPiBMaXN0W1R1cGxlW2Jvb2wsIFR1cGxlW3N0ciwgc3RyXV1dOgogICAgIiIiCiAgICBSdW4gbXVsdGlwbGUgbm9pc2UgcmVkdWNlIHdvcmtlcnMgd2l0aCBtdWx0aXByb2Nlc3NpbmcgdG8gY29tcGxldGUgdGhlIHRhc2tzIHRoYXQgd2lsbCBiZSBjcmVhdGVkIG9uIHRoZSBwcm92aWRlZAogICAgZmlsZXMgdXNpbmcgdGhlIGdpdmVuIHRhc2sgY3JlYXRvci4KCiAgICA6cGFyYW0gbm9pc2VfcmVkdWNlX3R5cGU6ICAgVGhlIG5vaXNlIHJlZHVjZSB0eXBlIHRvIHVzZS4KICAgIDpwYXJhbSBuX3dvcmtlcnM6ICAgICAgICAgICBUaGUgbnVtYmVyIG9mIHdvcmtlcnMgdG8gdXNlLgogICAgOnBhcmFtIGF1ZGlvX2ZpbGVzOiAgICAgICAgIFRoZSBhdWRpbyBmaWxlcyB0byB1c2UuCiAgICA6cGFyYW0gZGVzY3JpcHRpb246ICAgICAgICAgVGhlIGRlc2NyaXB0aW9uIHRvIHVzZSBmb3IgdGhlIHByb2dyZXNzIGJhci4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICBWZXJib3NpdHkuCgogICAgOnJldHVybnM6IFRoZSBjb2xsZWN0ZWQgcmVzdWx0cy4KICAgICIiIgogICAgIyBDaGVjayB0aGUgbnVtYmVyIG9mIHdvcmtlcnM6CiAgICBpZiBuX3dvcmtlcnMgPiBsZW4oYXVkaW9fZmlsZXMpOgogICAgICAgIF9MT0dHRVIud2FybmluZygKICAgICAgICAgICAgZiJUaGUgbnVtYmVyIG9mIHdvcmtlcnMgKHtuX3dvcmtlcnN9KSBpcyBsYXJnZXIgdGhhbiB0aGUgbnVtYmVyIG9mIGF1ZGlvIGZpbGVzICh7bGVuKGF1ZGlvX2ZpbGVzKX0pLiAiCiAgICAgICAgICAgIGYiU2V0dGluZyB0aGUgbnVtYmVyIG9mIHdvcmtlcnMgdG8ge2xlbihhdWRpb19maWxlcyl9LiIKICAgICAgICApCiAgICAgICAgbl93b3JrZXJzID0gbGVuKGF1ZGlvX2ZpbGVzKQoKICAgICMgSW5pdGlhbGl6ZSB0aGUgbXVsdGlwcm9jZXNzaW5nIHF1ZXVlczoKICAgIHRhc2tzX3F1ZXVlID0gUXVldWUoKQogICAgcmVzdWx0c19xdWV1ZSA9IFF1ZXVlKCkKCiAgICAjIEluaXRpYWxpemUgdGhlIG11bHRpcHJvY2Vzc2luZyBwcm9jZXNzZXM6CiAgICB0YXNrX2NvbXBsZXRpb25fcHJvY2Vzc2VzID0gWwogICAgICAgIFByb2Nlc3MoCiAgICAgICAgICAgIHRhcmdldD1fbXVsdGlwcm9jZXNzaW5nX2NvbXBsZXRlX3Rhc2tzLAogICAgICAgICAgICBrd2FyZ3M9ewogICAgICAgICAgICAgICAgIm5vaXNlX3JlZHVjZV90eXBlIjogbm9pc2VfcmVkdWNlX3R5cGUsCiAgICAgICAgICAgICAgICAibm9pc2VfcmVkdWNlX2FyZ3VtZW50cyI6IG5vaXNlX3JlZHVjZV9hcmd1bWVudHMsCiAgICAgICAgICAgICAgICAidGFza3NfcXVldWUiOiB0YXNrc19xdWV1ZSwKICAgICAgICAgICAgICAgICJyZXN1bHRzX3F1ZXVlIjogcmVzdWx0c19xdWV1ZSwKICAgICAgICAgICAgfSwKICAgICAgICApCiAgICAgICAgZm9yIF8gaW4gcmFuZ2Uobl93b3JrZXJzKQogICAgXQoKICAgICMgU3RhcnQgdGhlIG11bHRpcHJvY2Vzc2luZyBwcm9jZXNzZXM6CiAgICBmb3IgcCBpbiB0YXNrX2NvbXBsZXRpb25fcHJvY2Vzc2VzOgogICAgICAgIHAuc3RhcnQoKQoKICAgICMgUHV0IHRoZSB0YXNrcyBpbiB0aGUgcXVldWU6CiAgICBmb3IgYXVkaW9fZmlsZSBpbiBhdWRpb19maWxlczoKICAgICAgICAjIHRhc2tzX3F1ZXVlLnB1dCh0YXNrX2NyZWF0b3IuY3JlYXRlX3Rhc2soYXVkaW9fZmlsZT1hdWRpb19maWxlKS50b190dXBsZSgpKQogICAgICAgIHRhc2tzX3F1ZXVlLnB1dChhdWRpb19maWxlKQoKICAgICMgUHV0IHRoZSBzdG9wIG1hcmtzIGluIHRoZSBxdWV1ZToKICAgIGZvciBfIGluIHJhbmdlKG5fd29ya2Vycyk6CiAgICAgICAgdGFza3NfcXVldWUucHV0KF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLKQoKICAgICMgQ29sbGVjdCB0aGUgcmVzdWx0czoKICAgIHJlc3VsdHMgPSBbXQogICAgc3RvcF9tYXJrc19jb3VudGVyID0gMAogICAgd2l0aCB0cWRtKAogICAgICAgIGRlc2M9ZGVzY3JpcHRpb24sCiAgICAgICAgdW5pdD0iZmlsZSIsCiAgICAgICAgdG90YWw9bGVuKGF1ZGlvX2ZpbGVzKSwKICAgICAgICBkaXNhYmxlPW5vdCB2ZXJib3NlLAogICAgKSBhcyBwcm9ncmVzc2JhcjoKICAgICAgICB3aGlsZSBUcnVlOgogICAgICAgICAgICAjIEdldCBhIHJlc3VsdCBmcm9tIHRoZSBxdWV1ZToKICAgICAgICAgICAgcmVzdWx0OiBUdXBsZVtib29sLCBUdXBsZVtzdHIsIHN0cl1dID0gcmVzdWx0c19xdWV1ZS5nZXQoKQogICAgICAgICAgICBpZiByZXN1bHQgPT0gX01VTFRJUFJPQ0VTU0lOR19TVE9QX01BUks6CiAgICAgICAgICAgICAgICBzdG9wX21hcmtzX2NvdW50ZXIgKz0gMQogICAgICAgICAgICAgICAgaWYgc3RvcF9tYXJrc19jb3VudGVyID09IG5fd29ya2VyczoKICAgICAgICAgICAgICAgICAgICBicmVhawogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgIyBDb2xsZWN0IHRoZSByZXN1bHQ6CiAgICAgICAgICAgICAgICByZXN1bHRzLmFwcGVuZChyZXN1bHQpCiAgICAgICAgICAgICAgICBwcm9ncmVzc2Jhci51cGRhdGUoMSkKCiAgICAjIFdhaXQgZm9yIHRoZSBwcm9jZXNzZXMgdG8gZmluaXNoOgogICAgZm9yIHAgaW4gdGFza19jb21wbGV0aW9uX3Byb2Nlc3NlczoKICAgICAgICBwLmpvaW4oKQoKICAgIHJldHVybiByZXN1bHRzCgoKZGVmIF9ydW4oCiAgICBub2lzZV9yZWR1Y2VfdHlwZTogVHlwZVtSZWR1Y2VOb2lzZUJhc2VdLAogICAgbm9pc2VfcmVkdWNlX2FyZ3VtZW50czogZGljdCwKICAgIGF1ZGlvX2ZpbGVzOiBMaXN0W1BhdGhdLAogICAgZGVzY3JpcHRpb246IHN0ciwKICAgIHZlcmJvc2U6IGJvb2wsCikgLT4gTGlzdFtUdXBsZVtib29sLCBUdXBsZVtzdHIsIHN0cl1dXToKICAgICIiIgogICAgUnVuIHRoZSBub2lzZSByZWR1Y2UgYWxnb3JpdGhtIG9uIHRoZSBnaXZlbiBhdWRpbyBmaWxlcyBhbmQgY29sbGVjdCB0aGUgcmVzdWx0cy4KCiAgICA6cGFyYW0gbm9pc2VfcmVkdWNlX3R5cGU6ICAgICAgIFRoZSBub2lzZSByZWR1Y2UgdHlwZSB0byB1c2UuCiAgICA6cGFyYW0gbm9pc2VfcmVkdWNlX2FyZ3VtZW50czogIFRoZSBub2lzZXJlZHVjZSBpbml0aWFsaXphdGlvbiBrd2FyZ3MuCiAgICA6cGFyYW0gYXVkaW9fZmlsZXM6ICAgICAgICAgICAgIFRoZSBhdWRpbyBmaWxlcyB0byB1c2UuCiAgICA6cGFyYW0gZGVzY3JpcHRpb246ICAgICAgICAgICAgIFRoZSBkZXNjcmlwdGlvbiB0byB1c2UgZm9yIHRoZSBwcm9ncmVzcyBiYXIuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICAgICAgIFZlcmJvc2l0eS4KCiAgICA6cmV0dXJuczogVGhlIGNvbGxlY3RlZCByZXN1bHRzLgogICAgIiIiCiAgICAjIENyZWF0ZSB0aGUgcmVkdWNlIG5vaXNlIG9iamVjdDoKICAgIG5vaXNlX3JlZHVjZXIgPSBub2lzZV9yZWR1Y2VfdHlwZSgqKm5vaXNlX3JlZHVjZV9hcmd1bWVudHMpCgogICAgIyBSdW4gdGhlIG5vaXNlIHJlZHVjZSBhbGdvcml0aG0gb24gdGhlIGF1ZGlvIGZpbGVzIGFuZCBjb2xsZWN0IHRoZSByZXN1bHRzOgogICAgcmVzdWx0cyA9IFtdCiAgICBmb3IgYXVkaW9fZmlsZSBpbiB0cWRtKAogICAgICAgIGF1ZGlvX2ZpbGVzLAogICAgICAgIGRlc2M9ZGVzY3JpcHRpb24sCiAgICAgICAgdW5pdD0iZmlsZSIsCiAgICAgICAgdG90YWw9bGVuKGF1ZGlvX2ZpbGVzKSwKICAgICAgICBkaXNhYmxlPW5vdCB2ZXJib3NlLAogICAgKToKICAgICAgICByZXN1bHRzLmFwcGVuZChub2lzZV9yZWR1Y2VyLnJlZHVjZV9ub2lzZShhdWRpb19maWxlPWF1ZGlvX2ZpbGUpKQoKICAgIHJldHVybiByZXN1bHRzCgoKZGVmIF9wcm9jZXNzX3Jlc3VsdHMoCiAgICByZXN1bHRzOiBMaXN0W1R1cGxlW2Jvb2wsIFR1cGxlW3N0ciwgc3RyXV1dLCB2ZXJib3NlOiBib29sCikgLT4gVHVwbGVbZGljdCwgZGljdF06CiAgICAiIiIKICAgIFByb2Nlc3MgdGhlIHJlc3VsdHMgb2YgdGhlIHRhc2tzLgoKICAgIDpwYXJhbSByZXN1bHRzOiBUaGUgcmVzdWx0cyB0byBwcm9jZXNzLgogICAgOnBhcmFtIHZlcmJvc2U6IFZlcmJvc2l0eS4KCiAgICA6cmV0dXJuczogVGhlIHByb2Nlc3NlZCByZXN1bHRzIGFzIGEgdHVwbGUgb2Ygc3VjY2Vzc2VzIGFuZCBlcnJvcnMuCiAgICAiIiIKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJTdW1tYXJpemluZyB0aGUgcmVzdWx0cy4iKQogICAgc3VjY2Vzc2VzID0ge30KICAgIGVycm9ycyA9IHt9CiAgICBmb3IgaXNfZXJyb3IsIHJlc3VsdCBpbiByZXN1bHRzOgogICAgICAgIGlmIGlzX2Vycm9yOgogICAgICAgICAgICBlcnJvcnNbcmVzdWx0WzBdXSA9IHJlc3VsdFsxXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHN1Y2Nlc3Nlc1tyZXN1bHRbMF1dID0gcmVzdWx0WzFdCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkRvbmUgKHtsZW4oc3VjY2Vzc2VzKX0ve2xlbihzdWNjZXNzZXMpICsgbGVuKGVycm9ycyl9KVxuIikKCiAgICByZXR1cm4gc3VjY2Vzc2VzLCBlcnJvcnMK - origin_filename: '' - description: Reduce noise from audio files command: '' - image: '' + description: Reduce noise from audio files default_handler: reduce_noise - disable_auto_mount: false -metadata: - name: noise-reduction - tag: '' - categories: - - data-preparation - - audio -kind: job -verbose: false diff --git a/functions/src/noise_reduction/noise_reduction.py b/functions/src/noise_reduction/noise_reduction.py index f0fff5504..c9184922d 100644 --- a/functions/src/noise_reduction/noise_reduction.py +++ b/functions/src/noise_reduction/noise_reduction.py @@ -2,7 +2,6 @@ from abc import ABCMeta, abstractmethod from multiprocessing import Process, Queue from pathlib import Path -from typing import List, Tuple, Type, Union import librosa import numpy as np @@ -33,6 +32,7 @@ class ReduceNoiseBase(metaclass=ABCMeta): After implementing the above methods, you can use the reduce_noise method to reduce noise from audio files. """ + def __init__( self, target_directory: Path, @@ -43,7 +43,7 @@ def __init__( self.verbose = verbose self.silence_threshold = silence_threshold - def reduce_noise(self, audio_file: Path) -> Tuple[bool, Tuple[str, str]]: + def reduce_noise(self, audio_file: Path) -> tuple[bool, tuple[str, str]]: """ Reduce noise from the given audio file. @@ -89,7 +89,7 @@ def reduce_noise(self, audio_file: Path) -> Tuple[bool, Tuple[str, str]]: return True, (audio_file.name, str(exception)) @abstractmethod - def clean_audio(self, data) -> Union[np.ndarray, torch.Tensor]: + def clean_audio(self, data) -> np.ndarray | torch.Tensor: """ Clean the audio from noise. Here you should implement the noise reduction algorithm. @@ -110,7 +110,7 @@ def save_audio(self, audio: np.ndarray, target_path: Path): pass @abstractmethod - def load_audio(self, file: str) -> Tuple[Union[np.ndarray, torch.Tensor], int]: + def load_audio(self, file: str) -> tuple[np.ndarray | torch.Tensor, int]: """ Load the audio from a file. @@ -288,7 +288,7 @@ def clean_audio(self, data: torch.Tensor) -> torch.Tensor: def _multiprocessing_complete_tasks( - noise_reduce_type: Type[ReduceNoiseBase], + noise_reduce_type: type[ReduceNoiseBase], noise_reduce_arguments: dict, tasks_queue: Queue, results_queue: Queue, @@ -478,13 +478,13 @@ def _get_audio_files(audio_source: str): def _parallel_run( - noise_reduce_type: Type[ReduceNoiseBase], + noise_reduce_type: type[ReduceNoiseBase], noise_reduce_arguments: dict, n_workers: int, - audio_files: List[Path], + audio_files: list[Path], description: str, verbose: bool, -) -> List[Tuple[bool, Tuple[str, str]]]: +) -> list[tuple[bool, tuple[str, str]]]: """ Run multiple noise reduce workers with multiprocessing to complete the tasks that will be created on the provided files using the given task creator. @@ -547,7 +547,7 @@ def _parallel_run( ) as progressbar: while True: # Get a result from the queue: - result: Tuple[bool, Tuple[str, str]] = results_queue.get() + result: tuple[bool, tuple[str, str]] = results_queue.get() if result == _MULTIPROCESSING_STOP_MARK: stop_marks_counter += 1 if stop_marks_counter == n_workers: @@ -565,12 +565,12 @@ def _parallel_run( def _run( - noise_reduce_type: Type[ReduceNoiseBase], + noise_reduce_type: type[ReduceNoiseBase], noise_reduce_arguments: dict, - audio_files: List[Path], + audio_files: list[Path], description: str, verbose: bool, -) -> List[Tuple[bool, Tuple[str, str]]]: +) -> list[tuple[bool, tuple[str, str]]]: """ Run the noise reduce algorithm on the given audio files and collect the results. @@ -600,8 +600,8 @@ def _run( def _process_results( - results: List[Tuple[bool, Tuple[str, str]]], verbose: bool -) -> Tuple[dict, dict]: + results: list[tuple[bool, tuple[str, str]]], verbose: bool +) -> tuple[dict, dict]: """ Process the results of the tasks. diff --git a/functions/src/onnx_utils/function.yaml b/functions/src/onnx_utils/function.yaml index 023c034d3..c163f0e5a 100644 --- a/functions/src/onnx_utils/function.yaml +++ b/functions/src/onnx_utils/function.yaml @@ -1,17 +1,18 @@ -kind: job metadata: + tag: '' + name: onnx-utils categories: - utils - deep-learning - name: onnx-utils - tag: '' verbose: false +kind: job spec: + image: '' + disable_auto_mount: false build: - code_origin: '' - base_image: mlrun/mlrun origin_filename: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgQ2FsbGFibGUsIERpY3QsIExpc3QsIFR1cGxlCgppbXBvcnQgbWxydW4KCgpjbGFzcyBfVG9PTk5YQ29udmVyc2lvbnM6CiAgICAiIiIKICAgIEFuIE9OTlggY29udmVyc2lvbiBmdW5jdGlvbnMgbGlicmFyeSBjbGFzcy4KICAgICIiIgoKICAgIEBzdGF0aWNtZXRob2QKICAgIGRlZiB0Zl9rZXJhc190b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50XSwgc3RyXV0gPSBOb25lLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBDb252ZXJ0IGEgVEYuS2VyYXMgbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICBBbiBpbml0aWFsaXplZCBURktlcmFzTW9kZWxIYW5kbGVyIHdpdGggYSBsb2FkZWQgbW9kZWwgdG8gY29udmVydCB0byBPTk5YLgogICAgICAgIDpwYXJhbSBvbm54X21vZGVsX25hbWU6IFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbiBgbW9kZWxfbmFtZWAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICBXaGV0aGVyIG9yIG5vdCB0byBvcHRpbWl6ZSB0aGUgT05OWCBtb2RlbCB1c2luZyAnb25ueG9wdGltaXplcicgYmVmb3JlIHNhdmluZyB0aGUgbW9kZWwuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgRGVmYXVsdGVkIHRvIFRydWUuCiAgICAgICAgOnBhcmFtIGlucHV0X3NpZ25hdHVyZTogQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEgbGlzdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHdoZXJlIGVhY2ggZWxlbWVudCBpcyBhbiBpbnB1dCBsYXllciB0dXBsZS4gQW4gaW5wdXQgbGF5ZXIgdHVwbGUgaXMgYSB0dXBsZSBvZjoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBbMF0gPSBMYXllcidzIHNoYXBlLCBhIHR1cGxlIG9mIGludGVnZXJzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBOb25lLCB0aGUgaW5wdXQgc2lnbmF0dXJlIHdpbGwgYmUgdHJpZWQgdG8gYmUgcmVhZCBmcm9tIHRoZSBtb2RlbCBhcnRpZmFjdC4gRGVmYXVsdGVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG8gTm9uZS4KICAgICAgICAiIiIKICAgICAgICAjIEltcG9ydCB0aGUgZnJhbWV3b3JrIGFuZCBoYW5kbGVyOgogICAgICAgIGltcG9ydCB0ZW5zb3JmbG93IGFzIHRmCiAgICAgICAgZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLnRmX2tlcmFzIGltcG9ydCBURktlcmFzVXRpbHMKCiAgICAgICAgIyBDaGVjayB0aGUgZ2l2ZW4gJ2lucHV0X3NpZ25hdHVyZScgcGFyYW1ldGVyOgogICAgICAgIGlmIGlucHV0X3NpZ25hdHVyZSBpcyBOb25lOgogICAgICAgICAgICAjIFJlYWQgdGhlIGlucHV0cyBmcm9tIHRoZSBtb2RlbDoKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgbW9kZWxfaGFuZGxlci5yZWFkX2lucHV0c19mcm9tX21vZGVsKCkKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlcnJvcjoKICAgICAgICAgICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1blJ1bnRpbWVFcnJvcigKICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSBwcm92aWRlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXIuIFRoZSBmdW5jdGlvbiB0cmllZCByZWFkaW5nIHRoZSBpbnB1dCBsYXllcnMgIgogICAgICAgICAgICAgICAgICAgIGYiaW5mb3JtYXRpb24gYXV0b21hdGljYWxseSBidXQgZmFpbGVkIHdpdGggdGhlIGZvbGxvd2luZyBlcnJvcjoge2Vycm9yfSIKICAgICAgICAgICAgICAgICkKICAgICAgICBlbHNlOgogICAgICAgICAgICAjIFBhcnNlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXI6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IFsKICAgICAgICAgICAgICAgIHRmLlRlbnNvclNwZWMoCiAgICAgICAgICAgICAgICAgICAgc2hhcGU9c2hhcGUsCiAgICAgICAgICAgICAgICAgICAgZHR5cGU9VEZLZXJhc1V0aWxzLmNvbnZlcnRfdmFsdWVfdHlwZV90b190Zl9kdHlwZSgKICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICBdCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICkKCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgcHl0b3JjaF90b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50LCAuLi5dLCBzdHJdXSA9IE5vbmUsCiAgICAgICAgaW5wdXRfbGF5ZXJzX25hbWVzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgICAgIG91dHB1dF9sYXllcnNfbmFtZXM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICAgICAgZHluYW1pY19heGVzOiBEaWN0W3N0ciwgRGljdFtpbnQsIHN0cl1dID0gTm9uZSwKICAgICAgICBpc19iYXRjaGVkOiBib29sID0gVHJ1ZSwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCBhIFB5VG9yY2ggbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICAgICAgQW4gaW5pdGlhbGl6ZWQgUHlUb3JjaE1vZGVsSGFuZGxlciB3aXRoIGEgbG9hZGVkIG1vZGVsIHRvIGNvbnZlcnQgdG8gT05OWC4KICAgICAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiAgICAgVGhlIG5hbWUgdG8gdXNlIHRvIGxvZyB0aGUgY29udmVydGVkIE9OTlggbW9kZWwuIElmIG5vdCBnaXZlbiwgdGhlIGdpdmVuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtb2RlbF9uYW1lYCB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgICAgV2hldGhlciBvciBub3QgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsLiBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgICAgICA6cGFyYW0gaW5wdXRfc2lnbmF0dXJlOiAgICAgQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdCB3aGVyZSBlYWNoIGVsZW1lbnQgaXMgYW4gaW5wdXQgbGF5ZXIgdHVwbGUuIEFuIGlucHV0IGxheWVyIHR1cGxlIGlzIGEgdHVwbGUgb2Y6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFswXSA9IExheWVyJ3Mgc2hhcGUsIGEgdHVwbGUgb2YgaW50ZWdlcnMuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgdGhlIGlucHV0IHNpZ25hdHVyZSB3aWxsIGJlIHRyaWVkIHRvIGJlIHJlYWQgZnJvbSB0aGUgbW9kZWwgYXJ0aWZhY3QuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpbnB1dF9sYXllcnNfbmFtZXM6ICBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgaW5wdXQgbm9kZXMgb2YgdGhlIGdyYXBoIGluIG9yZGVyLiBBbGwgb2YgdGhlIG90aGVyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHBhcmFtZXRlcnMgKGlubmVyIGxheWVycykgY2FuIGJlIHNldCBhcyB3ZWxsIGJ5IHBhc3NpbmcgYWRkaXRpb25hbCBuYW1lcyBpbiB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdC4gVGhlIG9yZGVyIGlzIGJ5IHRoZSBvcmRlciBvZiB0aGUgcGFyYW1ldGVycyBpbiB0aGUgbW9kZWwuIElmIE5vbmUsIHRoZSBpbnB1dHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBpbnB1dHMuIElmIGl0cyBhbHNvIE5vbmUsIGl0IGlzIGRlZmF1bHRlZCB0bzoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImlucHV0XzAiLCAiaW5wdXRfMSIsIC4uLgogICAgICAgIDpwYXJhbSBvdXRwdXRfbGF5ZXJzX25hbWVzOiBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgb3V0cHV0IG5vZGVzIG9mIHRoZSBncmFwaCBpbiBvcmRlci4gSWYgTm9uZSwgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG91dHB1dHMgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBvdXRwdXRzLiBJZiBpdHMgYWxzbyBOb25lLCBpdCBpcyBkZWZhdWx0ZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG86ICJvdXRwdXRfMCIgKGZvciBtdWx0aXBsZSBvdXRwdXRzLCB0aGlzIHBhcmFtZXRlciBtdXN0IGJlIHByb3ZpZGVkKS4KICAgICAgICA6cGFyYW0gZHluYW1pY19heGVzOiAgICAgICAgSWYgcGFydCBvZiB0aGUgaW5wdXQgLyBvdXRwdXQgc2hhcGUgaXMgZHluYW1pYywgbGlrZSAoYmF0Y2hfc2l6ZSwgMywgMzIsIDMyKSB5b3UgY2FuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNwZWNpZnkgaXQgYnkgZ2l2aW5nIGEgZHluYW1pYyBheGlzIHRvIHRoZSBpbnB1dCAvIG91dHB1dCBsYXllciBieSBpdHMgbmFtZSBhcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb2xsb3dzOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiaW5wdXQgbGF5ZXIgbmFtZSI6IHswOiAiYmF0Y2hfc2l6ZSJ9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm91dHB1dCBsYXllciBuYW1lIjogezA6ICJiYXRjaF9zaXplIn0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgcHJvdmlkZWQsIHRoZSAnaXNfYmF0Y2hlZCcgZmxhZyB3aWxsIGJlIGlnbm9yZWQuIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpc19iYXRjaGVkOiAgICAgICAgICBXaGV0aGVyIHRvIGluY2x1ZGUgYSBiYXRjaCBzaXplIGFzIHRoZSBmaXJzdCBheGlzIGluIGV2ZXJ5IGlucHV0IGFuZCBvdXRwdXQgbGF5ZXIuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBUcnVlLiBXaWxsIGJlIGlnbm9yZWQgaWYgJ2R5bmFtaWNfYXhlcycgaXMgcHJvdmlkZWQuCiAgICAgICAgIiIiCiAgICAgICAgIyBJbXBvcnQgdGhlIGZyYW1ld29yayBhbmQgaGFuZGxlcjoKICAgICAgICBpbXBvcnQgdG9yY2gKICAgICAgICBmcm9tIG1scnVuLmZyYW1ld29ya3MucHl0b3JjaCBpbXBvcnQgUHlUb3JjaFV0aWxzCgogICAgICAgICMgUGFyc2UgdGhlICdpbnB1dF9zaWduYXR1cmUnIHBhcmFtZXRlcjoKICAgICAgICBpZiBpbnB1dF9zaWduYXR1cmUgaXMgbm90IE5vbmU6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IHR1cGxlKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIHRvcmNoLnplcm9zKAogICAgICAgICAgICAgICAgICAgICAgICBzaXplPXNoYXBlLAogICAgICAgICAgICAgICAgICAgICAgICBkdHlwZT1QeVRvcmNoVXRpbHMuY29udmVydF92YWx1ZV90eXBlX3RvX3RvcmNoX2R0eXBlKAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgICAgICksCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICAgICAgXQogICAgICAgICAgICApCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NhbXBsZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICAgICBpbnB1dF9sYXllcnNfbmFtZXM9aW5wdXRfbGF5ZXJzX25hbWVzLAogICAgICAgICAgICBvdXRwdXRfbGF5ZXJzX25hbWVzPW91dHB1dF9sYXllcnNfbmFtZXMsCiAgICAgICAgICAgIGR5bmFtaWNfYXhlcz1keW5hbWljX2F4ZXMsCiAgICAgICAgICAgIGlzX2JhdGNoZWQ9aXNfYmF0Y2hlZCwKICAgICAgICApCgoKIyBNYXAgZm9yIGdldHRpbmcgdGhlIGNvbnZlcnNpb24gZnVuY3Rpb24gYWNjb3JkaW5nIHRvIHRoZSBwcm92aWRlZCBmcmFtZXdvcms6Cl9DT05WRVJTSU9OX01BUCA9IHsKICAgICJ0ZW5zb3JmbG93LmtlcmFzIjogX1RvT05OWENvbnZlcnNpb25zLnRmX2tlcmFzX3RvX29ubngsCiAgICAidG9yY2giOiBfVG9PTk5YQ29udmVyc2lvbnMucHl0b3JjaF90b19vbm54LAp9ICAjIHR5cGU6IERpY3Rbc3RyLCBDYWxsYWJsZV0KCgpkZWYgdG9fb25ueCgKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgbW9kZWxfcGF0aDogc3RyLAogICAgbG9hZF9tb2RlbF9rd2FyZ3M6IGRpY3QgPSBOb25lLAogICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgb3B0aW1pemVfbW9kZWw6IGJvb2wgPSBUcnVlLAogICAgZnJhbWV3b3JrX2t3YXJnczogRGljdFtzdHIsIEFueV0gPSBOb25lLAopOgogICAgIiIiCiAgICBDb252ZXJ0IHRoZSBnaXZlbiBtb2RlbCB0byBhbiBPTk5YIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgVGhlIE1MUnVuIGZ1bmN0aW9uIGV4ZWN1dGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxfcGF0aDogICAgICAgIFRoZSBtb2RlbCBwYXRoIHN0b3JlIG9iamVjdC4KICAgIDpwYXJhbSBsb2FkX21vZGVsX2t3YXJnczogS2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUgYEF1dG9NTFJ1bi5sb2FkX21vZGVsYCBtZXRob2QuCiAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiAgIFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbiBgbW9kZWxfbmFtZWAgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSB1c2VkIHdpdGggYW4gYWRkaXRpb25hbCBzdWZmaXggYF9vbm54YC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgIFdoZXRoZXIgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlIG1vZGVsLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgIDpwYXJhbSBmcmFtZXdvcmtfa3dhcmdzOiAgQWRkaXRpb25hbCBhcmd1bWVudHMgZWFjaCBmcmFtZXdvcmsgbWF5IHJlcXVpcmUgdG8gY29udmVydCB0byBPTk5YLiBUbyBnZXQgdGhlIGRvYyBzdHJpbmcKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb2YgdGhlIGRlc2lyZWQgZnJhbWV3b3JrIG9ubnggY29udmVyc2lvbiBmdW5jdGlvbiwgcGFzcyAiaGVscCIuCiAgICAiIiIKICAgIGZyb20gbWxydW4uZnJhbWV3b3Jrcy5hdXRvX21scnVuLmF1dG9fbWxydW4gaW1wb3J0IEF1dG9NTFJ1bgoKICAgICMgR2V0IGEgbW9kZWwgaGFuZGxlciBvZiB0aGUgcmVxdWlyZWQgZnJhbWV3b3JrOgogICAgbG9hZF9tb2RlbF9rd2FyZ3MgPSBsb2FkX21vZGVsX2t3YXJncyBvciB7fQogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWxfcGF0aCwgY29udGV4dD1jb250ZXh0LCAqKmxvYWRfbW9kZWxfa3dhcmdzCiAgICApCgogICAgIyBHZXQgdGhlIG1vZGVsJ3MgZnJhbWV3b3JrOgogICAgZnJhbWV3b3JrID0gbW9kZWxfaGFuZGxlci5GUkFNRVdPUktfTkFNRQoKICAgICMgVXNlIHRoZSBjb252ZXJzaW9uIG1hcCB0byBnZXQgdGhlIHNwZWNpZmljIGZyYW1ld29yayB0byBvbm54IGNvbnZlcnNpb246CiAgICBpZiBmcmFtZXdvcmsgbm90IGluIF9DT05WRVJTSU9OX01BUDoKICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgZiJUaGUgZm9sbG93aW5nIGZyYW1ld29yazogJ3tmcmFtZXdvcmt9JywgaGFzIG5vIE9OTlggY29udmVyc2lvbi4iCiAgICAgICAgKQogICAgY29udmVyc2lvbl9mdW5jdGlvbiA9IF9DT05WRVJTSU9OX01BUFtmcmFtZXdvcmtdCgogICAgIyBDaGVjayBpZiBuZWVkZWQgdG8gcHJpbnQgdGhlIGZ1bmN0aW9uJ3MgZG9jIHN0cmluZyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzID09ICJoZWxwIjoKICAgICAgICBwcmludChjb252ZXJzaW9uX2Z1bmN0aW9uLl9fZG9jX18pCiAgICAgICAgcmV0dXJuCgogICAgIyBTZXQgdGhlIGRlZmF1bHQgZW1wdHkgZnJhbWV3b3JrIGt3YXJncyBpZiBuZWVkZWQ6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzIGlzIE5vbmU6CiAgICAgICAgZnJhbWV3b3JrX2t3YXJncyA9IHt9CgogICAgIyBSdW4gdGhlIGNvbnZlcnNpb246CiAgICB0cnk6CiAgICAgICAgY29udmVyc2lvbl9mdW5jdGlvbigKICAgICAgICAgICAgbW9kZWxfaGFuZGxlcj1tb2RlbF9oYW5kbGVyLAogICAgICAgICAgICBvbm54X21vZGVsX25hbWU9b25ueF9tb2RlbF9uYW1lLAogICAgICAgICAgICBvcHRpbWl6ZV9tb2RlbD1vcHRpbWl6ZV9tb2RlbCwKICAgICAgICAgICAgKipmcmFtZXdvcmtfa3dhcmdzLAogICAgICAgICkKICAgIGV4Y2VwdCBUeXBlRXJyb3IgYXMgZXhjZXB0aW9uOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIkVSUk9SOiBBIFR5cGVFcnJvciBleGNlcHRpb24gd2FzIHJhaXNlZCBkdXJpbmcgdGhlIGNvbnZlcnNpb246XG57ZXhjZXB0aW9ufS4gIgogICAgICAgICAgICBmIlBsZWFzZSByZWFkIHRoZSB7ZnJhbWV3b3JrfSBmcmFtZXdvcmsgY29udmVyc2lvbiBmdW5jdGlvbiBkb2Mgc3RyaW5nIGJ5IHBhc3NpbmcgJ2hlbHAnIGluIHRoZSAiCiAgICAgICAgICAgIGYiJ2ZyYW1ld29ya19rd2FyZ3MnIGRpY3Rpb25hcnkgcGFyYW1ldGVyLiIKICAgICAgICApCgoKZGVmIG9wdGltaXplKAogICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICBtb2RlbF9wYXRoOiBzdHIsCiAgICBoYW5kbGVyX2luaXRfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIG9wdGltaXphdGlvbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBmaXhlZF9wb2ludDogYm9vbCA9IEZhbHNlLAogICAgb3B0aW1pemVkX21vZGVsX25hbWU6IHN0ciA9IE5vbmUsCik6CiAgICAiIiIKICAgIE9wdGltaXplIHRoZSBnaXZlbiBPTk5YIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgVGhlIE1MUnVuIGZ1bmN0aW9uIGV4ZWN1dGlvbiBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsX3BhdGg6ICAgICAgICAgICBQYXRoIHRvIHRoZSBPTk5YIG1vZGVsIG9iamVjdC4KICAgIDpwYXJhbSBoYW5kbGVyX2luaXRfa3dhcmdzOiAgS2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUgYE9OTlhNb2RlbEhhbmRsZXJgIGluaXQgbWV0aG9kIHByZWxvYWRpbmcuCiAgICA6cGFyYW0gb3B0aW1pemF0aW9uczogICAgICAgIExpc3Qgb2YgcG9zc2libGUgb3B0aW1pemF0aW9ucy4gVG8gc2VlIHdoYXQgb3B0aW1pemF0aW9ucyBhcmUgYXZhaWxhYmxlLCBwYXNzICJoZWxwIi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgYWxsIHRoZSBvcHRpbWl6YXRpb25zIHdpbGwgYmUgdXNlZC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gZml4ZWRfcG9pbnQ6ICAgICAgICAgIE9wdGltaXplIHRoZSB3ZWlnaHRzIHVzaW5nIGZpeGVkIHBvaW50LiBEZWZhdWx0ZWQgdG8gRmFsc2UuCiAgICA6cGFyYW0gb3B0aW1pemVkX21vZGVsX25hbWU6IFRoZSBuYW1lIG9mIHRoZSBvcHRpbWl6ZWQgbW9kZWwuIElmIE5vbmUsIHRoZSBvcmlnaW5hbCBtb2RlbCB3aWxsIGJlIG92ZXJyaWRkZW4uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgIiIiCiAgICAjIEltcG9ydCB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIGltcG9ydCBvbm54b3B0aW1pemVyCiAgICBmcm9tIG1scnVuLmZyYW1ld29ya3Mub25ueCBpbXBvcnQgT05OWE1vZGVsSGFuZGxlcgoKICAgICMgQ2hlY2sgaWYgbmVlZGVkIHRvIHByaW50IHRoZSBhdmFpbGFibGUgb3B0aW1pemF0aW9ucyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBvcHRpbWl6YXRpb25zID09ICJoZWxwIjoKICAgICAgICBhdmFpbGFibGVfcGFzc2VzID0gIlxuKiAiLmpvaW4ob25ueG9wdGltaXplci5nZXRfYXZhaWxhYmxlX3Bhc3NlcygpKQogICAgICAgIHByaW50KGYiVGhlIGF2YWlsYWJsZSBvcHRpbWl6YXRpb25zIGFyZTpcbioge2F2YWlsYWJsZV9wYXNzZXN9IikKICAgICAgICByZXR1cm4KCiAgICAjIENyZWF0ZSB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIGhhbmRsZXJfaW5pdF9rd2FyZ3MgPSBoYW5kbGVyX2luaXRfa3dhcmdzIG9yIHt9CiAgICBtb2RlbF9oYW5kbGVyID0gT05OWE1vZGVsSGFuZGxlcigKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsX3BhdGgsIGNvbnRleHQ9Y29udGV4dCwgKipoYW5kbGVyX2luaXRfa3dhcmdzCiAgICApCgogICAgIyBMb2FkIHRoZSBPTk5YIG1vZGVsOgogICAgbW9kZWxfaGFuZGxlci5sb2FkKCkKCiAgICAjIE9wdGltaXplIHRoZSBtb2RlbCB1c2luZyB0aGUgZ2l2ZW4gY29uZmlndXJhdGlvbnM6CiAgICBtb2RlbF9oYW5kbGVyLm9wdGltaXplKG9wdGltaXphdGlvbnM9b3B0aW1pemF0aW9ucywgZml4ZWRfcG9pbnQ9Zml4ZWRfcG9pbnQpCgogICAgIyBSZW5hbWUgaWYgbmVlZGVkOgogICAgaWYgb3B0aW1pemVkX21vZGVsX25hbWUgaXMgbm90IE5vbmU6CiAgICAgICAgbW9kZWxfaGFuZGxlci5zZXRfbW9kZWxfbmFtZShtb2RlbF9uYW1lPW9wdGltaXplZF9tb2RlbF9uYW1lKQoKICAgICMgTG9nIHRoZSBvcHRpbWl6ZWQgbW9kZWw6CiAgICBtb2RlbF9oYW5kbGVyLmxvZygpCg== + with_mlrun: false + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueQoKaW1wb3J0IG1scnVuCgoKY2xhc3MgX1RvT05OWENvbnZlcnNpb25zOgogICAgIiIiCiAgICBBbiBPTk5YIGNvbnZlcnNpb24gZnVuY3Rpb25zIGxpYnJhcnkgY2xhc3MuCiAgICAiIiIKCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgdGZfa2VyYXNfdG9fb25ueCgKICAgICAgICBtb2RlbF9oYW5kbGVyLAogICAgICAgIG9ubnhfbW9kZWxfbmFtZTogc3RyID0gTm9uZSwKICAgICAgICBvcHRpbWl6ZV9tb2RlbDogYm9vbCA9IFRydWUsCiAgICAgICAgaW5wdXRfc2lnbmF0dXJlOiBsaXN0W3R1cGxlW3R1cGxlW2ludF0sIHN0cl1dID0gTm9uZSwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCBhIFRGLktlcmFzIG1vZGVsIHRvIGFuIE9OTlggbW9kZWwgYW5kIGxvZyBpdCBiYWNrIHRvIE1MUnVuIGFzIGEgbmV3IG1vZGVsIG9iamVjdC4KCiAgICAgICAgOnBhcmFtIG1vZGVsX2hhbmRsZXI6ICAgQW4gaW5pdGlhbGl6ZWQgVEZLZXJhc01vZGVsSGFuZGxlciB3aXRoIGEgbG9hZGVkIG1vZGVsIHRvIGNvbnZlcnQgdG8gT05OWC4KICAgICAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiBUaGUgbmFtZSB0byB1c2UgdG8gbG9nIHRoZSBjb252ZXJ0ZWQgT05OWCBtb2RlbC4gSWYgbm90IGdpdmVuLCB0aGUgZ2l2ZW4gYG1vZGVsX25hbWVgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSB1c2VkIHdpdGggYW4gYWRkaXRpb25hbCBzdWZmaXggYF9vbm54YC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICAgICAgOnBhcmFtIG9wdGltaXplX21vZGVsOiAgV2hldGhlciBvciBub3QgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlIG1vZGVsLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBUcnVlLgogICAgICAgIDpwYXJhbSBpbnB1dF9zaWduYXR1cmU6IEEgbGlzdCBvZiB0aGUgaW5wdXQgbGF5ZXJzIHNoYXBlIGFuZCBkYXRhIHR5cGUgcHJvcGVydGllcy4gRXhwZWN0ZWQgdG8gcmVjZWl2ZSBhIGxpc3QKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aGVyZSBlYWNoIGVsZW1lbnQgaXMgYW4gaW5wdXQgbGF5ZXIgdHVwbGUuIEFuIGlucHV0IGxheWVyIHR1cGxlIGlzIGEgdHVwbGUgb2Y6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgWzBdID0gTGF5ZXIncyBzaGFwZSwgYSB0dXBsZSBvZiBpbnRlZ2Vycy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBbMV0gPSBMYXllcidzIGRhdGEgdHlwZSwgYSBtbHJ1bi5kYXRhX3R5cGVzLlZhbHVlVHlwZSBzdHJpbmcuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgdGhlIGlucHV0IHNpZ25hdHVyZSB3aWxsIGJlIHRyaWVkIHRvIGJlIHJlYWQgZnJvbSB0aGUgbW9kZWwgYXJ0aWZhY3QuIERlZmF1bHRlZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRvIE5vbmUuCiAgICAgICAgIiIiCiAgICAgICAgIyBJbXBvcnQgdGhlIGZyYW1ld29yayBhbmQgaGFuZGxlcjoKICAgICAgICBpbXBvcnQgdGVuc29yZmxvdyBhcyB0ZgogICAgICAgIGZyb20gbWxydW4uZnJhbWV3b3Jrcy50Zl9rZXJhcyBpbXBvcnQgVEZLZXJhc1V0aWxzCgogICAgICAgICMgQ2hlY2sgdGhlIGdpdmVuICdpbnB1dF9zaWduYXR1cmUnIHBhcmFtZXRlcjoKICAgICAgICBpZiBpbnB1dF9zaWduYXR1cmUgaXMgTm9uZToKICAgICAgICAgICAgIyBSZWFkIHRoZSBpbnB1dHMgZnJvbSB0aGUgbW9kZWw6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIG1vZGVsX2hhbmRsZXIucmVhZF9pbnB1dHNfZnJvbV9tb2RlbCgpCiAgICAgICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXJyb3I6CiAgICAgICAgICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5SdW50aW1lRXJyb3IoCiAgICAgICAgICAgICAgICAgICAgZiJQbGVhc2UgcHJvdmlkZSB0aGUgJ2lucHV0X3NpZ25hdHVyZScgcGFyYW1ldGVyLiBUaGUgZnVuY3Rpb24gdHJpZWQgcmVhZGluZyB0aGUgaW5wdXQgbGF5ZXJzICIKICAgICAgICAgICAgICAgICAgICBmImluZm9ybWF0aW9uIGF1dG9tYXRpY2FsbHkgYnV0IGZhaWxlZCB3aXRoIHRoZSBmb2xsb3dpbmcgZXJyb3I6IHtlcnJvcn0iCiAgICAgICAgICAgICAgICApCiAgICAgICAgZWxzZToKICAgICAgICAgICAgIyBQYXJzZSB0aGUgJ2lucHV0X3NpZ25hdHVyZScgcGFyYW1ldGVyOgogICAgICAgICAgICBpbnB1dF9zaWduYXR1cmUgPSBbCiAgICAgICAgICAgICAgICB0Zi5UZW5zb3JTcGVjKAogICAgICAgICAgICAgICAgICAgIHNoYXBlPXNoYXBlLAogICAgICAgICAgICAgICAgICAgIGR0eXBlPVRGS2VyYXNVdGlscy5jb252ZXJ0X3ZhbHVlX3R5cGVfdG9fdGZfZHR5cGUoCiAgICAgICAgICAgICAgICAgICAgICAgIHZhbHVlX3R5cGU9dmFsdWVfdHlwZQogICAgICAgICAgICAgICAgICAgICksCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBmb3IgKHNoYXBlLCB2YWx1ZV90eXBlKSBpbiBpbnB1dF9zaWduYXR1cmUKICAgICAgICAgICAgXQoKICAgICAgICAjIENvbnZlcnQgdG8gT05OWDoKICAgICAgICBtb2RlbF9oYW5kbGVyLnRvX29ubngoCiAgICAgICAgICAgIG1vZGVsX25hbWU9b25ueF9tb2RlbF9uYW1lLAogICAgICAgICAgICBpbnB1dF9zaWduYXR1cmU9aW5wdXRfc2lnbmF0dXJlLAogICAgICAgICAgICBvcHRpbWl6ZT1vcHRpbWl6ZV9tb2RlbCwKICAgICAgICApCgogICAgQHN0YXRpY21ldGhvZAogICAgZGVmIHB5dG9yY2hfdG9fb25ueCgKICAgICAgICBtb2RlbF9oYW5kbGVyLAogICAgICAgIG9ubnhfbW9kZWxfbmFtZTogc3RyID0gTm9uZSwKICAgICAgICBvcHRpbWl6ZV9tb2RlbDogYm9vbCA9IFRydWUsCiAgICAgICAgaW5wdXRfc2lnbmF0dXJlOiBsaXN0W3R1cGxlW3R1cGxlW2ludCwgLi4uXSwgc3RyXV0gPSBOb25lLAogICAgICAgIGlucHV0X2xheWVyc19uYW1lczogbGlzdFtzdHJdID0gTm9uZSwKICAgICAgICBvdXRwdXRfbGF5ZXJzX25hbWVzOiBsaXN0W3N0cl0gPSBOb25lLAogICAgICAgIGR5bmFtaWNfYXhlczogZGljdFtzdHIsIGRpY3RbaW50LCBzdHJdXSA9IE5vbmUsCiAgICAgICAgaXNfYmF0Y2hlZDogYm9vbCA9IFRydWUsCiAgICApOgogICAgICAgICIiIgogICAgICAgIENvbnZlcnQgYSBQeVRvcmNoIG1vZGVsIHRvIGFuIE9OTlggbW9kZWwgYW5kIGxvZyBpdCBiYWNrIHRvIE1MUnVuIGFzIGEgbmV3IG1vZGVsIG9iamVjdC4KCiAgICAgICAgOnBhcmFtIG1vZGVsX2hhbmRsZXI6ICAgICAgIEFuIGluaXRpYWxpemVkIFB5VG9yY2hNb2RlbEhhbmRsZXIgd2l0aCBhIGxvYWRlZCBtb2RlbCB0byBjb252ZXJ0IHRvIE9OTlguCiAgICAgICAgOnBhcmFtIG9ubnhfbW9kZWxfbmFtZTogICAgIFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBgbW9kZWxfbmFtZWAgd2lsbCBiZSB1c2VkIHdpdGggYW4gYWRkaXRpb25hbCBzdWZmaXggYF9vbm54YC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICAgICAgOnBhcmFtIG9wdGltaXplX21vZGVsOiAgICAgIFdoZXRoZXIgb3Igbm90IHRvIG9wdGltaXplIHRoZSBPTk5YIG1vZGVsIHVzaW5nICdvbm54b3B0aW1pemVyJyBiZWZvcmUgc2F2aW5nIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlbC4gRGVmYXVsdGVkIHRvIFRydWUuCiAgICAgICAgOnBhcmFtIGlucHV0X3NpZ25hdHVyZTogICAgIEEgbGlzdCBvZiB0aGUgaW5wdXQgbGF5ZXJzIHNoYXBlIGFuZCBkYXRhIHR5cGUgcHJvcGVydGllcy4gRXhwZWN0ZWQgdG8gcmVjZWl2ZSBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxpc3Qgd2hlcmUgZWFjaCBlbGVtZW50IGlzIGFuIGlucHV0IGxheWVyIHR1cGxlLiBBbiBpbnB1dCBsYXllciB0dXBsZSBpcyBhIHR1cGxlIG9mOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBbMF0gPSBMYXllcidzIHNoYXBlLCBhIHR1cGxlIG9mIGludGVnZXJzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBbMV0gPSBMYXllcidzIGRhdGEgdHlwZSwgYSBtbHJ1bi5kYXRhX3R5cGVzLlZhbHVlVHlwZSBzdHJpbmcuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIElmIE5vbmUsIHRoZSBpbnB1dCBzaWduYXR1cmUgd2lsbCBiZSB0cmllZCB0byBiZSByZWFkIGZyb20gdGhlIG1vZGVsIGFydGlmYWN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gaW5wdXRfbGF5ZXJzX25hbWVzOiAgTGlzdCBvZiBuYW1lcyB0byBhc3NpZ24gdG8gdGhlIGlucHV0IG5vZGVzIG9mIHRoZSBncmFwaCBpbiBvcmRlci4gQWxsIG9mIHRoZSBvdGhlcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBwYXJhbWV0ZXJzIChpbm5lciBsYXllcnMpIGNhbiBiZSBzZXQgYXMgd2VsbCBieSBwYXNzaW5nIGFkZGl0aW9uYWwgbmFtZXMgaW4gdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxpc3QuIFRoZSBvcmRlciBpcyBieSB0aGUgb3JkZXIgb2YgdGhlIHBhcmFtZXRlcnMgaW4gdGhlIG1vZGVsLiBJZiBOb25lLCB0aGUgaW5wdXRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHdpbGwgYmUgcmVhZCBmcm9tIHRoZSBoYW5kbGVyJ3MgaW5wdXRzLiBJZiBpdHMgYWxzbyBOb25lLCBpdCBpcyBkZWZhdWx0ZWQgdG86CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJpbnB1dF8wIiwgImlucHV0XzEiLCAuLi4KICAgICAgICA6cGFyYW0gb3V0cHV0X2xheWVyc19uYW1lczogTGlzdCBvZiBuYW1lcyB0byBhc3NpZ24gdG8gdGhlIG91dHB1dCBub2RlcyBvZiB0aGUgZ3JhcGggaW4gb3JkZXIuIElmIE5vbmUsIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvdXRwdXRzIHdpbGwgYmUgcmVhZCBmcm9tIHRoZSBoYW5kbGVyJ3Mgb3V0cHV0cy4gSWYgaXRzIGFsc28gTm9uZSwgaXQgaXMgZGVmYXVsdGVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRvOiAib3V0cHV0XzAiIChmb3IgbXVsdGlwbGUgb3V0cHV0cywgdGhpcyBwYXJhbWV0ZXIgbXVzdCBiZSBwcm92aWRlZCkuCiAgICAgICAgOnBhcmFtIGR5bmFtaWNfYXhlczogICAgICAgIElmIHBhcnQgb2YgdGhlIGlucHV0IC8gb3V0cHV0IHNoYXBlIGlzIGR5bmFtaWMsIGxpa2UgKGJhdGNoX3NpemUsIDMsIDMyLCAzMikgeW91IGNhbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzcGVjaWZ5IGl0IGJ5IGdpdmluZyBhIGR5bmFtaWMgYXhpcyB0byB0aGUgaW5wdXQgLyBvdXRwdXQgbGF5ZXIgYnkgaXRzIG5hbWUgYXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZm9sbG93czogewogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImlucHV0IGxheWVyIG5hbWUiOiB7MDogImJhdGNoX3NpemUifSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJvdXRwdXQgbGF5ZXIgbmFtZSI6IHswOiAiYmF0Y2hfc2l6ZSJ9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIElmIHByb3ZpZGVkLCB0aGUgJ2lzX2JhdGNoZWQnIGZsYWcgd2lsbCBiZSBpZ25vcmVkLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gaXNfYmF0Y2hlZDogICAgICAgICAgV2hldGhlciB0byBpbmNsdWRlIGEgYmF0Y2ggc2l6ZSBhcyB0aGUgZmlyc3QgYXhpcyBpbiBldmVyeSBpbnB1dCBhbmQgb3V0cHV0IGxheWVyLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0ZWQgdG8gVHJ1ZS4gV2lsbCBiZSBpZ25vcmVkIGlmICdkeW5hbWljX2F4ZXMnIGlzIHByb3ZpZGVkLgogICAgICAgICIiIgogICAgICAgICMgSW1wb3J0IHRoZSBmcmFtZXdvcmsgYW5kIGhhbmRsZXI6CiAgICAgICAgaW1wb3J0IHRvcmNoCiAgICAgICAgZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLnB5dG9yY2ggaW1wb3J0IFB5VG9yY2hVdGlscwoKICAgICAgICAjIFBhcnNlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXI6CiAgICAgICAgaWYgaW5wdXRfc2lnbmF0dXJlIGlzIG5vdCBOb25lOgogICAgICAgICAgICBpbnB1dF9zaWduYXR1cmUgPSB0dXBsZSgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICB0b3JjaC56ZXJvcygKICAgICAgICAgICAgICAgICAgICAgICAgc2l6ZT1zaGFwZSwKICAgICAgICAgICAgICAgICAgICAgICAgZHR5cGU9UHlUb3JjaFV0aWxzLmNvbnZlcnRfdmFsdWVfdHlwZV90b190b3JjaF9kdHlwZSgKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHZhbHVlX3R5cGU9dmFsdWVfdHlwZQogICAgICAgICAgICAgICAgICAgICAgICApLAogICAgICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgICAgICBmb3IgKHNoYXBlLCB2YWx1ZV90eXBlKSBpbiBpbnB1dF9zaWduYXR1cmUKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQoKICAgICAgICAjIENvbnZlcnQgdG8gT05OWDoKICAgICAgICBtb2RlbF9oYW5kbGVyLnRvX29ubngoCiAgICAgICAgICAgIG1vZGVsX25hbWU9b25ueF9tb2RlbF9uYW1lLAogICAgICAgICAgICBpbnB1dF9zYW1wbGU9aW5wdXRfc2lnbmF0dXJlLAogICAgICAgICAgICBvcHRpbWl6ZT1vcHRpbWl6ZV9tb2RlbCwKICAgICAgICAgICAgaW5wdXRfbGF5ZXJzX25hbWVzPWlucHV0X2xheWVyc19uYW1lcywKICAgICAgICAgICAgb3V0cHV0X2xheWVyc19uYW1lcz1vdXRwdXRfbGF5ZXJzX25hbWVzLAogICAgICAgICAgICBkeW5hbWljX2F4ZXM9ZHluYW1pY19heGVzLAogICAgICAgICAgICBpc19iYXRjaGVkPWlzX2JhdGNoZWQsCiAgICAgICAgKQoKCiMgTWFwIGZvciBnZXR0aW5nIHRoZSBjb252ZXJzaW9uIGZ1bmN0aW9uIGFjY29yZGluZyB0byB0aGUgcHJvdmlkZWQgZnJhbWV3b3JrOgpfQ09OVkVSU0lPTl9NQVAgPSB7CiAgICAidGVuc29yZmxvdy5rZXJhcyI6IF9Ub09OTlhDb252ZXJzaW9ucy50Zl9rZXJhc190b19vbm54LAogICAgInRvcmNoIjogX1RvT05OWENvbnZlcnNpb25zLnB5dG9yY2hfdG9fb25ueCwKfSAgIyB0eXBlOiBEaWN0W3N0ciwgQ2FsbGFibGVdCgoKZGVmIHRvX29ubngoCiAgICBjb250ZXh0OiBtbHJ1bi5NTENsaWVudEN0eCwKICAgIG1vZGVsX3BhdGg6IHN0ciwKICAgIGxvYWRfbW9kZWxfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIG9ubnhfbW9kZWxfbmFtZTogc3RyID0gTm9uZSwKICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgIGZyYW1ld29ya19rd2FyZ3M6IGRpY3Rbc3RyLCBBbnldID0gTm9uZSwKKToKICAgICIiIgogICAgQ29udmVydCB0aGUgZ2l2ZW4gbW9kZWwgdG8gYW4gT05OWCBtb2RlbC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgIFRoZSBNTFJ1biBmdW5jdGlvbiBleGVjdXRpb24gY29udGV4dAogICAgOnBhcmFtIG1vZGVsX3BhdGg6ICAgICAgICBUaGUgbW9kZWwgcGF0aCBzdG9yZSBvYmplY3QuCiAgICA6cGFyYW0gbG9hZF9tb2RlbF9rd2FyZ3M6IEtleXdvcmQgYXJndW1lbnRzIHRvIHBhc3MgdG8gdGhlIGBBdXRvTUxSdW4ubG9hZF9tb2RlbGAgbWV0aG9kLgogICAgOnBhcmFtIG9ubnhfbW9kZWxfbmFtZTogICBUaGUgbmFtZSB0byB1c2UgdG8gbG9nIHRoZSBjb252ZXJ0ZWQgT05OWCBtb2RlbC4gSWYgbm90IGdpdmVuLCB0aGUgZ2l2ZW4gYG1vZGVsX25hbWVgIHdpbGwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYmUgdXNlZCB3aXRoIGFuIGFkZGl0aW9uYWwgc3VmZml4IGBfb25ueGAuIERlZmF1bHRlZCB0byBOb25lLgogICAgOnBhcmFtIG9wdGltaXplX21vZGVsOiAgICBXaGV0aGVyIHRvIG9wdGltaXplIHRoZSBPTk5YIG1vZGVsIHVzaW5nICdvbm54b3B0aW1pemVyJyBiZWZvcmUgc2F2aW5nIHRoZSBtb2RlbC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgRGVmYXVsdGVkIHRvIFRydWUuCiAgICA6cGFyYW0gZnJhbWV3b3JrX2t3YXJnczogIEFkZGl0aW9uYWwgYXJndW1lbnRzIGVhY2ggZnJhbWV3b3JrIG1heSByZXF1aXJlIHRvIGNvbnZlcnQgdG8gT05OWC4gVG8gZ2V0IHRoZSBkb2Mgc3RyaW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9mIHRoZSBkZXNpcmVkIGZyYW1ld29yayBvbm54IGNvbnZlcnNpb24gZnVuY3Rpb24sIHBhc3MgImhlbHAiLgogICAgIiIiCiAgICBmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1bi5hdXRvX21scnVuIGltcG9ydCBBdXRvTUxSdW4KCiAgICAjIEdldCBhIG1vZGVsIGhhbmRsZXIgb2YgdGhlIHJlcXVpcmVkIGZyYW1ld29yazoKICAgIGxvYWRfbW9kZWxfa3dhcmdzID0gbG9hZF9tb2RlbF9rd2FyZ3Mgb3Ige30KICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbCgKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsX3BhdGgsIGNvbnRleHQ9Y29udGV4dCwgKipsb2FkX21vZGVsX2t3YXJncwogICAgKQoKICAgICMgR2V0IHRoZSBtb2RlbCdzIGZyYW1ld29yazoKICAgIGZyYW1ld29yayA9IG1vZGVsX2hhbmRsZXIuRlJBTUVXT1JLX05BTUUKCiAgICAjIFVzZSB0aGUgY29udmVyc2lvbiBtYXAgdG8gZ2V0IHRoZSBzcGVjaWZpYyBmcmFtZXdvcmsgdG8gb25ueCBjb252ZXJzaW9uOgogICAgaWYgZnJhbWV3b3JrIG5vdCBpbiBfQ09OVkVSU0lPTl9NQVA6CiAgICAgICAgcmFpc2UgbWxydW4uZXJyb3JzLk1MUnVuSW52YWxpZEFyZ3VtZW50RXJyb3IoCiAgICAgICAgICAgIGYiVGhlIGZvbGxvd2luZyBmcmFtZXdvcms6ICd7ZnJhbWV3b3JrfScsIGhhcyBubyBPTk5YIGNvbnZlcnNpb24uIgogICAgICAgICkKICAgIGNvbnZlcnNpb25fZnVuY3Rpb24gPSBfQ09OVkVSU0lPTl9NQVBbZnJhbWV3b3JrXQoKICAgICMgQ2hlY2sgaWYgbmVlZGVkIHRvIHByaW50IHRoZSBmdW5jdGlvbidzIGRvYyBzdHJpbmcgKCJoZWxwIiBpcyBwYXNzZWQpOgogICAgaWYgZnJhbWV3b3JrX2t3YXJncyA9PSAiaGVscCI6CiAgICAgICAgcHJpbnQoY29udmVyc2lvbl9mdW5jdGlvbi5fX2RvY19fKQogICAgICAgIHJldHVybgoKICAgICMgU2V0IHRoZSBkZWZhdWx0IGVtcHR5IGZyYW1ld29yayBrd2FyZ3MgaWYgbmVlZGVkOgogICAgaWYgZnJhbWV3b3JrX2t3YXJncyBpcyBOb25lOgogICAgICAgIGZyYW1ld29ya19rd2FyZ3MgPSB7fQoKICAgICMgUnVuIHRoZSBjb252ZXJzaW9uOgogICAgdHJ5OgogICAgICAgIGNvbnZlcnNpb25fZnVuY3Rpb24oCiAgICAgICAgICAgIG1vZGVsX2hhbmRsZXI9bW9kZWxfaGFuZGxlciwKICAgICAgICAgICAgb25ueF9tb2RlbF9uYW1lPW9ubnhfbW9kZWxfbmFtZSwKICAgICAgICAgICAgb3B0aW1pemVfbW9kZWw9b3B0aW1pemVfbW9kZWwsCiAgICAgICAgICAgICoqZnJhbWV3b3JrX2t3YXJncywKICAgICAgICApCiAgICBleGNlcHQgVHlwZUVycm9yIGFzIGV4Y2VwdGlvbjoKICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgZiJFUlJPUjogQSBUeXBlRXJyb3IgZXhjZXB0aW9uIHdhcyByYWlzZWQgZHVyaW5nIHRoZSBjb252ZXJzaW9uOlxue2V4Y2VwdGlvbn0uICIKICAgICAgICAgICAgZiJQbGVhc2UgcmVhZCB0aGUge2ZyYW1ld29ya30gZnJhbWV3b3JrIGNvbnZlcnNpb24gZnVuY3Rpb24gZG9jIHN0cmluZyBieSBwYXNzaW5nICdoZWxwJyBpbiB0aGUgIgogICAgICAgICAgICBmIidmcmFtZXdvcmtfa3dhcmdzJyBkaWN0aW9uYXJ5IHBhcmFtZXRlci4iCiAgICAgICAgKQoKCmRlZiBvcHRpbWl6ZSgKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgbW9kZWxfcGF0aDogc3RyLAogICAgaGFuZGxlcl9pbml0X2t3YXJnczogZGljdCA9IE5vbmUsCiAgICBvcHRpbWl6YXRpb25zOiBsaXN0W3N0cl0gPSBOb25lLAogICAgZml4ZWRfcG9pbnQ6IGJvb2wgPSBGYWxzZSwKICAgIG9wdGltaXplZF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAopOgogICAgIiIiCiAgICBPcHRpbWl6ZSB0aGUgZ2l2ZW4gT05OWCBtb2RlbC4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgIFRoZSBNTFJ1biBmdW5jdGlvbiBleGVjdXRpb24gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbF9wYXRoOiAgICAgICAgICAgUGF0aCB0byB0aGUgT05OWCBtb2RlbCBvYmplY3QuCiAgICA6cGFyYW0gaGFuZGxlcl9pbml0X2t3YXJnczogIEtleXdvcmQgYXJndW1lbnRzIHRvIHBhc3MgdG8gdGhlIGBPTk5YTW9kZWxIYW5kbGVyYCBpbml0IG1ldGhvZCBwcmVsb2FkaW5nLgogICAgOnBhcmFtIG9wdGltaXphdGlvbnM6ICAgICAgICBMaXN0IG9mIHBvc3NpYmxlIG9wdGltaXphdGlvbnMuIFRvIHNlZSB3aGF0IG9wdGltaXphdGlvbnMgYXJlIGF2YWlsYWJsZSwgcGFzcyAiaGVscCIuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIElmIE5vbmUsIGFsbCB0aGUgb3B0aW1pemF0aW9ucyB3aWxsIGJlIHVzZWQuIERlZmF1bHRlZCB0byBOb25lLgogICAgOnBhcmFtIGZpeGVkX3BvaW50OiAgICAgICAgICBPcHRpbWl6ZSB0aGUgd2VpZ2h0cyB1c2luZyBmaXhlZCBwb2ludC4gRGVmYXVsdGVkIHRvIEZhbHNlLgogICAgOnBhcmFtIG9wdGltaXplZF9tb2RlbF9uYW1lOiBUaGUgbmFtZSBvZiB0aGUgb3B0aW1pemVkIG1vZGVsLiBJZiBOb25lLCB0aGUgb3JpZ2luYWwgbW9kZWwgd2lsbCBiZSBvdmVycmlkZGVuLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICIiIgogICAgIyBJbXBvcnQgdGhlIG1vZGVsIGhhbmRsZXI6CiAgICBpbXBvcnQgb25ueG9wdGltaXplcgogICAgZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLm9ubnggaW1wb3J0IE9OTlhNb2RlbEhhbmRsZXIKCiAgICAjIENoZWNrIGlmIG5lZWRlZCB0byBwcmludCB0aGUgYXZhaWxhYmxlIG9wdGltaXphdGlvbnMgKCJoZWxwIiBpcyBwYXNzZWQpOgogICAgaWYgb3B0aW1pemF0aW9ucyA9PSAiaGVscCI6CiAgICAgICAgYXZhaWxhYmxlX3Bhc3NlcyA9ICJcbiogIi5qb2luKG9ubnhvcHRpbWl6ZXIuZ2V0X2F2YWlsYWJsZV9wYXNzZXMoKSkKICAgICAgICBwcmludChmIlRoZSBhdmFpbGFibGUgb3B0aW1pemF0aW9ucyBhcmU6XG4qIHthdmFpbGFibGVfcGFzc2VzfSIpCiAgICAgICAgcmV0dXJuCgogICAgIyBDcmVhdGUgdGhlIG1vZGVsIGhhbmRsZXI6CiAgICBoYW5kbGVyX2luaXRfa3dhcmdzID0gaGFuZGxlcl9pbml0X2t3YXJncyBvciB7fQogICAgbW9kZWxfaGFuZGxlciA9IE9OTlhNb2RlbEhhbmRsZXIoCiAgICAgICAgbW9kZWxfcGF0aD1tb2RlbF9wYXRoLCBjb250ZXh0PWNvbnRleHQsICoqaGFuZGxlcl9pbml0X2t3YXJncwogICAgKQoKICAgICMgTG9hZCB0aGUgT05OWCBtb2RlbDoKICAgIG1vZGVsX2hhbmRsZXIubG9hZCgpCgogICAgIyBPcHRpbWl6ZSB0aGUgbW9kZWwgdXNpbmcgdGhlIGdpdmVuIGNvbmZpZ3VyYXRpb25zOgogICAgbW9kZWxfaGFuZGxlci5vcHRpbWl6ZShvcHRpbWl6YXRpb25zPW9wdGltaXphdGlvbnMsIGZpeGVkX3BvaW50PWZpeGVkX3BvaW50KQoKICAgICMgUmVuYW1lIGlmIG5lZWRlZDoKICAgIGlmIG9wdGltaXplZF9tb2RlbF9uYW1lIGlzIG5vdCBOb25lOgogICAgICAgIG1vZGVsX2hhbmRsZXIuc2V0X21vZGVsX25hbWUobW9kZWxfbmFtZT1vcHRpbWl6ZWRfbW9kZWxfbmFtZSkKCiAgICAjIExvZyB0aGUgb3B0aW1pemVkIG1vZGVsOgogICAgbW9kZWxfaGFuZGxlci5sb2coKQo= requirements: - tqdm~=4.67.1 - tensorflow~=2.19.0 @@ -24,17 +25,13 @@ spec: - onnxmltools~=1.13.0 - tf2onnx~=1.16.1 - plotly~=5.23 - with_mlrun: false + code_origin: '' auto_build: true - disable_auto_mount: false - description: ONNX intigration in MLRun, some utils functions for the ONNX framework, - optimizing and converting models from different framework to ONNX using MLRun. - image: '' + base_image: mlrun/mlrun + allow_empty_resources: true + filename: onnx_utils.py entry_points: tf_keras_to_onnx: - doc: Convert a TF.Keras model to an ONNX model and log it back to MLRun as a - new model object. - name: tf_keras_to_onnx parameters: - name: model_handler doc: An initialized TFKerasModelHandler with a loaded model to convert to @@ -51,20 +48,20 @@ spec: saving the model. Defaulted to True. default: true - name: input_signature - type: List[Tuple[Tuple[int], str]] + type: list[tuple[tuple[int], str]] doc: 'A list of the input layers shape and data type properties. Expected to receive a list where each element is an input layer tuple. An input layer tuple is a tuple of: [0] = Layer''s shape, a tuple of integers. [1] = Layer''s data type, a mlrun.data_types.ValueType string. If None, the input signature will be tried to be read from the model artifact. Defaulted to None.' default: null - has_varargs: false + name: tf_keras_to_onnx + doc: Convert a TF.Keras model to an ONNX model and log it back to MLRun as a + new model object. has_kwargs: false + has_varargs: false lineno: 26 pytorch_to_onnx: - doc: Convert a PyTorch model to an ONNX model and log it back to MLRun as a - new model object. - name: pytorch_to_onnx parameters: - name: model_handler doc: An initialized PyTorchModelHandler with a loaded model to convert to @@ -81,7 +78,7 @@ spec: saving the model. Defaulted to True. default: true - name: input_signature - type: List[Tuple[Tuple[int, ], str]] + type: list[tuple[tuple[int, ], str]] doc: 'A list of the input layers shape and data type properties. Expected to receive a list where each element is an input layer tuple. An input layer tuple is a tuple of: [0] = Layer''s shape, a tuple of integers. [1] = Layer''s @@ -89,7 +86,7 @@ spec: will be tried to be read from the model artifact. Defaulted to None.' default: null - name: input_layers_names - type: List[str] + type: list[str] doc: 'List of names to assign to the input nodes of the graph in order. All of the other parameters (inner layers) can be set as well by passing additional names in the list. The order is by the order of the parameters in the model. @@ -97,14 +94,14 @@ spec: None, it is defaulted to: "input_0", "input_1", ...' default: null - name: output_layers_names - type: List[str] + type: list[str] doc: 'List of names to assign to the output nodes of the graph in order. If None, the outputs will be read from the handler''s outputs. If its also None, it is defaulted to: "output_0" (for multiple outputs, this parameter must be provided).' default: null - name: dynamic_axes - type: Dict[str, Dict[int, str]] + type: dict[str, dict[int, str]] doc: 'If part of the input / output shape is dynamic, like (batch_size, 3, 32, 32) you can specify it by giving a dynamic axis to the input / output layer by its name as follows: { "input layer name": {0: "batch_size"}, "output @@ -116,12 +113,13 @@ spec: doc: Whether to include a batch size as the first axis in every input and output layer. Defaulted to True. Will be ignored if 'dynamic_axes' is provided. default: true - has_varargs: false + name: pytorch_to_onnx + doc: Convert a PyTorch model to an ONNX model and log it back to MLRun as a + new model object. has_kwargs: false + has_varargs: false lineno: 81 to_onnx: - doc: Convert the given model to an ONNX model. - name: to_onnx parameters: - name: context type: MLClientCtx @@ -145,17 +143,17 @@ spec: the model. Defaulted to True. default: true - name: framework_kwargs - type: Dict[str, Any] + type: dict[str, Any] doc: Additional arguments each framework may require to convert to ONNX. To get the doc string of the desired framework onnx conversion function, pass "help". default: null - has_varargs: false + name: to_onnx + doc: Convert the given model to an ONNX model. has_kwargs: false + has_varargs: false lineno: 160 optimize: - doc: Optimize the given ONNX model. - name: optimize parameters: - name: context type: MLClientCtx @@ -168,7 +166,7 @@ spec: doc: Keyword arguments to pass to the `ONNXModelHandler` init method preloading. default: null - name: optimizations - type: List[str] + type: list[str] doc: List of possible optimizations. To see what optimizations are available, pass "help". If None, all the optimizations will be used. Defaulted to None. default: null @@ -181,9 +179,12 @@ spec: doc: The name of the optimized model. If None, the original model will be overridden. Defaulted to None. default: null - has_varargs: false + name: optimize + doc: Optimize the given ONNX model. has_kwargs: false + has_varargs: false lineno: 224 - default_handler: to_onnx - allow_empty_resources: true command: '' + description: ONNX intigration in MLRun, some utils functions for the ONNX framework, + optimizing and converting models from different framework to ONNX using MLRun. + default_handler: to_onnx diff --git a/functions/src/onnx_utils/onnx_utils.py b/functions/src/onnx_utils/onnx_utils.py index c26e011be..ed6890b55 100644 --- a/functions/src/onnx_utils/onnx_utils.py +++ b/functions/src/onnx_utils/onnx_utils.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from typing import Any, Callable, Dict, List, Tuple +from typing import Any import mlrun @@ -27,7 +27,7 @@ def tf_keras_to_onnx( model_handler, onnx_model_name: str = None, optimize_model: bool = True, - input_signature: List[Tuple[Tuple[int], str]] = None, + input_signature: list[tuple[tuple[int], str]] = None, ): """ Convert a TF.Keras model to an ONNX model and log it back to MLRun as a new model object. @@ -82,10 +82,10 @@ def pytorch_to_onnx( model_handler, onnx_model_name: str = None, optimize_model: bool = True, - input_signature: List[Tuple[Tuple[int, ...], str]] = None, - input_layers_names: List[str] = None, - output_layers_names: List[str] = None, - dynamic_axes: Dict[str, Dict[int, str]] = None, + input_signature: list[tuple[tuple[int, ...], str]] = None, + input_layers_names: list[str] = None, + output_layers_names: list[str] = None, + dynamic_axes: dict[str, dict[int, str]] = None, is_batched: bool = True, ): """ @@ -163,7 +163,7 @@ def to_onnx( load_model_kwargs: dict = None, onnx_model_name: str = None, optimize_model: bool = True, - framework_kwargs: Dict[str, Any] = None, + framework_kwargs: dict[str, Any] = None, ): """ Convert the given model to an ONNX model. @@ -225,7 +225,7 @@ def optimize( context: mlrun.MLClientCtx, model_path: str, handler_init_kwargs: dict = None, - optimizations: List[str] = None, + optimizations: list[str] = None, fixed_point: bool = False, optimized_model_name: str = None, ): diff --git a/functions/src/open_archive/function.yaml b/functions/src/open_archive/function.yaml index bf78b5fcd..451279f43 100644 --- a/functions/src/open_archive/function.yaml +++ b/functions/src/open_archive/function.yaml @@ -1,20 +1,20 @@ -kind: job +metadata: + tag: '' + name: open-archive + categories: + - utils verbose: false +kind: job spec: - command: '' + image: mlrun/mlrun disable_auto_mount: false - default_handler: open_archive build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAyNSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKCmltcG9ydCBvcwppbXBvcnQgemlwZmlsZQppbXBvcnQgdGFyZmlsZQoKZnJvbSBtbHJ1bi5leGVjdXRpb24gaW1wb3J0IE1MQ2xpZW50Q3R4CmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLmFydGlmYWN0cy5iYXNlIGltcG9ydCBEaXJBcnRpZmFjdAoKZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybHBhcnNlCgoKZGVmIG9wZW5fYXJjaGl2ZSgKICAgICAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgICAgICBhcmNoaXZlX3VybDogRGF0YUl0ZW0sCiAgICAgICAgc3ViZGlyOiBzdHIgPSAiY29udGVudC8iLAogICAgICAgIGtleTogc3RyID0gImNvbnRlbnQiLAogICAgICAgIHRhcmdldF9wYXRoOiBzdHIgPSBOb25lLAopOgogICAgIiIiT3BlbiBhIGZpbGUvb2JqZWN0IGFyY2hpdmUgaW50byBhIHRhcmdldCBkaXJlY3RvcnkuIEN1cnJlbnRseSwgc3VwcG9ydHMgemlwIGFuZCB0YXIuZ3ouCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgZnVuY3Rpb24gZXhlY3V0aW9uIGNvbnRleHQKICAgIDpwYXJhbSBhcmNoaXZlX3VybDogIHVybCBvZiBhcmNoaXZlIGZpbGUKICAgIDpwYXJhbSBzdWJkaXI6ICAgICAgIHBhdGggd2l0aGluIGFydGlmYWN0IHN0b3JlIHdoZXJlIGV4dHJhY3RlZCBmaWxlcyBhcmUgc3RvcmVkLCBkZWZhdWx0IGlzICIvY29udGVudCIKICAgIDpwYXJhbSBrZXk6ICAgICAgICAgIGtleSBvZiBhcmNoaXZlIGNvbnRlbnRzIGluIGFydGlmYWN0IHN0b3JlCiAgICA6cGFyYW0gdGFyZ2V0X3BhdGg6ICBmaWxlIHN5c3RlbSBwYXRoIHRvIHN0b3JlIGV4dHJhY3RlZCBmaWxlcwogICAgIiIiCgogICAgIyBSZXNvbHZlcyB0aGUgYXJjaGl2ZSBsb2NhbGx5CiAgICBhcmNoaXZlX3VybCA9IGFyY2hpdmVfdXJsLmxvY2FsKCkKICAgIHYzaW9fc3ViZGlyID0gTm9uZQogICAgIyBXaGVuIGN1c3RvbSBhcnRpZmFjdCBwYXRoIGlzIGRlZmluZWQKICAgIGlmIG5vdCB0YXJnZXRfcGF0aCBhbmQgY29udGV4dC5hcnRpZmFjdF9wYXRoOgogICAgICAgIHBhcnNlZF9zdWJkaXIgPSB1cmxwYXJzZShjb250ZXh0LmFydGlmYWN0X3BhdGgpCiAgICAgICAgaWYgcGFyc2VkX3N1YmRpci5zY2hlbWUgPT0gJ3MzJzoKICAgICAgICAgICAgc3ViZGlyID0gb3MucGF0aC5qb2luKGNvbnRleHQuYXJ0aWZhY3RfcGF0aCwgc3ViZGlyKQogICAgICAgIGVsaWYgcGFyc2VkX3N1YmRpci5zY2hlbWUgPT0gJ3YzaW8nOgogICAgICAgICAgICB2M2lvX3N1YmRpciA9IG9zLnBhdGguam9pbihjb250ZXh0LmFydGlmYWN0X3BhdGgsIHN1YmRpcikgICMgVXNpbmcgdjNpb19zdWJkaXIgZm9yIGxvZ2dpbmcKICAgICAgICAgICAgc3ViZGlyID0gJy92M2lvJyArIHBhcnNlZF9zdWJkaXIucGF0aCArICcvJyArIHN1YmRpcgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYnVXNpbmcgdjNpbyBzY2hlbWUsIGV4dHJhY3RpbmcgdG8ge3N1YmRpcn0nKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZidVbnJlY29nbml6YWJsZSBzY2hlbWUsIGV4dHJhY3RpbmcgdG8ge3N1YmRpcn0nKQoKICAgICMgV2hlbiB3b3JraW5nIG9uIENFLCB0YXJnZXQgcGF0aCBtaWdodCBiZSBvbiBzMwogICAgaWYgJ3MzJyBpbiAodGFyZ2V0X3BhdGggb3Igc3ViZGlyKToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYnVXNpbmcgczMgc2NoZW1lLCBleHRyYWN0aW5nIHRvIHt0YXJnZXRfcGF0aCBvciBzdWJkaXJ9JykKCiAgICAgICAgaWYgYXJjaGl2ZV91cmwuZW5kc3dpdGgoImd6Iik6CiAgICAgICAgICAgIF9leHRyYWN0X2d6X2ZpbGUoYXJjaGl2ZV91cmw9YXJjaGl2ZV91cmwsIHN1YmRpcj1zdWJkaXIsIHRhcmdldF9wYXRoPXRhcmdldF9wYXRoLCBpbl9zMz1UcnVlKQoKICAgICAgICBlbGlmIGFyY2hpdmVfdXJsLmVuZHN3aXRoKCJ6aXAiKToKICAgICAgICAgICAgX2V4dHJhY3RfemlwX2ZpbGUoYXJjaGl2ZV91cmw9YXJjaGl2ZV91cmwsIHN1YmRpcj1zdWJkaXIsIHRhcmdldF9wYXRoPXRhcmdldF9wYXRoLCBpbl9zMz1UcnVlKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoZiJ1bnN1cHBvcnRlZCBhcmNoaXZlIHR5cGUgaW4ge2FyY2hpdmVfdXJsfSIpCiAgICBlbHNlOgogICAgICAgIGlmIGFyY2hpdmVfdXJsLmVuZHN3aXRoKCJneiIpOgogICAgICAgICAgICBfZXh0cmFjdF9nel9maWxlKGFyY2hpdmVfdXJsPWFyY2hpdmVfdXJsLCBzdWJkaXI9c3ViZGlyLCB0YXJnZXRfcGF0aD10YXJnZXRfcGF0aCkKICAgICAgICBlbGlmIGFyY2hpdmVfdXJsLmVuZHN3aXRoKCJ6aXAiKToKICAgICAgICAgICAgX2V4dHJhY3RfemlwX2ZpbGUoYXJjaGl2ZV91cmw9YXJjaGl2ZV91cmwsIHN1YmRpcj1zdWJkaXIsIHRhcmdldF9wYXRoPXRhcmdldF9wYXRoKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoZiJ1bnN1cHBvcnRlZCBhcmNoaXZlIHR5cGUgaW4ge2FyY2hpdmVfdXJsfSIpCgogICAgaWYgdjNpb19zdWJkaXI6CiAgICAgICAgc3ViZGlyID0gdjNpb19zdWJkaXIKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYnTG9nZ2luZyBhcnRpZmFjdCB0byB7KHRhcmdldF9wYXRoIG9yIHN1YmRpcil9JykKICAgIGNvbnRleHQubG9nX2FydGlmYWN0KERpckFydGlmYWN0KGtleT1rZXksIHRhcmdldF9wYXRoPSh0YXJnZXRfcGF0aCBvciBzdWJkaXIpKSkKCgpkZWYgX2V4dHJhY3RfZ3pfZmlsZShhcmNoaXZlX3VybDogc3RyLCB0YXJnZXRfcGF0aDogc3RyID0gTm9uZSwgc3ViZGlyOiBzdHIgPSAiY29udGVudC8iLCBpbl9zMzogYm9vbCA9IEZhbHNlKToKICAgIGlmIGluX3MzOgogICAgICAgIGNsaWVudCA9IF9pbml0X2JvdG8zX2NsaWVudCgpCiAgICAgICAgd2l0aCB0YXJmaWxlLm9wZW4oYXJjaGl2ZV91cmwsIG1vZGU9InJ8Z3oiKSBhcyByZWY6CiAgICAgICAgICAgIGZvciBtZW1iZXIgaW4gcmVmLmdldG1lbWJlcnMoKToKICAgICAgICAgICAgICAgIGRhdGEgPSByZWYuZXh0cmFjdGZpbGUobWVtYmVyPW1lbWJlcikucmVhZCgpCiAgICAgICAgICAgICAgICBjbGllbnQucHV0X29iamVjdChCb2R5PWRhdGEsIEJ1Y2tldD11cmxwYXJzZSh0YXJnZXRfcGF0aCBvciBzdWJkaXIpLm5ldGxvYywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIEtleT1mJ3t1cmxwYXJzZSh0YXJnZXRfcGF0aCBvciBzdWJkaXIpLnBhdGhbMTpdfXttZW1iZXIubmFtZX0nKQogICAgZWxzZToKICAgICAgICBvcy5tYWtlZGlycyh0YXJnZXRfcGF0aCBvciBzdWJkaXIsIGV4aXN0X29rPVRydWUpCiAgICAgICAgd2l0aCB0YXJmaWxlLm9wZW4oYXJjaGl2ZV91cmwsIG1vZGU9InI6Z3oiKSBhcyByZWY6CiAgICAgICAgICAgIGZvciBlbnRyeSBpbiByZWY6CiAgICAgICAgICAgICAgICAjIFZhbGlkYXRlIHRoYXQgdGhlcmUgaXMgbm8gcGF0aCB0cmF2ZXJzYWwgaW4gdGhlIGFyY2hpdmUKICAgICAgICAgICAgICAgIGlmIG9zLnBhdGguaXNhYnMoZW50cnkubmFtZSkgb3IgIi4uIiBpbiBlbnRyeS5uYW1lOgogICAgICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoZiJJbGxlZ2FsIHRhciBhcmNoaXZlIGVudHJ5OiB7ZW50cnkubmFtZX0iKQoKICAgICAgICAgICAgICAgIHJlZi5leHRyYWN0KGVudHJ5LCB0YXJnZXRfcGF0aCBvciBzdWJkaXIpCgoKZGVmIF9leHRyYWN0X3ppcF9maWxlKGFyY2hpdmVfdXJsLCB0YXJnZXRfcGF0aDogc3RyID0gTm9uZSwgc3ViZGlyOiBzdHIgPSAiY29udGVudC8iLCBpbl9zMzogYm9vbCA9IEZhbHNlKToKICAgIGlmIGluX3MzOgogICAgICAgIGNsaWVudCA9IF9pbml0X2JvdG8zX2NsaWVudCgpCiAgICAgICAgd2l0aCB6aXBmaWxlLlppcEZpbGUoYXJjaGl2ZV91cmwsICJyIikgYXMgcmVmOgogICAgICAgICAgICBmb3IgZmlsZW5hbWUgaW4gcmVmLm5hbWVsaXN0KCk6CiAgICAgICAgICAgICAgICBkYXRhID0gcmVmLnJlYWQoZmlsZW5hbWUpCiAgICAgICAgICAgICAgICBjbGllbnQucHV0X29iamVjdChCb2R5PWRhdGEsIEJ1Y2tldD11cmxwYXJzZSh0YXJnZXRfcGF0aCBvciBzdWJkaXIpLm5ldGxvYywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIEtleT1mJ3t1cmxwYXJzZSh0YXJnZXRfcGF0aCBvciBzdWJkaXIpLnBhdGhbMTpdfXtmaWxlbmFtZX0nKQogICAgZWxzZToKICAgICAgICB3aXRoIHppcGZpbGUuWmlwRmlsZShhcmNoaXZlX3VybCwgInIiKSBhcyByZWY6CiAgICAgICAgICAgICMgVmFsaWRhdGUgdGhhdCB0aGVyZSBpcyBubyBwYXRoIHRyYXZlcnNhbCBpbiB0aGUgYXJjaGl2ZQogICAgICAgICAgICBmb3IgZW50cnkgaW4gcmVmLm5hbWVsaXN0KCk6CiAgICAgICAgICAgICAgICBpZiBvcy5wYXRoLmlzYWJzKGVudHJ5KSBvciAiLi4iIGluIGVudHJ5OgogICAgICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoZiJJbGxlZ2FsIHppcCBhcmNoaXZlIGVudHJ5OiB7ZW50cnl9IikKICAgICAgICAgICAgb3MubWFrZWRpcnModGFyZ2V0X3BhdGggb3Igc3ViZGlyLCBleGlzdF9vaz1UcnVlKQogICAgICAgICAgICByZWYuZXh0cmFjdGFsbCh0YXJnZXRfcGF0aCBvciBzdWJkaXIpCgoKZGVmIF9pbml0X2JvdG8zX2NsaWVudCgpOgogICAgaW1wb3J0IGJvdG8zCiAgICBpZiBvcy5lbnZpcm9uLmdldCgnUzNfRU5EUE9JTlRfVVJMJyk6CiAgICAgICAgY2xpZW50ID0gYm90bzMuY2xpZW50KCdzMycsIGVuZHBvaW50X3VybD1vcy5lbnZpcm9uLmdldCgnUzNfRU5EUE9JTlRfVVJMJykpCiAgICBlbHNlOgogICAgICAgIGNsaWVudCA9IGJvdG8zLmNsaWVudCgnczMnKQogICAgcmV0dXJuIGNsaWVudA== - code_origin: '' origin_filename: '' - description: Open a file/object archive into a target directory - image: mlrun/mlrun + functionSourceCode: IyBDb3B5cmlnaHQgMjAyNSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKCmltcG9ydCBvcwppbXBvcnQgdGFyZmlsZQppbXBvcnQgemlwZmlsZQpmcm9tIHVybGxpYi5wYXJzZSBpbXBvcnQgdXJscGFyc2UKCmZyb20gbWxydW4uYXJ0aWZhY3RzLmJhc2UgaW1wb3J0IERpckFydGlmYWN0CmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKCgpkZWYgb3Blbl9hcmNoaXZlKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBhcmNoaXZlX3VybDogRGF0YUl0ZW0sCiAgICBzdWJkaXI6IHN0ciA9ICJjb250ZW50LyIsCiAgICBrZXk6IHN0ciA9ICJjb250ZW50IiwKICAgIHRhcmdldF9wYXRoOiBzdHIgPSBOb25lLAopOgogICAgIiIiT3BlbiBhIGZpbGUvb2JqZWN0IGFyY2hpdmUgaW50byBhIHRhcmdldCBkaXJlY3RvcnkuIEN1cnJlbnRseSwgc3VwcG9ydHMgemlwIGFuZCB0YXIuZ3ouCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgZnVuY3Rpb24gZXhlY3V0aW9uIGNvbnRleHQKICAgIDpwYXJhbSBhcmNoaXZlX3VybDogIHVybCBvZiBhcmNoaXZlIGZpbGUKICAgIDpwYXJhbSBzdWJkaXI6ICAgICAgIHBhdGggd2l0aGluIGFydGlmYWN0IHN0b3JlIHdoZXJlIGV4dHJhY3RlZCBmaWxlcyBhcmUgc3RvcmVkLCBkZWZhdWx0IGlzICIvY29udGVudCIKICAgIDpwYXJhbSBrZXk6ICAgICAgICAgIGtleSBvZiBhcmNoaXZlIGNvbnRlbnRzIGluIGFydGlmYWN0IHN0b3JlCiAgICA6cGFyYW0gdGFyZ2V0X3BhdGg6ICBmaWxlIHN5c3RlbSBwYXRoIHRvIHN0b3JlIGV4dHJhY3RlZCBmaWxlcwogICAgIiIiCgogICAgIyBSZXNvbHZlcyB0aGUgYXJjaGl2ZSBsb2NhbGx5CiAgICBhcmNoaXZlX3VybCA9IGFyY2hpdmVfdXJsLmxvY2FsKCkKICAgIHYzaW9fc3ViZGlyID0gTm9uZQogICAgIyBXaGVuIGN1c3RvbSBhcnRpZmFjdCBwYXRoIGlzIGRlZmluZWQKICAgIGlmIG5vdCB0YXJnZXRfcGF0aCBhbmQgY29udGV4dC5hcnRpZmFjdF9wYXRoOgogICAgICAgIHBhcnNlZF9zdWJkaXIgPSB1cmxwYXJzZShjb250ZXh0LmFydGlmYWN0X3BhdGgpCiAgICAgICAgaWYgcGFyc2VkX3N1YmRpci5zY2hlbWUgPT0gInMzIjoKICAgICAgICAgICAgc3ViZGlyID0gb3MucGF0aC5qb2luKGNvbnRleHQuYXJ0aWZhY3RfcGF0aCwgc3ViZGlyKQogICAgICAgIGVsaWYgcGFyc2VkX3N1YmRpci5zY2hlbWUgPT0gInYzaW8iOgogICAgICAgICAgICB2M2lvX3N1YmRpciA9IG9zLnBhdGguam9pbigKICAgICAgICAgICAgICAgIGNvbnRleHQuYXJ0aWZhY3RfcGF0aCwgc3ViZGlyCiAgICAgICAgICAgICkgICMgVXNpbmcgdjNpb19zdWJkaXIgZm9yIGxvZ2dpbmcKICAgICAgICAgICAgc3ViZGlyID0gIi92M2lvIiArIHBhcnNlZF9zdWJkaXIucGF0aCArICIvIiArIHN1YmRpcgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVXNpbmcgdjNpbyBzY2hlbWUsIGV4dHJhY3RpbmcgdG8ge3N1YmRpcn0iKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJVbnJlY29nbml6YWJsZSBzY2hlbWUsIGV4dHJhY3RpbmcgdG8ge3N1YmRpcn0iKQoKICAgICMgV2hlbiB3b3JraW5nIG9uIENFLCB0YXJnZXQgcGF0aCBtaWdodCBiZSBvbiBzMwogICAgaWYgInMzIiBpbiAodGFyZ2V0X3BhdGggb3Igc3ViZGlyKToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVXNpbmcgczMgc2NoZW1lLCBleHRyYWN0aW5nIHRvIHt0YXJnZXRfcGF0aCBvciBzdWJkaXJ9IikKCiAgICAgICAgaWYgYXJjaGl2ZV91cmwuZW5kc3dpdGgoImd6Iik6CiAgICAgICAgICAgIF9leHRyYWN0X2d6X2ZpbGUoCiAgICAgICAgICAgICAgICBhcmNoaXZlX3VybD1hcmNoaXZlX3VybCwKICAgICAgICAgICAgICAgIHN1YmRpcj1zdWJkaXIsCiAgICAgICAgICAgICAgICB0YXJnZXRfcGF0aD10YXJnZXRfcGF0aCwKICAgICAgICAgICAgICAgIGluX3MzPVRydWUsCiAgICAgICAgICAgICkKCiAgICAgICAgZWxpZiBhcmNoaXZlX3VybC5lbmRzd2l0aCgiemlwIik6CiAgICAgICAgICAgIF9leHRyYWN0X3ppcF9maWxlKAogICAgICAgICAgICAgICAgYXJjaGl2ZV91cmw9YXJjaGl2ZV91cmwsCiAgICAgICAgICAgICAgICBzdWJkaXI9c3ViZGlyLAogICAgICAgICAgICAgICAgdGFyZ2V0X3BhdGg9dGFyZ2V0X3BhdGgsCiAgICAgICAgICAgICAgICBpbl9zMz1UcnVlLAogICAgICAgICAgICApCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcihmInVuc3VwcG9ydGVkIGFyY2hpdmUgdHlwZSBpbiB7YXJjaGl2ZV91cmx9IikKICAgIGVsc2U6CiAgICAgICAgaWYgYXJjaGl2ZV91cmwuZW5kc3dpdGgoImd6Iik6CiAgICAgICAgICAgIF9leHRyYWN0X2d6X2ZpbGUoCiAgICAgICAgICAgICAgICBhcmNoaXZlX3VybD1hcmNoaXZlX3VybCwgc3ViZGlyPXN1YmRpciwgdGFyZ2V0X3BhdGg9dGFyZ2V0X3BhdGgKICAgICAgICAgICAgKQogICAgICAgIGVsaWYgYXJjaGl2ZV91cmwuZW5kc3dpdGgoInppcCIpOgogICAgICAgICAgICBfZXh0cmFjdF96aXBfZmlsZSgKICAgICAgICAgICAgICAgIGFyY2hpdmVfdXJsPWFyY2hpdmVfdXJsLCBzdWJkaXI9c3ViZGlyLCB0YXJnZXRfcGF0aD10YXJnZXRfcGF0aAogICAgICAgICAgICApCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcihmInVuc3VwcG9ydGVkIGFyY2hpdmUgdHlwZSBpbiB7YXJjaGl2ZV91cmx9IikKCiAgICBpZiB2M2lvX3N1YmRpcjoKICAgICAgICBzdWJkaXIgPSB2M2lvX3N1YmRpcgoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJMb2dnaW5nIGFydGlmYWN0IHRvIHsodGFyZ2V0X3BhdGggb3Igc3ViZGlyKX0iKQogICAgY29udGV4dC5sb2dfYXJ0aWZhY3QoRGlyQXJ0aWZhY3Qoa2V5PWtleSwgdGFyZ2V0X3BhdGg9KHRhcmdldF9wYXRoIG9yIHN1YmRpcikpKQoKCmRlZiBfZXh0cmFjdF9nel9maWxlKAogICAgYXJjaGl2ZV91cmw6IHN0ciwKICAgIHRhcmdldF9wYXRoOiBzdHIgPSBOb25lLAogICAgc3ViZGlyOiBzdHIgPSAiY29udGVudC8iLAogICAgaW5fczM6IGJvb2wgPSBGYWxzZSwKKToKICAgIGlmIGluX3MzOgogICAgICAgIGNsaWVudCA9IF9pbml0X2JvdG8zX2NsaWVudCgpCiAgICAgICAgd2l0aCB0YXJmaWxlLm9wZW4oYXJjaGl2ZV91cmwsIG1vZGU9InJ8Z3oiKSBhcyByZWY6CiAgICAgICAgICAgIGZvciBtZW1iZXIgaW4gcmVmLmdldG1lbWJlcnMoKToKICAgICAgICAgICAgICAgIGRhdGEgPSByZWYuZXh0cmFjdGZpbGUobWVtYmVyPW1lbWJlcikucmVhZCgpCiAgICAgICAgICAgICAgICBjbGllbnQucHV0X29iamVjdCgKICAgICAgICAgICAgICAgICAgICBCb2R5PWRhdGEsCiAgICAgICAgICAgICAgICAgICAgQnVja2V0PXVybHBhcnNlKHRhcmdldF9wYXRoIG9yIHN1YmRpcikubmV0bG9jLAogICAgICAgICAgICAgICAgICAgIEtleT1mInt1cmxwYXJzZSh0YXJnZXRfcGF0aCBvciBzdWJkaXIpLnBhdGhbMTpdfXttZW1iZXIubmFtZX0iLAogICAgICAgICAgICAgICAgKQogICAgZWxzZToKICAgICAgICBvcy5tYWtlZGlycyh0YXJnZXRfcGF0aCBvciBzdWJkaXIsIGV4aXN0X29rPVRydWUpCiAgICAgICAgd2l0aCB0YXJmaWxlLm9wZW4oYXJjaGl2ZV91cmwsIG1vZGU9InI6Z3oiKSBhcyByZWY6CiAgICAgICAgICAgIGZvciBlbnRyeSBpbiByZWY6CiAgICAgICAgICAgICAgICAjIFZhbGlkYXRlIHRoYXQgdGhlcmUgaXMgbm8gcGF0aCB0cmF2ZXJzYWwgaW4gdGhlIGFyY2hpdmUKICAgICAgICAgICAgICAgIGlmIG9zLnBhdGguaXNhYnMoZW50cnkubmFtZSkgb3IgIi4uIiBpbiBlbnRyeS5uYW1lOgogICAgICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoZiJJbGxlZ2FsIHRhciBhcmNoaXZlIGVudHJ5OiB7ZW50cnkubmFtZX0iKQoKICAgICAgICAgICAgICAgIHJlZi5leHRyYWN0KGVudHJ5LCB0YXJnZXRfcGF0aCBvciBzdWJkaXIpCgoKZGVmIF9leHRyYWN0X3ppcF9maWxlKAogICAgYXJjaGl2ZV91cmwsIHRhcmdldF9wYXRoOiBzdHIgPSBOb25lLCBzdWJkaXI6IHN0ciA9ICJjb250ZW50LyIsIGluX3MzOiBib29sID0gRmFsc2UKKToKICAgIGlmIGluX3MzOgogICAgICAgIGNsaWVudCA9IF9pbml0X2JvdG8zX2NsaWVudCgpCiAgICAgICAgd2l0aCB6aXBmaWxlLlppcEZpbGUoYXJjaGl2ZV91cmwsICJyIikgYXMgcmVmOgogICAgICAgICAgICBmb3IgZmlsZW5hbWUgaW4gcmVmLm5hbWVsaXN0KCk6CiAgICAgICAgICAgICAgICBkYXRhID0gcmVmLnJlYWQoZmlsZW5hbWUpCiAgICAgICAgICAgICAgICBjbGllbnQucHV0X29iamVjdCgKICAgICAgICAgICAgICAgICAgICBCb2R5PWRhdGEsCiAgICAgICAgICAgICAgICAgICAgQnVja2V0PXVybHBhcnNlKHRhcmdldF9wYXRoIG9yIHN1YmRpcikubmV0bG9jLAogICAgICAgICAgICAgICAgICAgIEtleT1mInt1cmxwYXJzZSh0YXJnZXRfcGF0aCBvciBzdWJkaXIpLnBhdGhbMTpdfXtmaWxlbmFtZX0iLAogICAgICAgICAgICAgICAgKQogICAgZWxzZToKICAgICAgICB3aXRoIHppcGZpbGUuWmlwRmlsZShhcmNoaXZlX3VybCwgInIiKSBhcyByZWY6CiAgICAgICAgICAgICMgVmFsaWRhdGUgdGhhdCB0aGVyZSBpcyBubyBwYXRoIHRyYXZlcnNhbCBpbiB0aGUgYXJjaGl2ZQogICAgICAgICAgICBmb3IgZW50cnkgaW4gcmVmLm5hbWVsaXN0KCk6CiAgICAgICAgICAgICAgICBpZiBvcy5wYXRoLmlzYWJzKGVudHJ5KSBvciAiLi4iIGluIGVudHJ5OgogICAgICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoZiJJbGxlZ2FsIHppcCBhcmNoaXZlIGVudHJ5OiB7ZW50cnl9IikKICAgICAgICAgICAgb3MubWFrZWRpcnModGFyZ2V0X3BhdGggb3Igc3ViZGlyLCBleGlzdF9vaz1UcnVlKQogICAgICAgICAgICByZWYuZXh0cmFjdGFsbCh0YXJnZXRfcGF0aCBvciBzdWJkaXIpCgoKZGVmIF9pbml0X2JvdG8zX2NsaWVudCgpOgogICAgaW1wb3J0IGJvdG8zCgogICAgIyBCYWNrd2FyZCBjb21wYXRpYmlsaXR5OiBTdXBwb3J0IGJvdGggUzNfRU5EUE9JTlRfVVJMIChkZXByZWNhdGVkKSBhbmQgQVdTX0VORFBPSU5UX1VSTF9TMwogICAgIyBUT0RPOiBSZW1vdmUgdGhpcyBpbiAxLjEyLjAKICAgIGVuZHBvaW50X3VybCA9IG9zLmVudmlyb24uZ2V0KCJBV1NfRU5EUE9JTlRfVVJMX1MzIikgb3Igb3MuZW52aXJvbi5nZXQoCiAgICAgICAgIlMzX0VORFBPSU5UX1VSTCIKICAgICkKCiAgICBpZiBlbmRwb2ludF91cmw6CiAgICAgICAgY2xpZW50ID0gYm90bzMuY2xpZW50KCJzMyIsIGVuZHBvaW50X3VybD1lbmRwb2ludF91cmwpCiAgICBlbHNlOgogICAgICAgIGNsaWVudCA9IGJvdG8zLmNsaWVudCgiczMiKQogICAgcmV0dXJuIGNsaWVudAo= + code_origin: '' + filename: open_archive.py entry_points: open_archive: - has_kwargs: false - lineno: 27 - name: open_archive parameters: - name: context type: MLClientCtx @@ -35,11 +35,12 @@ spec: type: str doc: file system path to store extracted files default: null + name: open_archive doc: Open a file/object archive into a target directory. Currently, supports zip and tar.gz. + has_kwargs: false has_varargs: false -metadata: - name: open-archive - categories: - - utils - tag: '' + lineno: 26 + command: '' + description: Open a file/object archive into a target directory + default_handler: open_archive diff --git a/functions/src/open_archive/item.yaml b/functions/src/open_archive/item.yaml index c40a62e4a..adcc4c69e 100644 --- a/functions/src/open_archive/item.yaml +++ b/functions/src/open_archive/item.yaml @@ -11,7 +11,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.8.0-rc50 +mlrunVersion: 1.8.0 name: open-archive platformVersion: 3.5.0 spec: diff --git a/functions/src/open_archive/open_archive.py b/functions/src/open_archive/open_archive.py index 19d3c757b..225edb224 100644 --- a/functions/src/open_archive/open_archive.py +++ b/functions/src/open_archive/open_archive.py @@ -14,22 +14,21 @@ # import os -import zipfile import tarfile +import zipfile +from urllib.parse import urlparse -from mlrun.execution import MLClientCtx -from mlrun.datastore import DataItem from mlrun.artifacts.base import DirArtifact - -from urllib.parse import urlparse +from mlrun.datastore import DataItem +from mlrun.execution import MLClientCtx def open_archive( - context: MLClientCtx, - archive_url: DataItem, - subdir: str = "content/", - key: str = "content", - target_path: str = None, + context: MLClientCtx, + archive_url: DataItem, + subdir: str = "content/", + key: str = "content", + target_path: str = None, ): """Open a file/object archive into a target directory. Currently, supports zip and tar.gz. @@ -46,49 +45,73 @@ def open_archive( # When custom artifact path is defined if not target_path and context.artifact_path: parsed_subdir = urlparse(context.artifact_path) - if parsed_subdir.scheme == 's3': + if parsed_subdir.scheme == "s3": subdir = os.path.join(context.artifact_path, subdir) - elif parsed_subdir.scheme == 'v3io': - v3io_subdir = os.path.join(context.artifact_path, subdir) # Using v3io_subdir for logging - subdir = '/v3io' + parsed_subdir.path + '/' + subdir - context.logger.info(f'Using v3io scheme, extracting to {subdir}') + elif parsed_subdir.scheme == "v3io": + v3io_subdir = os.path.join( + context.artifact_path, subdir + ) # Using v3io_subdir for logging + subdir = "/v3io" + parsed_subdir.path + "/" + subdir + context.logger.info(f"Using v3io scheme, extracting to {subdir}") else: - context.logger.info(f'Unrecognizable scheme, extracting to {subdir}') + context.logger.info(f"Unrecognizable scheme, extracting to {subdir}") # When working on CE, target path might be on s3 - if 's3' in (target_path or subdir): - context.logger.info(f'Using s3 scheme, extracting to {target_path or subdir}') + if "s3" in (target_path or subdir): + context.logger.info(f"Using s3 scheme, extracting to {target_path or subdir}") if archive_url.endswith("gz"): - _extract_gz_file(archive_url=archive_url, subdir=subdir, target_path=target_path, in_s3=True) + _extract_gz_file( + archive_url=archive_url, + subdir=subdir, + target_path=target_path, + in_s3=True, + ) elif archive_url.endswith("zip"): - _extract_zip_file(archive_url=archive_url, subdir=subdir, target_path=target_path, in_s3=True) + _extract_zip_file( + archive_url=archive_url, + subdir=subdir, + target_path=target_path, + in_s3=True, + ) else: raise ValueError(f"unsupported archive type in {archive_url}") else: if archive_url.endswith("gz"): - _extract_gz_file(archive_url=archive_url, subdir=subdir, target_path=target_path) + _extract_gz_file( + archive_url=archive_url, subdir=subdir, target_path=target_path + ) elif archive_url.endswith("zip"): - _extract_zip_file(archive_url=archive_url, subdir=subdir, target_path=target_path) + _extract_zip_file( + archive_url=archive_url, subdir=subdir, target_path=target_path + ) else: raise ValueError(f"unsupported archive type in {archive_url}") if v3io_subdir: subdir = v3io_subdir - context.logger.info(f'Logging artifact to {(target_path or subdir)}') + context.logger.info(f"Logging artifact to {(target_path or subdir)}") context.log_artifact(DirArtifact(key=key, target_path=(target_path or subdir))) -def _extract_gz_file(archive_url: str, target_path: str = None, subdir: str = "content/", in_s3: bool = False): +def _extract_gz_file( + archive_url: str, + target_path: str = None, + subdir: str = "content/", + in_s3: bool = False, +): if in_s3: client = _init_boto3_client() with tarfile.open(archive_url, mode="r|gz") as ref: for member in ref.getmembers(): data = ref.extractfile(member=member).read() - client.put_object(Body=data, Bucket=urlparse(target_path or subdir).netloc, - Key=f'{urlparse(target_path or subdir).path[1:]}{member.name}') + client.put_object( + Body=data, + Bucket=urlparse(target_path or subdir).netloc, + Key=f"{urlparse(target_path or subdir).path[1:]}{member.name}", + ) else: os.makedirs(target_path or subdir, exist_ok=True) with tarfile.open(archive_url, mode="r:gz") as ref: @@ -100,14 +123,19 @@ def _extract_gz_file(archive_url: str, target_path: str = None, subdir: str = "c ref.extract(entry, target_path or subdir) -def _extract_zip_file(archive_url, target_path: str = None, subdir: str = "content/", in_s3: bool = False): +def _extract_zip_file( + archive_url, target_path: str = None, subdir: str = "content/", in_s3: bool = False +): if in_s3: client = _init_boto3_client() with zipfile.ZipFile(archive_url, "r") as ref: for filename in ref.namelist(): data = ref.read(filename) - client.put_object(Body=data, Bucket=urlparse(target_path or subdir).netloc, - Key=f'{urlparse(target_path or subdir).path[1:]}{filename}') + client.put_object( + Body=data, + Bucket=urlparse(target_path or subdir).netloc, + Key=f"{urlparse(target_path or subdir).path[1:]}{filename}", + ) else: with zipfile.ZipFile(archive_url, "r") as ref: # Validate that there is no path traversal in the archive @@ -120,13 +148,15 @@ def _extract_zip_file(archive_url, target_path: str = None, subdir: str = "conte def _init_boto3_client(): import boto3 - + # Backward compatibility: Support both S3_ENDPOINT_URL (deprecated) and AWS_ENDPOINT_URL_S3 # TODO: Remove this in 1.12.0 - endpoint_url = os.environ.get('AWS_ENDPOINT_URL_S3') or os.environ.get('S3_ENDPOINT_URL') - + endpoint_url = os.environ.get("AWS_ENDPOINT_URL_S3") or os.environ.get( + "S3_ENDPOINT_URL" + ) + if endpoint_url: - client = boto3.client('s3', endpoint_url=endpoint_url) + client = boto3.client("s3", endpoint_url=endpoint_url) else: - client = boto3.client('s3') - return client \ No newline at end of file + client = boto3.client("s3") + return client diff --git a/functions/src/open_archive/test_open_archive.py b/functions/src/open_archive/test_open_archive.py index 507c7ecbc..29fcafc99 100644 --- a/functions/src/open_archive/test_open_archive.py +++ b/functions/src/open_archive/test_open_archive.py @@ -12,17 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. # -from pathlib import Path -import shutil import os +import shutil import tarfile -from mlrun import code_to_function, import_function +from pathlib import Path + import open_archive import pytest +from mlrun import code_to_function, import_function -ARTIFACTS_PATH = 'artifacts' -CONTENT_PATH = 'content/data/images' -ARCHIVE_URL = "https://s3.wasabisys.com/iguazio/data/cats-vs-dogs/cats-vs-dogs-labeling-demo.zip" +ARTIFACTS_PATH = "artifacts" +CONTENT_PATH = "content/data/images" +ARCHIVE_URL = ( + "https://s3.wasabisys.com/iguazio/data/cats-vs-dogs/cats-vs-dogs-labeling-demo.zip" +) def _delete_outputs(paths): @@ -32,27 +35,32 @@ def _delete_outputs(paths): def test_open_archive(): - fn = code_to_function(name='test_open_archive', - filename="open_archive.py", - handler="open_archive", - kind="local", - ) + fn = code_to_function( + name="test_open_archive", + filename="open_archive.py", + handler="open_archive", + kind="local", + ) fn.spec.command = "open_archive.py" - fn.run(inputs={'archive_url': ARCHIVE_URL}, - params={'key': 'test_archive', 'target_path': os.getcwd() + '/content/'}, - local=True) + fn.run( + inputs={"archive_url": ARCHIVE_URL}, + params={"key": "test_archive", "target_path": os.getcwd() + "/content/"}, + local=True, + ) assert Path(CONTENT_PATH).is_dir() - _delete_outputs({'artifacts', 'runs', 'schedules', 'content'}) + _delete_outputs({"artifacts", "runs", "schedules", "content"}) def test_open_archive_import_function(): fn = import_function("function.yaml") - run = fn.run(inputs={'archive_url': ARCHIVE_URL}, - params={'key': 'test_archive', 'target_path': os.getcwd() + '/content/'}, - local=True) - assert (run.status.artifact_uris["test_archive"]) - _delete_outputs({'artifacts', 'runs', 'schedules', 'content'}) + run = fn.run( + inputs={"archive_url": ARCHIVE_URL}, + params={"key": "test_archive", "target_path": os.getcwd() + "/content/"}, + local=True, + ) + assert run.status.artifact_uris["test_archive"] + _delete_outputs({"artifacts", "runs", "schedules", "content"}) def test_traversal_entry(): @@ -65,6 +73,8 @@ def test_traversal_entry(): tar.add("malicious.txt", arcname="../malicious.txt") with pytest.raises(ValueError): - open_archive._extract_gz_file("malicious.tar.gz", target_path=os.getcwd() + '/content/') + open_archive._extract_gz_file( + "malicious.tar.gz", target_path=os.getcwd() + "/content/" + ) os.remove("malicious.txt") - os.remove("malicious.tar.gz") \ No newline at end of file + os.remove("malicious.tar.gz") diff --git a/functions/src/pii_recognizer/function.yaml b/functions/src/pii_recognizer/function.yaml index e7d6c1241..d3bc1516e 100644 --- a/functions/src/pii_recognizer/function.yaml +++ b/functions/src/pii_recognizer/function.yaml @@ -1,42 +1,61 @@ +metadata: + tag: '' + name: pii-recognizer + categories: + - data-preparation + - NLP verbose: false +kind: job spec: - default_handler: recognize_pii + image: '' + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgoKaW1wb3J0IGxvZ2dpbmcKaW1wb3J0IG9zCmltcG9ydCBwYXRobGliCmltcG9ydCB0ZW1wZmlsZQppbXBvcnQgd2FybmluZ3MKZnJvbSB0eXBpbmcgaW1wb3J0IExpc3QKCmltcG9ydCBhbm5vdGF0ZWRfdGV4dC51dGlsIGFzIGF0X3V0aWwKaW1wb3J0IG1scnVuCmltcG9ydCBubHRrCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IHByZXNpZGlvX2FuYWx5emVyIGFzIHBhCmltcG9ydCBwcmVzaWRpb19hbm9ueW1pemVyIGFzIHByZV9hbm95bWl6ZXIKZnJvbSBwcmVzaWRpb19hbm9ueW1pemVyLmVudGl0aWVzIGltcG9ydCBPcGVyYXRvckNvbmZpZwpmcm9tIHRxZG0gaW1wb3J0IHRxZG0KCnRyeToKICAgIGltcG9ydCBmbGFpciBhcyBmbApleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvcjoKICAgIHByaW50KCJGbGFpciBpcyBub3QgaW5zdGFsbGVkIikKCiMgVGhlcmUgaXMgYSBjb25mbGljdCBiZXR3ZWVuIFJ1c3QtYmFzZWQgdG9rZW5pemVycycgcGFyYWxsZWwgcHJvY2Vzc2luZwojIGFuZCBQeXRob24ncyBmb3JrIG9wZXJhdGlvbnMgZHVyaW5nIG11bHRpcHJvY2Vzc2luZy4gVG8gYXZvaWQgdGhpcywgd2UgbmVlZAojIHRoZSBmb2xsb3dpbmcgdHdvIGxpbmVzCgpvcy5lbnZpcm9uWyJUT0tFTklaRVJTX1BBUkFMTEVMSVNNIl0gPSAiZmFsc2UiCndhcm5pbmdzLmZpbHRlcndhcm5pbmdzKCJpZ25vcmUiKQoKbG9nZ2VyID0gbG9nZ2luZy5nZXRMb2dnZXIoInBpaS1yZWNvZ25pemVyIikKCgojIEFkZCB0aGUgY29uc3RhbnQgY2xhc3NlcyBvZiBNb2RlbHMgYW5kIEVudGl0aWVzIHRvIGdvdmVybiB0aGUgd2hvbGUgcGFja2FnZQpjbGFzcyBNb2RlbHM6CiAgICBXSE9MRSA9ICJ3aG9sZSIKICAgIFBBVFRFUk4gPSAicGF0dGVybiIKICAgIFNQQUNZID0gInNwYWN5IgogICAgRkxBSVIgPSAiZmxhaXIiCgoKY2xhc3MgRW50aXRpZXM6CiAgICBDUkVESVRfQ0FSRCA9ICJDUkVESVRfQ0FSRCIKICAgIFNTTiA9ICJTU04iCiAgICBQSE9ORSA9ICJQSE9ORSIKICAgIEVNQUlMID0gIkVNQUlMIgogICAgTE9DQVRJT04gPSAiTE9DQVRJT04iCiAgICBQRVJTT04gPSAiUEVSU09OIgogICAgTlJQID0gIk5SUCIKICAgIE9SR0FOSVpBVElPTiA9ICJPUkdBTklaQVRJT04iCiAgICBEQVRFX1RJTUUgPSAiREFURV9USU1FIgogICAgR1BFID0gKCJHUEUiLCkKICAgIE1BQ19BRERSRVNTID0gIk1BQ19BRERSRVNTIgogICAgVVNfQkFOS19OVU1CRVIgPSAiVVNfQkFOS19OVU1CRVIiCiAgICBJTUVJID0gIklNRUkiCiAgICBUSVRMRSA9ICJUSVRMRSIKICAgIExJQ0VOU0VfUExBVEUgPSAiTElDRU5TRV9QTEFURSIKICAgIFVTX1BBU1NQT1JUID0gIlVTX1BBU1NQT1JUIgogICAgQ1VSUkVOQ1kgPSAiQ1VSUkVOQ1kiCiAgICBST1VUSU5HX05VTUJFUiA9ICJST1VUSU5HX05VTUJFUiIKICAgIFVTX0lUSU4gPSAiVVNfSVRJTiIKICAgIFVTX0JBTktfTlVNQkVSID0gIlVTX0JBTktfTlVNQkVSIgogICAgVVNfRFJJVkVSX0xJQ0VOU0UgPSAiVVNfRFJJVkVSX0xJQ0VOU0UiCiAgICBBR0UgPSAiQUdFIgogICAgUEFTU1dPUkQgPSAiUEFTU1dPUkQiCiAgICBTV0lGVF9DT0RFID0gIlNXSUZUX0NPREUiCgoKY2xhc3MgUGF0dGVyblJlY29nbml6ZXJGYWN0b3J5OgogICAgIiIiCiAgICBGYWN0b3J5IGZvciBjcmVhdGluZyBwYXR0ZXJuIHJlY29nbml6ZXJzLCBpdCBjYW4gYmUgZXh0ZW5kZWQgaW4gdGhlIGZ1dHVyZSB0bwogICAgYWRkIG1vcmUgcmVnZXggcGF0dGVybiBmb3IgZGlmZmVyZW50IGVudGl0aWVzLiBGb3IgdGhlIHBhdHRlcm4gcmVjb2duaXplciB0byB3b3JrLAogICAgd2UgbmVlZCBjb25zdHJ1Y3QgYSBsaXN0IG9mIHJlZ2V4IHBhdHRlcm5zIGZvciBlYWNoIGVudGl0eS4KICAgICIiIgoKICAgIFJFQ09HTklaQUJMRV9FTlRJVElFUyA9IHsKICAgICAgICAiQ1JFRElUX0NBUkQiOiBbcGEuUGF0dGVybigiQ1JFRElUX0NBUkQiLCByIlxiKD86XGRbIC1dKj8pezEzLDE2fVxiIiwgMC41KV0sCiAgICAgICAgIlNTTiI6IFtwYS5QYXR0ZXJuKCJTU04iLCByIlxiXGR7M30tP1xkezJ9LT9cZHs0fVxiIiwgMC41KV0sCiAgICAgICAgIlBIT05FIjogW3BhLlBhdHRlcm4oIlBIT05FIiwgciJcKD9cZHszfVwpP1stLlxzXT9cZHszfVstLlxzXT9cZHs0fSIsIDAuNSldLAogICAgICAgICJFTUFJTCI6IFtwYS5QYXR0ZXJuKCJFTUFJTCIsIHIiXFMrQFxTKyIsIDAuNSldLAogICAgfQoKICAgICMgY3JlYXRlIGEgbGlzdCBvZiBwYXR0ZXJuIHJlY29nbml6ZXJzCiAgICBAY2xhc3NtZXRob2QKICAgIGRlZiBfY3JlYXRlX3BhdHRlcm5fcmVjb2duaXplcihjbHMpOgogICAgICAgICIiIgogICAgICAgIEZvciBlYWNoIGVudGl0eSwgY3JlYXRlIGEgbGlzdCBvZiBwYXR0ZXJucyB0byByZWNvZ25pemUgaXQKCiAgICAgICAgOnBhcmFtIGNsczogUGF0dGVyblJlY29nbml6ZXJGYWN0b3J5IGNsYXNzCgogICAgICAgIDpyZXR1cm5zOiBMaXN0IG9mIHBhdHRlcm4gcmVjb2duaXplcnMKICAgICAgICAiIiIKCiAgICAgICAgIyBFbnRpdGllcyB0byByZWNvZ25pemUgYW5kIHRoZWlyIHJlZ2V4IHBhdHRlcm5zCgogICAgICAgIHJldHVybiBbCiAgICAgICAgICAgIHBhLlBhdHRlcm5SZWNvZ25pemVyKHN1cHBvcnRlZF9lbnRpdHk9ZW50aXR5LCBwYXR0ZXJucz1wYXR0ZXJuKQogICAgICAgICAgICBmb3IgZW50aXR5LCBwYXR0ZXJuIGluIGNscy5SRUNPR05JWkFCTEVfRU5USVRJRVMuaXRlbXMoKQogICAgICAgIF0KCgpjbGFzcyBDdXN0b21TcGFjeVJlY29nbml6ZXIocGEuTG9jYWxSZWNvZ25pemVyKToKICAgICIiIgogICAgQ3VzdG9tIFNwYWN5IFJlY29nbml6ZXIgZnJvbSBQcmVzaWRpbyBBbmFseXplciB0cmFpbmVkIG9uIFByaXZ5IGRhdGEuCiAgICBUaGUgcHJpdnkgZGF0YSBpcyBnZW5lcmF0ZWQgdXNpbmcgdGhpcyBodHRwczovL2dpdGh1Yi5jb20vcGl4aWUtaW8vcGl4aWUvdHJlZS9tYWluL3NyYy9kYXRhZ2VuL3BpaS9wcml2eQogICAgSXQgY2FuIGJlIHVzZWQgdG8gcmVjb2duaXplIGN1c3RvbSBlbnRpdGllcywgU2luY2Ugd2Ugd2FudCB0byB1c2UgUHJlc2lkaW8ncyBSZWdpc3RyaWVzIHRvIGdlbmVyYXRlIEFuYWx5emVyRW5naW5lLAogICAgaXQgaW5oZXJpdHMgZnJvbSBQcmVzaWRpbyBBbmFseXplcidzIExvY2FsUmVjb2duaXplciBjbGFzcy4KICAgICIiIgoKICAgICMgRW50aXRpZXMgdG8gcmVjb2duaXplCgogICAgUkVDT0dOSVpBQkxFX0VOVElUSUVTID0gewogICAgICAgICJMT0NBVElPTiIsCiAgICAgICAgIlBFUlNPTiIsCiAgICAgICAgIk5SUCIsCiAgICAgICAgIk9SR0FOSVpBVElPTiIsCiAgICAgICAgIkRBVEVfVElNRSIsCiAgICB9CgogICAgIyBEZWZhdWx0IGV4cGxhbmF0aW9uIGZvciB0aGlzIHJlY29nbml6ZXIKCiAgICBfREVGQVVMVF9FWFBMQU5BVElPTiA9ICgKICAgICAgICAiSWRlbnRpZmllZCBhcyB7fSBieSBTcGFjeSdzIE5hbWVkIEVudGl0eSBSZWNvZ25pdGlvbiAoUHJpdnktdHJhaW5lZCkiCiAgICApCgogICAgIyBMYWJlbCBncm91cHMgdG8gY2hlY2sKCiAgICBfREVGQVVMVF9DSEVDS19MQUJFTF9HUk9VUFMgPSBbCiAgICAgICAgKHsiTE9DQVRJT04ifSwgeyJMT0MiLCAiTE9DQVRJT04iLCAiU1RSRUVUX0FERFJFU1MiLCAiQ09PUkRJTkFURSJ9KSwKICAgICAgICAoeyJQRVJTT04ifSwgeyJQRVIiLCAiUEVSU09OIn0pLAogICAgICAgICh7Ik5SUCJ9LCB7Ik5PUlAiLCAiTlJQIn0pLAogICAgICAgICh7Ik9SR0FOSVpBVElPTiJ9LCB7Ik9SRyJ9KSwKICAgICAgICAoeyJEQVRFX1RJTUUifSwgeyJEQVRFX1RJTUUifSksCiAgICBdCgogICAgIyBwcmV0cmFpbmVkIG1vZGVsIGZvciB0aGlzIHJlY29nbml6ZXIKCiAgICBfREVGQVVMVF9NT0RFTF9MQU5HVUFHRVMgPSB7CiAgICAgICAgImVuIjogImJla2kvZW5fc3BhY3lfcGlpX2Rpc3RpbGJlcnQiLAogICAgfQoKICAgIF9ERUZBVUxUX1BSRVNJRElPX0VRVUlWQUxFTkNFUyA9IHsKICAgICAgICAiUEVSIjogIlBFUlNPTiIsCiAgICAgICAgIkxPQyI6ICJMT0NBVElPTiIsCiAgICAgICAgIk9SRyI6ICJPUkdBTklaQVRJT04iLAogICAgICAgICJOUk9QIjogIk5SUCIsCiAgICAgICAgIkRBVEVfVElNRSI6ICJEQVRFX1RJTUUiLAogICAgfQoKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgIHN1cHBvcnRlZF9sYW5ndWFnZTogc3RyID0gImVuIiwKICAgICAgICBzdXBwb3J0ZWRfZW50aXRpZXM6IGxpc3Rbc3RyXSA9IE5vbmUsCiAgICAgICAgY2hlY2tfbGFiZWxfZ3JvdXBzOiB0dXBsZVtzZXQsIHNldF0gPSBOb25lLAogICAgICAgIGNvbnRleHQ6IGxpc3Rbc3RyXSA9IE5vbmUsCiAgICAgICAgbmVyX3N0cmVuZ3RoOiBmbG9hdCA9IDEsCiAgICApOgogICAgICAgICIiIgogICAgICAgIEluaXRpYWxpemUgU3BhY3kgUmVjb2duaXplci4KCiAgICAgICAgOnBhcmFtIHN1cHBvcnRlZF9sYW5ndWFnZTogTGFuZ3VhZ2UgdG8gdXNlLCBkZWZhdWx0IGlzIEVuZ2xpc2gKICAgICAgICA6cGFyYW0gc3VwcG9ydGVkX2VudGl0aWVzOiBFbnRpdGllcyB0byB1c2UgZm9yIHJlY29nbml0aW9uCiAgICAgICAgOnBhcmFtIGNoZWNrX2xhYmVsX2dyb3VwczogTGFiZWwgZ3JvdXBzIHRvIGNoZWNrIGZvciB0aGUgZW50aXRpZXMKICAgICAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICBDb250ZXh0IHRvIHVzZSBpZiBhbnkKICAgICAgICA6cGFyYW0gbmVyX3N0cmVuZ3RoOiAgICAgICBEZWZhdWx0IGNvbmZpZGVuY2UgZm9yIE5FUiBwcmVkaWN0aW9uCgogICAgICAgIDpyZXR1cm5zOiBTcGFjeVJlY29nbml6ZXIgb2JqZWN0CiAgICAgICAgIiIiCgogICAgICAgICMgRGVmYXVsdCBjb25maWRlbmNlIGZvciBORVIgcHJlZGljdGlvbgogICAgICAgIHNlbGYubmVyX3N0cmVuZ3RoID0gbmVyX3N0cmVuZ3RoCgogICAgICAgIHNlbGYuY2hlY2tfbGFiZWxfZ3JvdXBzID0gY2hlY2tfbGFiZWxfZ3JvdXBzIG9yIHNlbGYuX0RFRkFVTFRfQ0hFQ0tfTEFCRUxfR1JPVVBTCiAgICAgICAgc3VwcG9ydGVkX2VudGl0aWVzID0gc3VwcG9ydGVkX2VudGl0aWVzIG9yIHNlbGYuUkVDT0dOSVpBQkxFX0VOVElUSUVTCiAgICAgICAgc3VwZXIoKS5fX2luaXRfXygKICAgICAgICAgICAgc3VwcG9ydGVkX2VudGl0aWVzPXN1cHBvcnRlZF9lbnRpdGllcywKICAgICAgICAgICAgc3VwcG9ydGVkX2xhbmd1YWdlPXN1cHBvcnRlZF9sYW5ndWFnZSwKICAgICAgICApCgogICAgIyBnZXQgdGhlIHByZXNpZGlvIGV4cGxhbmF0aW9uIGZvciB0aGUgcmVzdWx0CgogICAgZGVmIF9idWlsZF9zcGFjeV9leHBsYW5hdGlvbigKICAgICAgICBzZWxmLCBvcmlnaW5hbF9zY29yZTogZmxvYXQsIGV4cGxhbmF0aW9uOiBzdHIKICAgICkgLT4gcGEuQW5hbHlzaXNFeHBsYW5hdGlvbjoKICAgICAgICAiIiIKICAgICAgICBDcmVhdGUgZXhwbGFuYXRpb24gZm9yIHdoeSB0aGlzIHJlc3VsdCB3YXMgZGV0ZWN0ZWQuCgogICAgICAgIDpwYXJhbSBvcmlnaW5hbF9zY29yZTogU2NvcmUgZ2l2ZW4gYnkgdGhpcyByZWNvZ25pemVyCiAgICAgICAgOnBhcmFtIGV4cGxhbmF0aW9uOiAgICBFeHBsYW5hdGlvbiBzdHJpbmcKCiAgICAgICAgOnJldHVybnM6IFByZXNpZGlvIEFuYWx5c2lzRXhwbGFuYXRpb24gb2JqZWN0CiAgICAgICAgIiIiCiAgICAgICAgZXhwbGFuYXRpb24gPSBwYS5BbmFseXNpc0V4cGxhbmF0aW9uKAogICAgICAgICAgICByZWNvZ25pemVyPXNlbGYuX19jbGFzc19fLl9fbmFtZV9fLAogICAgICAgICAgICBvcmlnaW5hbF9zY29yZT1vcmlnaW5hbF9zY29yZSwKICAgICAgICAgICAgdGV4dHVhbF9leHBsYW5hdGlvbj1leHBsYW5hdGlvbiwKICAgICAgICApCiAgICAgICAgcmV0dXJuIGV4cGxhbmF0aW9uCgogICAgIyBtYWluIG1ldGhvZCBmb3IgdGhlIHJlY29nbml6ZXIKICAgIGRlZiBhbmFseXplKHNlbGYsIHRleHQ6IHN0ciwgZW50aXRpZXM6IExpc3Rbc3RyXSwgbmxwX2FydGlmYWN0cz1Ob25lKTogICMgbm9xYSBEMTAyCiAgICAgICAgIiIiCiAgICAgICAgQW5hbHl6ZSB0ZXh0IHVzaW5nIFNwYWN5LgoKICAgICAgICA6cGFyYW0gdGV4dDogICAgICAgICAgVGV4dCB0byBhbmFseXplCiAgICAgICAgOnBhcmFtIGVudGl0aWVzOiAgICAgIEVudGl0aWVzIHRvIGFuYWx5emUKICAgICAgICA6cGFyYW0gbmxwX2FydGlmYWN0czogTkxQIGFydGlmYWN0cyB0byB1c2UKCiAgICAgICAgOnJldHVybnM6IExpc3Qgb2YgUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdCBvYmplY3RzCiAgICAgICAgIiIiCiAgICAgICAgcmVzdWx0cyA9IFtdCiAgICAgICAgaWYgbm90IG5scF9hcnRpZmFjdHM6CiAgICAgICAgICAgIGxvZ2dlci53YXJuaW5nKCJTa2lwcGluZyBTcGFDeSwgbmxwIGFydGlmYWN0cyBub3QgcHJvdmlkZWQuLi4iKQogICAgICAgICAgICByZXR1cm4gcmVzdWx0cwoKICAgICAgICBuZXJfZW50aXRpZXMgPSBubHBfYXJ0aWZhY3RzLmVudGl0aWVzCgogICAgICAgICMgcmVjb2duaXplIHRoZSBzdXBwb3J0ZWQgZW50aXRpZXMKICAgICAgICBmb3IgZW50aXR5IGluIGVudGl0aWVzOgogICAgICAgICAgICBpZiBlbnRpdHkgbm90IGluIHNlbGYuc3VwcG9ydGVkX2VudGl0aWVzOgogICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgZm9yIGVudCBpbiBuZXJfZW50aXRpZXM6CiAgICAgICAgICAgICAgICBpZiBub3Qgc2VsZi5fX2NoZWNrX2xhYmVsKGVudGl0eSwgZW50LmxhYmVsXywgc2VsZi5jaGVja19sYWJlbF9ncm91cHMpOgogICAgICAgICAgICAgICAgICAgIGNvbnRpbnVlCgogICAgICAgICAgICAgICAgIyBzdHJpbmcgb2YgdGhlIGV4cGxhbmF0aW9uIHNheWluZyB0aGUgZW50aXR5IGlzIHJlY29nbml6ZWQgYnkgc3BhY3kKICAgICAgICAgICAgICAgIHRleHR1YWxfZXhwbGFuYXRpb24gPSBzZWxmLl9ERUZBVUxUX0VYUExBTkFUSU9OLmZvcm1hdChlbnQubGFiZWxfKQogICAgICAgICAgICAgICAgZXhwbGFuYXRpb24gPSBzZWxmLl9idWlsZF9zcGFjeV9leHBsYW5hdGlvbigKICAgICAgICAgICAgICAgICAgICBzZWxmLm5lcl9zdHJlbmd0aCwgdGV4dHVhbF9leHBsYW5hdGlvbgogICAgICAgICAgICAgICAgKQoKICAgICAgICAgICAgICAgICMgY3JlYXRlIHRoZSBzdGFuZGFyZCByZXN1bHQgd2l0aCB0aGUgZW50aXR5LCBzdGFydCwgZW5kLCBzY29yZSwgYW5kIGV4cGxhbmF0aW9uCiAgICAgICAgICAgICAgICBzcGFjeV9yZXN1bHQgPSBwYS5SZWNvZ25pemVyUmVzdWx0KAogICAgICAgICAgICAgICAgICAgIGVudGl0eV90eXBlPWVudGl0eSwKICAgICAgICAgICAgICAgICAgICBzdGFydD1lbnQuc3RhcnRfY2hhciwKICAgICAgICAgICAgICAgICAgICBlbmQ9ZW50LmVuZF9jaGFyLAogICAgICAgICAgICAgICAgICAgIHNjb3JlPXNlbGYubmVyX3N0cmVuZ3RoLAogICAgICAgICAgICAgICAgICAgIGFuYWx5c2lzX2V4cGxhbmF0aW9uPWV4cGxhbmF0aW9uLAogICAgICAgICAgICAgICAgICAgIHJlY29nbml0aW9uX21ldGFkYXRhPXsKICAgICAgICAgICAgICAgICAgICAgICAgcGEuUmVjb2duaXplclJlc3VsdC5SRUNPR05JWkVSX05BTUVfS0VZOiBzZWxmLm5hbWUKICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgcmVzdWx0cy5hcHBlbmQoc3BhY3lfcmVzdWx0KQoKICAgICAgICByZXR1cm4gcmVzdWx0cwoKICAgIEBzdGF0aWNtZXRob2QKICAgIGRlZiBfX2NoZWNrX2xhYmVsKAogICAgICAgIGVudGl0eTogc3RyLCBsYWJlbDogc3RyLCBjaGVja19sYWJlbF9ncm91cHM6IHR1cGxlW3NldCwgc2V0XQogICAgKSAtPiBib29sOgogICAgICAgICIiIgogICAgICAgIENoZWNrIGlmIHRoZSBsYWJlbCBpcyBpbiB0aGUgbGFiZWwgZ3JvdXAuCgogICAgICAgIDpwYXJhbSBlbnRpdHk6ICAgICAgICAgICAgIEVudGl0eSB0byBjaGVjawogICAgICAgIDpwYXJhbSBsYWJlbDogICAgICAgICAgICAgIExhYmVsIHRvIGNoZWNrCiAgICAgICAgOnBhcmFtIGNoZWNrX2xhYmVsX2dyb3VwczogTGFiZWwgZ3JvdXBzIHRvIGNoZWNrCgogICAgICAgIDpyZXR1cm5zOiBUcnVlIGlmIHRoZSBsYWJlbCBpcyBpbiB0aGUgbGFiZWwgZ3JvdXAsIEZhbHNlIG90aGVyd2lzZQogICAgICAgICIiIgogICAgICAgIHJldHVybiBhbnkoCiAgICAgICAgICAgIGVudGl0eSBpbiBlZ3JwIGFuZCBsYWJlbCBpbiBsZ3JwIGZvciBlZ3JwLCBsZ3JwIGluIGNoZWNrX2xhYmVsX2dyb3VwcwogICAgICAgICkKCgojIENsYXNzIHRvIHVzZSBGbGFpciB3aXRoIFByZXNpZGlvIGFzIGFuIGV4dGVybmFsIHJlY29nbml6ZXIuCmNsYXNzIEZsYWlyUmVjb2duaXplcihwYS5FbnRpdHlSZWNvZ25pemVyKToKICAgICIiIgogICAgV3JhcHBlciBmb3IgYSBmbGFpciBtb2RlbCwgaWYgbmVlZGVkIHRvIGJlIHVzZWQgd2l0aGluIFByZXNpZGlvIEFuYWx5emVyLgogICAgVGhpcyBpcyB0byBtYWtlIHN1cmUgdGhlIHJlY29nbml6ZXIgY2FuIGJlIHJlZ2lzdGVyZWQgd2l0aCBQcmVzaWRpbyByZWdpc3RyeS4KICAgICIiIgoKICAgIFJFQ09HTklaQUJMRV9FTlRJVElFUyA9IHsKICAgICAgICAiTE9DQVRJT04iLAogICAgICAgICJQRVJTT04iLAogICAgICAgICJOUlAiLAogICAgICAgICJHUEUiLAogICAgICAgICJPUkdBTklaQVRJT04iLAogICAgICAgICJNQUNfQUREUkVTUyIsCiAgICAgICAgIlVTX0JBTktfTlVNQkVSIiwKICAgICAgICAiSU1FSSIsCiAgICAgICAgIlRJVExFIiwKICAgICAgICAiTElDRU5TRV9QTEFURSIsCiAgICAgICAgIlVTX1BBU1NQT1JUIiwKICAgICAgICAiQ1VSUkVOQ1kiLAogICAgICAgICJST1VUSU5HX05VTUJFUiIsCiAgICAgICAgIlVTX0lUSU4iLAogICAgICAgICJVU19CQU5LX05VTUJFUiIsCiAgICAgICAgIlVTX0RSSVZFUl9MSUNFTlNFIiwKICAgICAgICAiQUdFIiwKICAgICAgICAiUEFTU1dPUkQiLAogICAgICAgICJTV0lGVF9DT0RFIiwKICAgIH0KCiAgICAjIFRoaXMgaXMgdXNlZCB0byBjb25zdHJ1Y3QgdGhlIGV4cGxhbmF0aW9uIGZvciB0aGUgcmVzdWx0CgogICAgX0RFRkFVTFRfRVhQTEFOQVRJT04gPSAiSWRlbnRpZmllZCBhcyB7fSBieSBGbGFpcidzIE5hbWVkIEVudGl0eSBSZWNvZ25pdGlvbiIKCiAgICBfREVGQVVMVF9DSEVDS19MQUJFTF9HUk9VUFMgPSBbCiAgICAgICAgKHsiTE9DQVRJT04ifSwgeyJMT0MiLCAiTE9DQVRJT04iLCAiU1RSRUVUX0FERFJFU1MiLCAiQ09PUkRJTkFURSJ9KSwKICAgICAgICAoeyJQRVJTT04ifSwgeyJQRVIiLCAiUEVSU09OIn0pLAogICAgICAgICh7Ik5SUCJ9LCB7Ik5PUlAiLCAiTlJQIn0pLAogICAgICAgICh7IkdQRSJ9LCB7IkdQRSJ9KSwKICAgICAgICAoeyJPUkdBTklaQVRJT04ifSwgeyJPUkcifSksCiAgICAgICAgKHsiTUFDX0FERFJFU1MifSwgeyJNQUNfQUREUkVTUyJ9KSwKICAgICAgICAoeyJVU19CQU5LX05VTUJFUiJ9LCB7IlVTX0JBTktfTlVNQkVSIn0pLAogICAgICAgICh7IklNRUkifSwgeyJJTUVJIn0pLAogICAgICAgICh7IlRJVExFIn0sIHsiVElUTEUifSksCiAgICAgICAgKHsiTElDRU5TRV9QTEFURSJ9LCB7IkxJQ0VOU0VfUExBVEUifSksCiAgICAgICAgKHsiVVNfUEFTU1BPUlQifSwgeyJVU19QQVNTUE9SVCJ9KSwKICAgICAgICAoeyJDVVJSRU5DWSJ9LCB7IkNVUlJFTkNZIn0pLAogICAgICAgICh7IlJPVVRJTkdfTlVNQkVSIn0sIHsiUk9VVElOR19OVU1CRVIifSksCiAgICAgICAgKHsiQUdFIn0sIHsiQUdFIn0pLAogICAgICAgICh7IkNVUlJFTkNZIn0sIHsiQ1VSUkVOQ1kifSksCiAgICAgICAgKHsiU1dJRlRfQ09ERSJ9LCB7IlNXSUZUX0NPREUifSksCiAgICAgICAgKHsiVVNfSVRJTiJ9LCB7IlVTX0lUSU4ifSksCiAgICAgICAgKHsiVVNfQkFOS19OVU1CRVIifSwgeyJVU19CQU5LX05VTUJFUiJ9KSwKICAgICAgICAoeyJVU19EUklWRVJfTElDRU5TRSJ9LCB7IlVTX0RSSVZFUl9MSUNFTlNFIn0pLAogICAgXQoKICAgIF9ERUZBVUxUX01PREVMX0xBTkdVQUdFUyA9IHsKICAgICAgICAiZW4iOiAiYmVraS9mbGFpci1waWktZGlzdGlsYmVydCIsCiAgICB9CgogICAgX0RFRkFVTFRfUFJFU0lESU9fRVFVSVZBTEVOQ0VTID0gewogICAgICAgICJQRVIiOiAiUEVSU09OIiwKICAgICAgICAiTE9DIjogIkxPQ0FUSU9OIiwKICAgICAgICAiT1JHIjogIk9SR0FOSVpBVElPTiIsCiAgICAgICAgIk5ST1AiOiAiTlJQIiwKICAgICAgICAiVVJMIjogIlVSTCIsCiAgICAgICAgIlVTX0lUSU4iOiAiVVNfSVRJTiIsCiAgICAgICAgIlVTX1BBU1NQT1JUIjogIlVTX1BBU1NQT1JUIiwKICAgICAgICAiSUJBTl9DT0RFIjogIklCQU5fQ09ERSIsCiAgICAgICAgIklQX0FERFJFU1MiOiAiSVBfQUREUkVTUyIsCiAgICAgICAgIkVNQUlMX0FERFJFU1MiOiAiRU1BSUwiLAogICAgICAgICJVU19EUklWRVJfTElDRU5TRSI6ICJVU19EUklWRVJfTElDRU5TRSIsCiAgICAgICAgIlVTX0JBTktfTlVNQkVSIjogIlVTX0JBTktfTlVNQkVSIiwKICAgIH0KCiAgICBkZWYgX19pbml0X18oCiAgICAgICAgc2VsZiwKICAgICAgICBzdXBwb3J0ZWRfbGFuZ3VhZ2U6IHN0ciA9ICJlbiIsCiAgICAgICAgc3VwcG9ydGVkX2VudGl0aWVzOiBsaXN0W3N0cl0gPSBOb25lLAogICAgICAgIGNoZWNrX2xhYmVsX2dyb3VwczogdHVwbGVbc2V0LCBzZXRdID0gTm9uZSwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgSW5pdGlhbGl6ZSB0aGUgRmxhaXJSZWNvZ25pemVyLgoKICAgICAgICA6cGFyYW0gc3VwcG9ydGVkX2xhbmd1YWdlOiBMYW5ndWFnZSB0byB1c2UKICAgICAgICA6cGFyYW0gc3VwcG9ydGVkX2VudGl0aWVzOiBFbnRpdGllcyB0byB1c2UKICAgICAgICA6cGFyYW0gY2hlY2tfbGFiZWxfZ3JvdXBzOiBMYWJlbCBncm91cHMgdG8gY2hlY2sKCiAgICAgICAgOnJldHVybnM6IEZsYWlyUmVjb2duaXplciBvYmplY3QKCiAgICAgICAgIiIiCiAgICAgICAgc2VsZi5jaGVja19sYWJlbF9ncm91cHMgPSBjaGVja19sYWJlbF9ncm91cHMgb3Igc2VsZi5fREVGQVVMVF9DSEVDS19MQUJFTF9HUk9VUFMKCiAgICAgICAgc3VwcG9ydGVkX2VudGl0aWVzID0gc3VwcG9ydGVkX2VudGl0aWVzIG9yIHNlbGYuUkVDT0dOSVpBQkxFX0VOVElUSUVTCiAgICAgICAgc2VsZi5tb2RlbCA9IGZsLm1vZGVscy5TZXF1ZW5jZVRhZ2dlci5sb2FkKAogICAgICAgICAgICBzZWxmLl9ERUZBVUxUX01PREVMX0xBTkdVQUdFUy5nZXQoc3VwcG9ydGVkX2xhbmd1YWdlKQogICAgICAgICkKCiAgICAgICAgc3VwZXIoKS5fX2luaXRfXygKICAgICAgICAgICAgc3VwcG9ydGVkX2VudGl0aWVzPXN1cHBvcnRlZF9lbnRpdGllcywKICAgICAgICAgICAgc3VwcG9ydGVkX2xhbmd1YWdlPXN1cHBvcnRlZF9sYW5ndWFnZSwKICAgICAgICAgICAgbmFtZT0iRmxhaXIgQW5hbHl0aWNzIiwKICAgICAgICApCgogICAgIyBtYWluIG1ldGhvZCBmb3IgdGhlIHJlY29nbml6ZXIKICAgIGRlZiBhbmFseXplKAogICAgICAgIHNlbGYsCiAgICAgICAgdGV4dDogc3RyLAogICAgICAgIGVudGl0aWVzOiBsaXN0W3N0cl0sCiAgICAgICAgbmxwX2FydGlmYWN0czogcGEubmxwX2VuZ2luZS5ObHBBcnRpZmFjdHMgPSBOb25lLAogICAgKSAtPiBsaXN0W3BhLlJlY29nbml6ZXJSZXN1bHRdOgogICAgICAgICIiIgogICAgICAgIEFuYWx5emUgdGV4dCBhbmQgcmV0dXJuIHRoZSByZXN1bHRzLgoKICAgICAgICA6cGFyYW0gdGV4dDogICAgICAgICAgVGhlIHRleHQgZm9yIGFuYWx5c2lzLgogICAgICAgIDpwYXJhbSBlbnRpdGllczogICAgICBUaGUgbGlzdCBvZiBlbnRpdGllcyB0byByZWNvZ25pemUuCiAgICAgICAgOnBhcmFtIG5scF9hcnRpZmFjdHM6IE5vdCB1c2VkIGJ5IHRoaXMgcmVjb2duaXplciBidXQgbmVlZGVkIGZvciB0aGUgaW50ZXJmYWNlLgoKICAgICAgICA6cmV0dXJuczogVGhlIGxpc3Qgb2YgUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdCBjb25zdHJ1Y3RlZCBmcm9tIHRoZSByZWNvZ25pemVkIEZsYWlyIGRldGVjdGlvbnMuCiAgICAgICAgIiIiCgogICAgICAgIHJlc3VsdHMgPSBbXQoKICAgICAgICBzZW50ZW5jZXMgPSBmbC5kYXRhLlNlbnRlbmNlKHRleHQpCiAgICAgICAgc2VsZi5tb2RlbC5wcmVkaWN0KHNlbnRlbmNlcykKCiAgICAgICAgIyBJZiB0aGVyZSBhcmUgbm8gc3BlY2lmaWMgbGlzdCBvZiBlbnRpdGllcywgd2Ugd2lsbCBsb29rIGZvciBhbGwgb2YgaXQuCiAgICAgICAgaWYgbm90IGVudGl0aWVzOgogICAgICAgICAgICBlbnRpdGllcyA9IHNlbGYuc3VwcG9ydGVkX2VudGl0aWVzCgogICAgICAgICMgR28gb3ZlciB0aGUgZW50aXRpZXMgYW5kIGNoZWNrIGlmIHRoZXkgYXJlIGluIHRoZSBzdXBwb3J0ZWQgZW50aXRpZXMgbGlzdC4KICAgICAgICBmb3IgZW50aXR5IGluIGVudGl0aWVzOgogICAgICAgICAgICBpZiBlbnRpdHkgbm90IGluIHNlbGYuc3VwcG9ydGVkX2VudGl0aWVzOgogICAgICAgICAgICAgICAgY29udGludWUKCiAgICAgICAgICAgICMgR28gb3ZlciB0aGUgc2VudGVuY2VzIGFuZCBjaGVjayBpZiB0aGUgZW50aXR5IGlzIGluIHRoZSBzZW50ZW5jZS4KICAgICAgICAgICAgZm9yIGVudCBpbiBzZW50ZW5jZXMuZ2V0X3NwYW5zKCJuZXIiKToKICAgICAgICAgICAgICAgIGlmIG5vdCBzZWxmLl9fY2hlY2tfbGFiZWwoCiAgICAgICAgICAgICAgICAgICAgZW50aXR5LCBlbnQubGFiZWxzWzBdLnZhbHVlLCBzZWxmLmNoZWNrX2xhYmVsX2dyb3VwcwogICAgICAgICAgICAgICAgKToKICAgICAgICAgICAgICAgICAgICBjb250aW51ZQoKICAgICAgICAgICAgICAgICMgSWYgdGhlIGVudGl0eSBpcyBpbiB0aGUgc2VudGVuY2UsIHdlIHdpbGwgYWRkIGl0IHRvIHRoZSByZXN1bHRzLgogICAgICAgICAgICAgICAgdGV4dHVhbF9leHBsYW5hdGlvbiA9IHNlbGYuX0RFRkFVTFRfRVhQTEFOQVRJT04uZm9ybWF0KAogICAgICAgICAgICAgICAgICAgIGVudC5sYWJlbHNbMF0udmFsdWUKICAgICAgICAgICAgICAgICkKCiAgICAgICAgICAgICAgICAjIEJ1aWxkIHRoZSBleHBsYW5hdGlvbiBmb3IgdGhlIHJlc3VsdAogICAgICAgICAgICAgICAgZXhwbGFuYXRpb24gPSBzZWxmLl9idWlsZF9mbGFpcl9leHBsYW5hdGlvbigKICAgICAgICAgICAgICAgICAgICByb3VuZChlbnQuc2NvcmUsIDIpLCB0ZXh0dWFsX2V4cGxhbmF0aW9uCiAgICAgICAgICAgICAgICApCgogICAgICAgICAgICAgICAgZmxhaXJfcmVzdWx0ID0gc2VsZi5fY29udmVydF90b19yZWNvZ25pemVyX3Jlc3VsdChlbnQsIGV4cGxhbmF0aW9uKQoKICAgICAgICAgICAgICAgIHJlc3VsdHMuYXBwZW5kKGZsYWlyX3Jlc3VsdCkKCiAgICAgICAgcmV0dXJuIHJlc3VsdHMKCiAgICBkZWYgX2NvbnZlcnRfdG9fcmVjb2duaXplcl9yZXN1bHQoCiAgICAgICAgc2VsZiwgZW50aXR5OiBmbC5kYXRhLlNwYW4sIGV4cGxhbmF0aW9uOiBzdHIKICAgICkgLT4gcGEuUmVjb2duaXplclJlc3VsdDoKICAgICAgICAiIiIKICAgICAgICBDb252ZXJ0IEZsYWlyIHJlc3VsdCB0byBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0LgoKICAgICAgICA6cGFyYW0gZW50aXR5OiAgICAgIEZsYWlyIGVudGl0eSBvZiBTcGFuCiAgICAgICAgOnBhcmFtIGV4cGxhbmF0aW9uOiBQcmVzaWRpbyBBbmFseXNpc0V4cGxhbmF0aW9uCgogICAgICAgIDpyZXR1cm5zOiBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0CiAgICAgICAgIiIiCgogICAgICAgICMgQ29udmVydCB0aGUgZW50aXR5IHR5cGUgdG8gUHJlc2lkaW8gZW50aXR5IHR5cGUKICAgICAgICBlbnRpdHlfdHlwZSA9IHNlbGYuX0RFRkFVTFRfUFJFU0lESU9fRVFVSVZBTEVOQ0VTLmdldChlbnRpdHkudGFnLCBlbnRpdHkudGFnKQoKICAgICAgICAjIENvbnZlcnQgdGhlIHNjb3JlIHRvIFByZXNpZGlvIHNjb3JlCiAgICAgICAgZmxhaXJfc2NvcmUgPSByb3VuZChlbnRpdHkuc2NvcmUsIDIpCgogICAgICAgICMgQ3JlYXRlIHRoZSBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0IGZyb20gdGhlIEZsYWlyIGVudGl0eQogICAgICAgIGZsYWlyX3Jlc3VsdHMgPSBwYS5SZWNvZ25pemVyUmVzdWx0KAogICAgICAgICAgICBlbnRpdHlfdHlwZT1lbnRpdHlfdHlwZSwKICAgICAgICAgICAgc3RhcnQ9ZW50aXR5LnN0YXJ0X3Bvc2l0aW9uLAogICAgICAgICAgICBlbmQ9ZW50aXR5LmVuZF9wb3NpdGlvbiwKICAgICAgICAgICAgc2NvcmU9ZmxhaXJfc2NvcmUsCiAgICAgICAgICAgIGFuYWx5c2lzX2V4cGxhbmF0aW9uPWV4cGxhbmF0aW9uLAogICAgICAgICkKCiAgICAgICAgcmV0dXJuIGZsYWlyX3Jlc3VsdHMKCiAgICBkZWYgX2J1aWxkX2ZsYWlyX2V4cGxhbmF0aW9uKAogICAgICAgIHNlbGYsIG9yaWdpbmFsX3Njb3JlOiBmbG9hdCwgZXhwbGFuYXRpb246IHN0cgogICAgKSAtPiBwYS5BbmFseXNpc0V4cGxhbmF0aW9uOgogICAgICAgICIiIgogICAgICAgIENyZWF0ZSBleHBsYW5hdGlvbiBmb3Igd2h5IHRoaXMgcmVzdWx0IHdhcyBkZXRlY3RlZC4KCiAgICAgICAgOnBhcmFtIG9yaWdpbmFsX3Njb3JlOiBTY29yZSBnaXZlbiBieSB0aGlzIHJlY29nbml6ZXIKICAgICAgICA6cGFyYW0gZXhwbGFuYXRpb246ICAgIEV4cGxhbmF0aW9uIHN0cmluZwoKICAgICAgICA6cmV0dXJuczogUHJlc2lkaW8gQW5hbHlzaXNFeHBsYW5hdGlvbgogICAgICAgICIiIgoKICAgICAgICAjIENyZWF0ZSB0aGUgUHJlc2lkaW8gQW5hbHlzaXNFeHBsYW5hdGlvbiBmb3IgdGhlIHJlc3VsdAogICAgICAgIGV4cGxhbmF0aW9uID0gcGEuQW5hbHlzaXNFeHBsYW5hdGlvbigKICAgICAgICAgICAgcmVjb2duaXplcj1zZWxmLl9fY2xhc3NfXy5fX25hbWVfXywKICAgICAgICAgICAgb3JpZ2luYWxfc2NvcmU9b3JpZ2luYWxfc2NvcmUsCiAgICAgICAgICAgIHRleHR1YWxfZXhwbGFuYXRpb249ZXhwbGFuYXRpb24sCiAgICAgICAgKQogICAgICAgIHJldHVybiBleHBsYW5hdGlvbgoKICAgICMgc2FuaXR5IGNoZWNrIG9mIHRoZSBlbnRpdHkgYW5kIGxhYmVsIGJlZm9yZSByZWNvZ25pdGlvbgogICAgQHN0YXRpY21ldGhvZAogICAgZGVmIF9fY2hlY2tfbGFiZWwoCiAgICAgICAgZW50aXR5OiBzdHIsIGxhYmVsOiBzdHIsIGNoZWNrX2xhYmVsX2dyb3VwczogdHVwbGVbc2V0LCBzZXRdCiAgICApIC0+IGJvb2w6CiAgICAgICAgcmV0dXJuIGFueSgKICAgICAgICAgICAgZW50aXR5IGluIGVncnAgYW5kIGxhYmVsIGluIGxncnAgZm9yIGVncnAsIGxncnAgaW4gY2hlY2tfbGFiZWxfZ3JvdXBzCiAgICAgICAgKQoKCiMgZ2V0IHRoZSBhbmFseXplciBlbmdpbmUgYmFzZWQgb24gdGhlIG1vZGVsCmRlZiBfZ2V0X2FuYWx5emVyX2VuZ2luZSgKICAgIG1vZGVsOiBzdHIgPSBOb25lLCBlbnRpdGllczogbGlzdFtzdHJdID0gTm9uZQopIC0+IHBhLkFuYWx5emVyRW5naW5lOgogICAgIiIiCiAgICBSZXR1cm4gcGEuQW5hbHl6ZXJFbmdpbmUuCgogICAgOnBhcmFtIG1vZGVsOiBUaGUgbW9kZWwgdG8gdXNlLiBDYW4gYmUgInNwYWN5IiwgImZsYWlyIiwgInBhdHRlcm4iIG9yICJ3aG9sZSIuCiAgICA6cGFyYW0gZW50aXRpZXM6IFRoZSBsaXN0IG9mIGVudGl0aWVzIHRvIHVzZS4KCiAgICA6cmV0dXJuczogcGEuQW5hbHl6ZXJFbmdpbmUKICAgICIiIgogICAgIyByZWNvZ25pemVyIHJlZ2lzdHJ5IHRoYXQgY2FuIHN0b3JlIG11bHRpcGxlIHJlY29nbml6ZXJzCiAgICByZWdpc3RyeSA9IHBhLlJlY29nbml6ZXJSZWdpc3RyeSgpCiAgICBpZiBtb2RlbCA9PSBNb2RlbHMuU1BBQ1k6CiAgICAgICAgIyBjdXN0b20gc3BhY3kgcmVjb2duaXplcgogICAgICAgIHNwYWN5X3JlY29nbml6ZXIgPSBDdXN0b21TcGFjeVJlY29nbml6ZXIoKQogICAgICAgICMgYWRkIHRoZSBjdXN0b20gYnVpbGQgc3BhY3kgcmVjb2duaXplcgogICAgICAgIHJlZ2lzdHJ5LmFkZF9yZWNvZ25pemVyKHNwYWN5X3JlY29nbml6ZXIpCiAgICBlbGlmIG1vZGVsID09IE1vZGVscy5GTEFJUjoKICAgICAgICAjIHByZS10cmFpbmVkIGZsYWlyIHJlY29nbml6ZXIKICAgICAgICBmbGFpcl9yZWNvZ25pemVyID0gRmxhaXJSZWNvZ25pemVyKCkKICAgICAgICAjIGFkZCB0aGUgY3VzdG9tIGJ1aWxkIGZsYWlyIHJlY29nbml6ZXIKICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihmbGFpcl9yZWNvZ25pemVyKQogICAgZWxpZiBtb2RlbCA9PSBNb2RlbHMuUEFUVEVSTjoKICAgICAgICAjIGFkZCB0aGUgcGF0dGVybiByZWNvZ25pemVyCiAgICAgICAgcGF0dGVybl9yZWNvZ25pemVyX2ZhY3RvcnkgPSBQYXR0ZXJuUmVjb2duaXplckZhY3RvcnkoKQogICAgICAgIGZvciByZWNvZ25pemVyIGluIHBhdHRlcm5fcmVjb2duaXplcl9mYWN0b3J5Ll9jcmVhdGVfcGF0dGVybl9yZWNvZ25pemVyKCk6CiAgICAgICAgICAgIHJlZ2lzdHJ5LmFkZF9yZWNvZ25pemVyKHJlY29nbml6ZXIpCiAgICBlbGlmIG1vZGVsID09IE1vZGVscy5XSE9MRToKICAgICAgICBzcGFjeV9yZWNvZ25pemVyID0gQ3VzdG9tU3BhY3lSZWNvZ25pemVyKCkKICAgICAgICBmbGFpcl9yZWNvZ25pemVyID0gRmxhaXJSZWNvZ25pemVyKCkKICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihzcGFjeV9yZWNvZ25pemVyKQogICAgICAgIHJlZ2lzdHJ5LmFkZF9yZWNvZ25pemVyKGZsYWlyX3JlY29nbml6ZXIpCiAgICAgICAgIyBhZGQgdGhlIHBhdHRlcm4gcmVjb2duaXplcgogICAgICAgIHBhdHRlcm5fcmVjb2duaXplcl9mYWN0b3J5ID0gUGF0dGVyblJlY29nbml6ZXJGYWN0b3J5KCkKICAgICAgICBmb3IgcmVjb2duaXplciBpbiBwYXR0ZXJuX3JlY29nbml6ZXJfZmFjdG9yeS5fY3JlYXRlX3BhdHRlcm5fcmVjb2duaXplcigpOgogICAgICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihyZWNvZ25pemVyKQogICAgZWxpZiBub3QgbW9kZWwgYW5kIGVudGl0aWVzOgogICAgICAgIGlmIHNldChlbnRpdGllcykgJiBDdXN0b21TcGFjeVJlY29nbml6ZXIuUkVDT0dOSVpBQkxFX0VOVElUSUVTOgogICAgICAgICAgICBzcGFjeV9yZWNvZ25pemVyID0gQ3VzdG9tU3BhY3lSZWNvZ25pemVyKCkKICAgICAgICAgICAgcmVnaXN0cnkuYWRkX3JlY29nbml6ZXIoc3BhY3lfcmVjb2duaXplcikKICAgICAgICBpZiBzZXQoZW50aXRpZXMpICYgRmxhaXJSZWNvZ25pemVyLlJFQ09HTklaQUJMRV9FTlRJVElFUzoKICAgICAgICAgICAgZmxhaXJfcmVjb2duaXplciA9IEZsYWlyUmVjb2duaXplcigpCiAgICAgICAgICAgIHJlZ2lzdHJ5LmFkZF9yZWNvZ25pemVyKGZsYWlyX3JlY29nbml6ZXIpCiAgICAgICAgIyBhZGQgdGhlIHBhdHRlcm4gcmVjb2duaXplcgogICAgICAgIGlmIHNldChlbnRpdGllcykgJiAoc2V0KFBhdHRlcm5SZWNvZ25pemVyRmFjdG9yeS5SRUNPR05JWkFCTEVfRU5USVRJRVMua2V5cygpKSk6CiAgICAgICAgICAgIHBhdHRlcm5fcmVjb2duaXplcl9mYWN0b3J5ID0gUGF0dGVyblJlY29nbml6ZXJGYWN0b3J5KCkKICAgICAgICAgICAgZm9yIHJlY29nbml6ZXIgaW4gcGF0dGVybl9yZWNvZ25pemVyX2ZhY3RvcnkuX2NyZWF0ZV9wYXR0ZXJuX3JlY29nbml6ZXIoKToKICAgICAgICAgICAgICAgIHJlZ2lzdHJ5LmFkZF9yZWNvZ25pemVyKHJlY29nbml6ZXIpCiAgICBlbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICJhcmd1bWVudCBvZiBtb2RlbCBhbmQgZW50aXRpZXMgY2FuIG5vdCBiZSBOb25lIGF0IHRoZSBzYW1lIHRpbWUiCiAgICAgICAgKQogICAgYW5hbHl6ZXIgPSBwYS5BbmFseXplckVuZ2luZSgKICAgICAgICByZWdpc3RyeT1yZWdpc3RyeSwKICAgICAgICBzdXBwb3J0ZWRfbGFuZ3VhZ2VzPVsiZW4iXSwKICAgICkKCiAgICBzdXBwb3J0ZWRfZW50aXRpZXMgPSBhbmFseXplci5nZXRfc3VwcG9ydGVkX2VudGl0aWVzKCkKCiAgICBpZiBlbnRpdGllcyBhbmQgbm90IGFsbChpdGVtIGluIHN1cHBvcnRlZF9lbnRpdGllcyBmb3IgaXRlbSBpbiBlbnRpdGllcyk6CiAgICAgICAgbm90X3N1cHBvcnRlZF9lbnRpdGllcyA9IFsKICAgICAgICAgICAgaXRlbSBmb3IgaXRlbSBpbiBlbnRpdGllcyBpZiBpdGVtIG5vdCBpbiBzdXBwb3J0ZWRfZW50aXRpZXMKICAgICAgICBdCiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJUaGUgY3VycmVudCBtb2RlbCB7bW9kZWx9IGRvZXNuJ3Qgc3VwcG9ydCB0aGUgZm9sbG93aW5nIGVudGl0aWVzOiB7bm90X3N1cHBvcnRlZF9lbnRpdGllc30uICIKICAgICAgICAgICAgZiJTdXBwb3J0ZWQgZW50aXRpZXMgYXJlOiB7c3VwcG9ydGVkX2VudGl0aWVzfSIKICAgICAgICApCiAgICByZXR1cm4gYW5hbHl6ZXIKCgpkZWYgX2dldF9hbm9ueW1pemVyX2VuZ2luZSgpIC0+IHByZV9hbm95bWl6ZXIuQW5vbnltaXplckVuZ2luZToKICAgICIiIgogICAgUmV0dXJuIEFub255bWl6ZXJFbmdpbmUuCgogICAgOnJldHVybnM6IFRoZSBBbm9ueW1pemVyRW5naW5lLgogICAgIiIiCiAgICByZXR1cm4gcHJlX2Fub3ltaXplci5Bbm9ueW1pemVyRW5naW5lKCkKCgpkZWYgX2Fub255bWl6ZSgKICAgIHRleHQ6IHN0ciwKICAgIGFuYWx5emVfcmVzdWx0czogbGlzdFtwYS5SZWNvZ25pemVyUmVzdWx0XSwKICAgIGVudGl0eV9vcGVyYXRvcl9tYXA6IGRpY3QgPSBOb25lLAogICAgaXNfZnVsbF90ZXh0OiBib29sID0gVHJ1ZSwKKSAtPiBzdHI6CiAgICAiIiIKICAgIEFub255bWl6ZSBpZGVudGlmaWVkIGlucHV0IHVzaW5nIFByZXNpZGlvIEFib255bWl6ZXIuCgogICAgOnBhcmFtIHRleHQ6ICAgICAgICAgICAgICAgIFRoZSB0ZXh0IGZvciBhbmFseXNpcy4KICAgIDpwYXJhbSBhbmFseXplX3Jlc3VsdHM6ICAgICBUaGUgbGlzdCBvZiBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0IGNvbnN0cnVjdGVkIGZyb20KICAgIDpwYXJhbSBlbnRpdHlfb3BlcmF0b3JfbWFwOiBUaGUgZW50aXR5X29wZXJhdG9yX21hcCBpcyBhIGRpY3Rpb25hcnkgdGhhdCBtYXBzIGVudGl0eSB0byBvcGVyYXRvciBuYW1lIGFuZCBvcGVyYXRvciBwYXJhbXMuCiAgICA6cGFyYW0gaXNfZnVsbF90ZXh0OiAgICAgICAgV2hldGhlciB0aGUgdGV4dCBpcyBmdWxsIHRleHQgb3Igbm90LgoKICAgIDpyZXR1cm5zOiBUaGUgYW5vbnltaXplZCB0ZXh0LgogICAgIiIiCiAgICBpZiBub3QgdGV4dDoKICAgICAgICByZXR1cm4gIiIKCiAgICBhbm9ueW1pemVyX2VuZ2luZSA9IF9nZXRfYW5vbnltaXplcl9lbmdpbmUoKQogICAgaWYgbm90IGVudGl0eV9vcGVyYXRvcl9tYXA6CiAgICAgICAgb3BlcmF0b3JzID0gTm9uZQogICAgZWxzZToKICAgICAgICAjIENyZWF0ZSBPcGVyYXRvckNvbmZpZyBiYXNlZCBvbiB0aGUgZW50aXR5X29wZXJhdG9yX21hcAogICAgICAgIG9wZXJhdG9ycyA9IHsKICAgICAgICAgICAgZW50aXR5OiBPcGVyYXRvckNvbmZpZyhvcGVyYXRvcl9uYW1lLCBvcGVyYXRvcl9wYXJhbXMpCiAgICAgICAgICAgIGZvciBlbnRpdHksIChvcGVyYXRvcl9uYW1lLCBvcGVyYXRvcl9wYXJhbXMpIGluIGVudGl0eV9vcGVyYXRvcl9tYXAuaXRlbXMoKQogICAgICAgIH0KCiAgICBpZiBpc19mdWxsX3RleHQ6CiAgICAgICAgIyBBbm9ueW1pemUgdGhlIGVudGlyZSB0ZXh0CiAgICAgICAgcmV0dXJuIGFub255bWl6ZXJfZW5naW5lLmFub255bWl6ZSgKICAgICAgICAgICAgdGV4dD10ZXh0LCBhbmFseXplcl9yZXN1bHRzPWFuYWx5emVfcmVzdWx0cywgb3BlcmF0b3JzPW9wZXJhdG9ycwogICAgICAgICkudGV4dAogICAgIyBUb2tlbml6ZSB0aGUgdGV4dCB0byBzZW50ZW5jZXMKICAgIHNlbnRlbmNlcyA9IG5sdGsuc2VudF90b2tlbml6ZSh0ZXh0KQogICAgYW5vbnltaXplZF9zZW50ZW5jZXMgPSBbXQogICAgY3VycmVudF9pZHggPSAwCgogICAgIyBGaW5kIHRoZSBzZW50ZW5jZSB0aGF0IGhhcyBwaWkgZW50aXR5CiAgICBmb3Igc2VudGVuY2UgaW4gc2VudGVuY2VzOgogICAgICAgIHN0YXJ0X2lkeCA9IGN1cnJlbnRfaWR4CiAgICAgICAgZW5kX2lkeCA9IHN0YXJ0X2lkeCArIGxlbihzZW50ZW5jZSkKCiAgICAgICAgIyBHZXQgdGhlIGVudGl0aWVzIHRoYXQgYXJlIGluIHRoZSBzZW50ZW5jZSwgdXBkYXRlIGh0ZSBzdGFydF9pZHggYW5kIGVuZF9pZHgKICAgICAgICBzZW50ZW5jZV9yZXN1bHRzID0gWwogICAgICAgICAgICBwYS5SZWNvZ25pemVyUmVzdWx0KAogICAgICAgICAgICAgICAgcmVzdWx0LmVudGl0eV90eXBlLAogICAgICAgICAgICAgICAgc3RhcnQ9cmVzdWx0LnN0YXJ0IC0gc3RhcnRfaWR4LAogICAgICAgICAgICAgICAgZW5kPXJlc3VsdC5lbmQgLSBzdGFydF9pZHgsCiAgICAgICAgICAgICAgICBzY29yZT1yZXN1bHQuc2NvcmUsCiAgICAgICAgICAgICkKICAgICAgICAgICAgZm9yIHJlc3VsdCBpbiBhbmFseXplX3Jlc3VsdHMKICAgICAgICAgICAgaWYgcmVzdWx0LnN0YXJ0ID49IHN0YXJ0X2lkeCBhbmQgcmVzdWx0LmVuZCA8PSBlbmRfaWR4CiAgICAgICAgXQoKICAgICAgICAjIElmIFBJSSBpcyBkZXRlY3RlZAogICAgICAgIGlmIHNlbnRlbmNlX3Jlc3VsdHM6CiAgICAgICAgICAgIGFub255bWl6ZWRfc2VudGVuY2UgPSBhbm9ueW1pemVyX2VuZ2luZS5hbm9ueW1pemUoCiAgICAgICAgICAgICAgICB0ZXh0PXNlbnRlbmNlLCBhbmFseXplcl9yZXN1bHRzPXNlbnRlbmNlX3Jlc3VsdHMsIG9wZXJhdG9ycz1vcGVyYXRvcnMKICAgICAgICAgICAgKS50ZXh0CiAgICAgICAgICAgIGFub255bWl6ZWRfc2VudGVuY2VzLmFwcGVuZChhbm9ueW1pemVkX3NlbnRlbmNlKQoKICAgICAgICBjdXJyZW50X2lkeCA9IGVuZF9pZHgKCiAgICByZXR1cm4gIiAiLmpvaW4oYW5vbnltaXplZF9zZW50ZW5jZXMpCgoKZGVmIF9nZXRfdG9rZW5zKAogICAgdGV4dDogc3RyLCBhbmFseXplX3Jlc3VsdHM6IGxpc3RbcGEuUmVjb2duaXplclJlc3VsdF0sIGlzX2Z1bGw6IGJvb2wgPSBUcnVlCikgLT4gbGlzdFtzdHJdOgogICAgIiIiCiAgICBHZXQgdGhlIGZ1bGwgdG9rZW5zIG9yIG9ubHkgY29udGFpbnMgdGhlIGVudGl0aWVzIHRoYXQgY2FuIGZvcm0gYSBzZW50ZW5jZS4KCiAgICA6cGFyYW0gdGV4dDogICAgICAgICAgICBUaGUgdGV4dCBmb3IgYW5hbHlzaXMuCiAgICA6cGFyYW0gYW5hbHl6ZV9yZXN1bHRzOiBUaGUgbGlzdCBvZiBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0IGNvbnN0cnVjdGVkIGZyb20KICAgIDpwYXJhbSBpc19mdWxsOiAgICAgICAgIFdoZXRoZXIgcmV0dXJuIGZ1bGwgdG9rZW5zIG9yIGp1c3QgdGhlIHRva2VucyB0aGF0IG9ubHkgY29udGFpbnMgdGhlIGVudGl0aWVzIHRoYXQgY2FuIGZvcm0gYSBzZW50ZW5jZS4KCiAgICA6cmV0dXJuczogVGhlIHRva2Vucy4KICAgICIiIgoKICAgIHRva2VucyA9IFtdCiAgICAjIHNvcnQgYnkgc3RhcnQgaW5kZXgKICAgIHJlc3VsdHMgPSBzb3J0ZWQoYW5hbHl6ZV9yZXN1bHRzLCBrZXk9bGFtYmRhIHg6IHguc3RhcnQpCiAgICBmb3IgaSwgcmVzIGluIGVudW1lcmF0ZShyZXN1bHRzKToKICAgICAgICBpZiBpID09IDA6CiAgICAgICAgICAgIHRva2Vucy5hcHBlbmQodGV4dFs6IHJlcy5zdGFydF0pCgogICAgICAgICMgYXBwZW5kIGVudGl0eSB0ZXh0IGFuZCBlbnRpdHkgdHlwZQogICAgICAgIHRva2Vucy5hcHBlbmQoKHRleHRbcmVzLnN0YXJ0IDogcmVzLmVuZF0sIHJlcy5lbnRpdHlfdHlwZSkpCgogICAgICAgICMgaWYgYW5vdGhlciBlbnRpdHkgY29taW5nIGkuZS4gd2UncmUgbm90IGF0IHRoZSBsYXN0IHJlc3VsdHMgZWxlbWVudCwKICAgICAgICAjIGFkZCB0ZXh0IHVwIHRvIG5leHQgZW50aXR5CiAgICAgICAgaWYgaSAhPSBsZW4ocmVzdWx0cykgLSAxOgogICAgICAgICAgICB0b2tlbnMuYXBwZW5kKHRleHRbcmVzLmVuZCA6IHJlc3VsdHNbaSArIDFdLnN0YXJ0XSkKICAgICAgICAjIGlmIG5vIG1vcmUgZW50aXRpZXMgY29taW5nLCBhZGQgYWxsIHJlbWFpbmluZyB0ZXh0CiAgICAgICAgZWxzZToKICAgICAgICAgICAgdG9rZW5zLmFwcGVuZCh0ZXh0W3Jlcy5lbmQgOl0pCgogICAgIyBnZXQgdGhlIHRva2VucyB0aGF0IG9ubHkgY29udGFpbnMgdGhlIGVudGl0aWVzIHRoYXQgY2FuIGZvcm0gYSBzZW50ZW5jZQogICAgcGFydF9hbm5vbnRhdGVkX3Rva2VucyA9IFtdCiAgICBpZiBub3QgaXNfZnVsbDoKICAgICAgICBsYXN0X2VuZF9zZW50ZW5jZSA9IDAKICAgICAgICBmb3IgaSwgdG9rZW4gaW4gZW51bWVyYXRlKHRva2Vucyk6CiAgICAgICAgICAgIGlmIGFueShpdGVtIGluIHRva2VuIGZvciBpdGVtIGluIFsiLiIsICIhIiwgIj8iXSkgYW5kIGFueSgKICAgICAgICAgICAgICAgIHR5cGUoaXRlbSkgaXMgdHVwbGUgZm9yIGl0ZW0gaW4gdG9rZW5zW2xhc3RfZW5kX3NlbnRlbmNlOmldCiAgICAgICAgICAgICk6CiAgICAgICAgICAgICAgICBwYXJ0X2Fubm9udGF0ZWRfdG9rZW5zLmFwcGVuZCh0b2tlbnNbbGFzdF9lbmRfc2VudGVuY2U6aV0pCiAgICAgICAgICAgICAgICBsYXN0X2VuZF9zZW50ZW5jZSA9IGkKICAgICAgICByZXR1cm4gcGFydF9hbm5vbnRhdGVkX3Rva2VucwogICAgcmV0dXJuIHRva2VucwoKCmRlZiBfYW5ub3RhdGUoCiAgICB0ZXh0OiBzdHIsIHN0X2FuYWx5emVfcmVzdWx0czogbGlzdFtwYS5SZWNvZ25pemVyUmVzdWx0XSwgaXNfZnVsbF9odG1sOiBib29sID0gVHJ1ZQopIC0+IGxpc3Rbc3RyXToKICAgICIiIgogICAgQW5ub3RhdGUgaWRlbnRpZmllZCBpbnB1dCB1c2luZyBQcmVzaWRpbyBBbm9ueW1pemVyLgoKICAgIDpwYXJhbSB0ZXh0OiAgICAgICAgICAgICAgIFRoZSB0ZXh0IGZvciBhbmFseXNpcy4KICAgIDpwYXJhbSBzdF9hbmFseXplX3Jlc3VsdHM6IFRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcy4KICAgIDpwYXJhbSBpc19mdWxsX2h0bWw6ICAgICAgIFdoZXRoZXIgZ2VuZXJhdGUgZnVsbCBodG1sIG9yIG5vdC4KCiAgICA6cmV0dXJuczogVGhlIGxpc3Qgb2YgdG9rZW5zIHdpdGggdGhlIGlkZW50aWZpZWQgZW50aXRpZXMuCgogICAgIiIiCiAgICByZXR1cm4gX2dldF90b2tlbnModGV4dCwgc3RfYW5hbHl6ZV9yZXN1bHRzLCBpc19mdWxsX2h0bWwpCgoKZGVmIF9wcm9jZXNzKAogICAgdGV4dDogc3RyLAogICAgbW9kZWw6IHBhLkFuYWx5emVyRW5naW5lLAogICAgc2NvcmVfdGhyZXNob2xkOiBmbG9hdCwKICAgIGVudGl0aWVzOiBsaXN0W3N0cl0gPSBOb25lLAogICAgZW50aXRpZXNfb3BlcmF0b3JfbWFwOiBkaWN0ID0gTm9uZSwKICAgIGlzX2Z1bGxfdGV4dDogYm9vbCA9IFRydWUsCikgLT4gdHVwbGVbc3RyLCBsaXN0XToKICAgICIiIgogICAgUHJvY2VzcyB0aGUgdGV4dCBvZiBzdHIgdXNpbmcgdGhlIG1vZGVsLgoKICAgIDpwYXJhbSB0ZXh0OiAgICAgICAgICAgICAgICAgIFRleHQgdG8gcHJvY2VzcwogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgICAgTW9kZWwgdG8gdXNlIGZvciBwcm9jZXNzaW5nCiAgICA6cGFyYW0gZW50aXRpZXM6ICAgICAgICAgICAgICBFbnRpdGllcyB0byByZWNvZ25pemUKICAgIDpwYXJhbSBlbnRpdGllc19vcGVyYXRvcl9tYXA6IFRoZSBlbnRpdHlfb3BlcmF0b3JfbWFwIGlzIGEgZGljdGlvbmFyeSB0aGF0IG1hcHMgZW50aXR5IHRvIG9wZXJhdG9yIG5hbWUgYW5kIG9wZXJhdG9yIHBhcmFtcy4KICAgIDpwYXJhbSBzY29yZV90aHJlc2hvbGQ6ICAgICAgIFRoZSBzY29yZSB0aHJlc2hvbGQgdG8gdXNlIGZvciByZWNvZ25pdGlvbgogICAgOnBhcmFtIGlzX2Z1bGxfdGV4dDogICAgICAgICAgV2hldGhlciB0byByZXR1cm4gdGhlIGZ1bGwgdGV4dCBvciBqdXN0IHRoZSBhbm5vdGF0ZWQgdGV4dAoKICAgIDpyZXR1cm5zOiBBIHR1cGxlIG9mOgoKICAgICAgICAgICAgICAqIHRoZSBhbm9ueW1pemVkIHRleHQKICAgICAgICAgICAgICAqIHRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcwogICAgIiIiCgogICAgIyBnZXQgdGhlIGFuYWx5emVyIGVuZ2luZQogICAgYW5hbHl6ZXIgPSBtb2RlbAoKICAgICMgYW5hbHl6ZSB0aGUgdGV4dCB0aGF0IGNhbiBiZSB1c2VkIGZvciBhbm9ueW1pemF0aW9uCiAgICByZXN1bHRzID0gYW5hbHl6ZXIuYW5hbHl6ZSgKICAgICAgICB0ZXh0PXRleHQsCiAgICAgICAgbGFuZ3VhZ2U9ImVuIiwKICAgICAgICBlbnRpdGllcz1lbnRpdGllcywKICAgICAgICBzY29yZV90aHJlc2hvbGQ9c2NvcmVfdGhyZXNob2xkLAogICAgICAgIHJldHVybl9kZWNpc2lvbl9wcm9jZXNzPVRydWUsCiAgICApCgogICAgIyBhbm9ueW1pemUgdGhlIHRleHQsIHJlcGxhY2UgdGhlIHBpaSBlbnRpdGllcyB3aXRoIHRoZSBsYWJlbHMKICAgIGFub255bWl6ZWRfdGV4dCA9IF9hbm9ueW1pemUodGV4dCwgcmVzdWx0cywgZW50aXRpZXNfb3BlcmF0b3JfbWFwLCBpc19mdWxsX3RleHQpCgogICAgcmV0dXJuIGFub255bWl6ZWRfdGV4dCwgcmVzdWx0cwoKCmRlZiBfZ2V0X3NpbmdsZV9odG1sKAogICAgdGV4dDogc3RyLCByZXN1bHRzOiBsaXN0W3BhLlJlY29nbml6ZXJSZXN1bHRdLCBpc19mdWxsX2h0bWw6IGJvb2wgPSBUcnVlCik6CiAgICAiIiIKICAgIEdlbmVyYXRlIHRoZSBodG1sIGZvciBhIHNpbmdsZSB0eHQgZmlsZS4KCiAgICA6cGFyYW0gdGV4dDogICAgICAgICBUaGUgdGV4dCBmb3IgYW5hbHlzaXMuCiAgICA6cGFyYW0gcmVzdWx0czogICAgICBUaGUgbGlzdCBvZiBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0IGNvbnN0cnVjdGVkIGZyb20gYW5hbHlzaXMuCiAgICA6cGFyYW0gaXNfZnVsbF9odG1sOiBXaGV0aGVyIGdlbmVyYXRlIGZ1bGwgaHRtbCBvciBub3QuCgogICAgOnJldHVybnM6IFRoZSBodG1sIHN0cmluZyBmb3IgYSBzaW5nbGUgdHh0IGZpbGUuCiAgICAiIiIKICAgICMgY29udmVydCB0aGUgcmVzdWx0cyB0byB0b2tlbnMgdG8gZ2VuZXJhdGUgdGhlIGh0bWwKICAgIHRva2VucyA9IF9hbm5vdGF0ZSh0ZXh0LCByZXN1bHRzLCBpc19mdWxsX2h0bWwpCiAgICBodG1sID0gYXRfdXRpbC5nZXRfYW5ub3RhdGVkX2h0bWwoKnRva2VucykKCiAgICAjIGF2b2lkIHRoZSBlcnJvciBkdXJpbmcgcmVuZGVyaW5nIG9mIHRoZSBcbiBpbiB0aGUgaHRtbAogICAgYmFja3NsYXNoX2NoYXIgPSAiXFwiCgogICAgaHRtbF9zdHIgPSBmIjxwPntodG1sLnJlcGxhY2UoJ3tiYWNrc2xhc2hfY2hhcn1uJywgJzxicj4nKX08L3A+IgoKICAgIHJldHVybiBodG1sX3N0cgoKCmRlZiBfZ2V0X3NpbmdsZV9qc29uKHJlc3VsdHM6IGxpc3RbcGEuUmVjb2duaXplclJlc3VsdF0sIGlzX2Z1bGxfcmVwb3J0OiBib29sID0gVHJ1ZSk6CiAgICAiIiIKICAgIEdlbmVyYXRlIHRoZSBqc29uIGZvciBhIHNpbmdsZSB0eHQgZmlsZS4KCiAgICA6cGFyYW0gcmVzdWx0czogICAgICAgIFRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcy4KICAgIDpwYXJhbSBpc19mdWxsX3JlcG9ydDogV2hldGhlciBnZW5lcmF0ZSBmdWxsIGpzb24gb3Igbm90LgoKICAgIDpyZXR1cm5zOiBUaGUganNvbiBzdHJpbmcgZm9yIGEgc2luZ2xlIHR4dCBmaWxlLgogICAgIiIiCiAgICAjIGdlbmVyYXRlIHRoZSBzdGF0cyByZXBvcnQgaWYgbmVlZGVkCiAgICBpZiBub3QgaXNfZnVsbF9yZXBvcnQ6CiAgICAgICAgc3RhdHMgPSBbXQogICAgICAgICMgYWRkIHRoZSBzaW1wbGlmeSBzdGF0cyBsb2dpYyBoZXJlCiAgICAgICAgZm9yIGl0ZW0gaW4gcmVzdWx0czoKICAgICAgICAgICAgaXRlbS5hbmFseXNpc19leHBsYW5hdGlvbiA9IE5vbmUKICAgICAgICAgICAgc3RhdHMuYXBwZW5kKGl0ZW0pCiAgICBlbHNlOgogICAgICAgIHN0YXRzID0gcmVzdWx0cwoKICAgIHJldHVybiBzdGF0cwoKCmRlZiBfZ2V0X2FsbF9odG1sKAogICAgdHh0X2NvbnRlbnQ6IGRpY3QsCiAgICByZXNfZGljdDogZGljdCwKICAgIGlzX2Z1bGxfaHRtbDogYm9vbCA9IFRydWUsCik6CiAgICAiIiIKICAgIEdlbmVyYXRlIHRoZSBodG1sIGZvciBhbGwgdHh0IGZpbGVzLgoKICAgIDpwYXJhbSB0eHRfY29udGVudDogIFRoZSBkaWN0aW9uYXJ5IG9mIHR4dCBmaWxlIG5hbWUgYW5kIGNvbnRlbnQuCiAgICA6cGFyYW0gcmVzX2RpY3Q6ICAgICBUaGUgZGljdGlvbmFyeSBvZiB0eHQgZmlsZSBuYW1lIGFuZCB0aGUgbGlzdCBvZiBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0IGNvbnN0cnVjdGVkIGZyb20gYW5hbHlzaXMuCiAgICA6cGFyYW0gaXNfZnVsbF9odG1sOiBXaGV0aGVyIGdlbmVyYXRlIGZ1bGwgaHRtbCBvciBub3QuCgogICAgOnJldHVybnM6IFRoZSBodG1sIHN0cmluZyBmb3IgYWxsIHR4dCBmaWxlcy4KCiAgICAiIiIKICAgICMgVGhlc2UgYXJlIHBsYWNlaG9sZGVyIGZvciB0aGUgaHRtbCBzdHJpbmcKICAgIGh0bWxfaW5kZXggPSAiPGh0bWw+PGhlYWQ+PHRpdGxlPkhpZ2hsaWdodGVkIFBpaSBFbnRpdGllczwvdGl0bGU+PC9oZWFkPjxib2R5PjxoMT5IaWdobGlnaHRlZCBQaWkgRW50aXRpZXM8L2gxPjx1bD4iCiAgICBodG1sX2NvbnRlbnQgPSAiIgogICAgZm9yIHR4dF9maWxlLCByZXN1bHRzIGluIHJlc19kaWN0Lml0ZW1zKCk6CiAgICAgICAgdHh0ID0gdHh0X2NvbnRlbnRbdHh0X2ZpbGVdCiAgICAgICAgaHRtbF9pbmRleCArPSBmIjxsaT48YSBocmVmPScje3R4dF9maWxlfSc+e3R4dF9maWxlfTwvYT48L2xpPiIKICAgICAgICBodG1sX2NvbnRlbnQgKz0gZiI8bGk+PGgyPnt0eHRfZmlsZX08L2gyPjxwPntfZ2V0X3NpbmdsZV9odG1sKHR4dCwgcmVzdWx0cywgaXNfZnVsbF9odG1sKX08L3A+PC9saT4iCiAgICBodG1sX2luZGV4ICs9ICI8L3VsPiIKICAgIGh0bWxfcmVzID0gZiJ7aHRtbF9pbmRleH17aHRtbF9jb250ZW50fTwvYm9keT48L2h0bWw+IgoKICAgIHJldHVybiBodG1sX3JlcwoKCmRlZiBfZ2V0X2FsbF9ycHQocmVzX2RpY3Q6IGRpY3QsIGlzX2Z1bGxfcmVwb3J0OiBib29sID0gVHJ1ZSk6CiAgICAiIiIKICAgIEdlbmVyYXRlIHRoZSBzdGF0cyByZXBvcnQgZm9yIGFsbCB0eHQgZmlsZXMuCgogICAgOnBhcmFtIHJlc19kaWN0OiAgICAgICBUaGUgZGljdGlvbmFyeSBvZiB0eHQgZmlsZSBuYW1lIGFuZCB0aGUgbGlzdCBvZiBQcmVzaWRpbyBSZWNvZ25pemVyUmVzdWx0IGNvbnN0cnVjdGVkIGZyb20gYW5hbHlzaXMuCiAgICA6cGFyYW0gaXNfZnVsbF9yZXBvcnQ6IFdoZXRoZXIgZ2VuZXJhdGUgZnVsbCByZXBvcnQgb3Igbm90LgoKICAgIDpyZXR1cm5zOiBUaGUgc3RhdHMgcmVwb3J0IGZvciBhbGwgdHh0IGZpbGVzLgogICAgIiIiCiAgICAjIFRoZXNlIGFyZSBwbGFjZWhvbGRlciBmb3IgdGhlIGpzb24gcmVwb3J0CiAgICBzdGF0c19kaWN0ID0ge30KICAgIGZvciB0eHRfZmlsZSwgcmVzdWx0cyBpbiByZXNfZGljdC5pdGVtcygpOgogICAgICAgIG5ld19zdGF0cyA9IFtdCiAgICAgICAgZm9yIGl0ZW0gaW4gX2dldF9zaW5nbGVfanNvbihyZXN1bHRzLCBpc19mdWxsX3JlcG9ydCk6CiAgICAgICAgICAgIGlmIGlzX2Z1bGxfcmVwb3J0OgogICAgICAgICAgICAgICAgaXRlbS5hbmFseXNpc19leHBsYW5hdGlvbiA9IGl0ZW0uYW5hbHlzaXNfZXhwbGFuYXRpb24udG9fZGljdCgpCiAgICAgICAgICAgICAgICBuZXdfc3RhdHMuYXBwZW5kKGl0ZW0udG9fZGljdCgpKQogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgdG1wX2RpY3QgPSBpdGVtLnRvX2RpY3QoKQogICAgICAgICAgICAgICAgdG1wX2RpY3QucG9wKCJhbmFseXNpc19leHBsYW5hdGlvbiIpCiAgICAgICAgICAgICAgICB0bXBfZGljdC5wb3AoInJlY29nbml0aW9uX21ldGFkYXRhIikKICAgICAgICAgICAgICAgIG5ld19zdGF0cy5hcHBlbmQodG1wX2RpY3QpCiAgICAgICAgc3RhdHNfZGljdFt0eHRfZmlsZV0gPSBuZXdfc3RhdHMKICAgIHJldHVybiBzdGF0c19kaWN0CgoKZGVmIHJlY29nbml6ZV9waWkoCiAgICBjb250ZXh0OiBtbHJ1bi5NTENsaWVudEN0eCwKICAgIGlucHV0X3BhdGg6IHN0ciB8IHBhdGhsaWIuUGF0aCwKICAgIGh0bWxfa2V5OiBzdHIsCiAgICBzY29yZV90aHJlc2hvbGQ6IGZsb2F0LAogICAgb3V0cHV0X2RpcmVjdG9yeTogc3RyID0gTm9uZSwKICAgIGVudGl0aWVzOiBsaXN0WwogICAgICAgIHN0cgogICAgXSA9IE5vbmUsICAjIExpc3Qgb2YgZW50aXRpZXMgdG8gcmVjb2duaXplLCBkZWZhdWx0IGlzIHJlY29nbml6aW5nIGFsbAogICAgZW50aXR5X29wZXJhdG9yX21hcDogZGljdCA9IE5vbmUsCiAgICBtb2RlbDogc3RyID0gTm9uZSwKICAgIGdlbmVyYXRlX2pzb246IGJvb2wgPSBUcnVlLAogICAgZ2VuZXJhdGVfaHRtbDogYm9vbCA9IFRydWUsCiAgICBpc19mdWxsX3RleHQ6IGJvb2wgPSBUcnVlLAogICAgaXNfZnVsbF9odG1sOiBib29sID0gVHJ1ZSwKICAgIGlzX2Z1bGxfcmVwb3J0OiBib29sID0gVHJ1ZSwKKSAtPiB0dXBsZVtzdHIsIHBkLkRhdGFGcmFtZSwgZGljdCwgZGljdF0gfCB0dXBsZVtzdHIsIHBkLkRhdGFGcmFtZSwgZGljdF06CiAgICAiIiIKICAgIFdhbGsgdGhyb3VnaCB0aGUgaW5wdXQgcGF0aCwgcmVjb2duaXplIFBJSSBpbiB0ZXh0IGFuZCBzdG9yZSB0aGUgYW5vbnltaXplZCB0ZXh0IGluIHRoZSBvdXRwdXQgcGF0aC4KICAgIEdlbmVyYXRlIHRoZSBodG1sIHdpdGggZGlmZmVyZW50IGNvbG9ycyBmb3IgZWFjaCBlbnRpdHksIGpzb24gcmVwb3J0IG9mIHRoZSBleHBsYW5hdGlvbi4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgIFRoZSBNTFJ1biBjb250ZXh0LiB0aGlzIGlzIG5lZWRlZCBmb3IgbG9nIHRoZSBhcnRpZmFjdHMuCiAgICA6cGFyYW0gaW5wdXRfcGF0aDogICAgICAgICAgIFRoZSBpbnB1dCBwYXRoIG9mIHRoZSB0ZXh0IGZpbGVzIG5lZWRzIHRvIGJlIGFuYWx5emVkLgogICAgOnBhcmFtIGh0bWxfa2V5OiAgICAgICAgICAgICBUaGUgaHRtbCBrZXkgZm9yIHRoZSBhcnRpZmFjdC4KICAgIDpwYXJhbSBzY29yZV90aHJlc2hvbGQ6ICAgICAgVGhlIHNjb3JlIHRocmVzaG9sZCB0byBtYXJrIHRoZSByZWNvZ25pdGlvbiBhcyB0cnVzdGVkLgogICAgOnBhcmFtIG91dHB1dF9kaXJlY3Rvcnk6ICAgICBUaGUgb3V0cHV0IGRpcmVjdG9yeSBwYXRoIHRvIHN0b3JlIHRoZSBhbm9ueW1pemVkIHRleHQuCiAgICA6cGFyYW0gZW50aXRpZXM6ICAgICAgICAgICAgIFRoZSBsaXN0IG9mIGVudGl0aWVzIHRvIHJlY29nbml6ZS4KICAgIDpwYXJhbSBlbnRpdHlfb3BlcmF0b3JfbWFwOiAgVGhlIG1hcCBvZiBlbnRpdHkgdG8gb3BlcmF0b3IgKG1hc2ssIHJlZGFjdCwgcmVwbGFjZSwga2VlcCwgaGFzaCwgYW5kIGl0cyBwYXJhbXMpCiAgICA6cGFyYW0gbW9kZWw6ICAgICAgICAgICAgICAgIFRoZSBtb2RlbCB0byB1c2UuIENhbiBiZSAic3BhY3kiLCAiZmxhaXIiLCAicGF0dGVybiIgb3IgIndob2xlIi4KICAgIDpwYXJhbSBnZW5lcmF0ZV9qc29uOiAgICAgICAgV2hldGhlciB0byBnZW5lcmF0ZSB0aGUganNvbiByZXBvcnQgb2YgdGhlIGV4cGxhbmF0aW9uLgogICAgOnBhcmFtIGdlbmVyYXRlX2h0bWw6ICAgICAgICBXaGV0aGVyIHRvIGdlbmVyYXRlIHRoZSBodG1sIHJlcG9ydCBvZiB0aGUgZXhwbGFuYXRpb24uCiAgICA6cGFyYW0gaXNfZnVsbF90ZXh0OiAgICAgICAgIFdoZXRoZXIgdG8gcmV0dXJuIHRoZSBmdWxsIHRleHQgb3Igb25seSB0aGUgbWFza2VkIHRleHQuCiAgICA6cGFyYW0gaXNfZnVsbF9odG1sOiAgICAgICAgIFdoZXRoZXIgdG8gcmV0dXJuIHRoZSBmdWxsIGh0bWwgb3IganVzdCB0aGUgYW5ub3RhdGVkIHRleHQKICAgIDpwYXJhbSBpc19mdWxsX3JlcG9ydDogICAgICAgV2hldGhlciB0byByZXR1cm4gdGhlIGZ1bGwgcmVwb3J0IG9yIGp1c3QgdGhlIHNjb3JlIGFuZCBzdGFydCwgZW5kIGluZGV4CgogICAgOnJldHVybnM6IEEgdHVwbGUgb2Y6CgogICAgICAgICAgICAgICogUGF0aCB0byB0aGUgb3V0cHV0IGRpcmVjdG9yeQogICAgICAgICAgICAgICogVGhlIGpzb24gcmVwb3J0IG9mIHRoZSBleHBsYW5hdGlvbiAoaWYgZ2VuZXJhdGVfanNvbiBpcyBUcnVlKQogICAgICAgICAgICAgICogQSBkaWN0aW9uYXJ5IG9mIGVycm9ycyBmaWxlcyB0aGF0IHdlcmUgbm90IHByb2Nlc3NlZAoKICAgICIiIgoKICAgICMgU2V0IG91dHB1dCBkaXJlY3RvcnkKICAgIGlmIG91dHB1dF9kaXJlY3RvcnkgaXMgTm9uZToKICAgICAgICBvdXRwdXRfZGlyZWN0b3J5ID0gdGVtcGZpbGUubWtkdGVtcCgpCgogICAgIyBDcmVhdGUgdGhlIG91dHB1dCBkaXJlY3Rvcnk6CiAgICBvdXRwdXRfZGlyZWN0b3J5ID0gcGF0aGxpYi5QYXRoKG91dHB1dF9kaXJlY3RvcnkpCiAgICBpZiBub3Qgb3V0cHV0X2RpcmVjdG9yeS5leGlzdHMoKToKICAgICAgICBvdXRwdXRfZGlyZWN0b3J5Lm1rZGlyKHBhcmVudHM9VHJ1ZSwgZXhpc3Rfb2s9VHJ1ZSkKCiAgICB0eHRfZmlsZXNfZGlyZWN0b3J5ID0gcGF0aGxpYi5QYXRoKGlucHV0X3BhdGgpCiAgICBzdWNjZXNzZXMgPSBbXQogICAgZXJyb3JzID0ge30KCiAgICByZXNfZGljdCA9IHt9CiAgICB0eHRfY29udGVudCA9IHt9CiAgICAjIExvYWQgdGhlIG1vZGVsOgogICAgYW5hbHl6ZXIgPSBfZ2V0X2FuYWx5emVyX2VuZ2luZShtb2RlbCwgZW50aXRpZXMpCiAgICBsb2dnZXIuaW5mbygiTW9kZWwgbG9hZGVkIikKICAgICMgR28gb3ZlciB0aGUgdGV4dCBmaWxlcyBpbiB0aGUgaW5wdXQgcGF0aCwgYW5hbHl6ZSBhbmQgYW5vbnltaXplIHRoZW06CiAgICBmb3IgdHh0X2ZpbGUgaW4gdHFkbSgKICAgICAgICBsaXN0KHR4dF9maWxlc19kaXJlY3RvcnkuZ2xvYigiKi50eHQiKSksCiAgICAgICAgZGVzYz0iUHJvY2Vzc2luZyBmaWxlcyIsCiAgICAgICAgdW5pdD0iZmlsZSIsCiAgICApOgogICAgICAgIHRyeToKICAgICAgICAgICAgIyBMb2FkIHRoZSBzdHIgZnJvbSB0aGUgdGV4dCBmaWxlCiAgICAgICAgICAgIHRleHQgPSB0eHRfZmlsZS5yZWFkX3RleHQoKQogICAgICAgICAgICB0eHRfY29udGVudFtzdHIodHh0X2ZpbGUpXSA9IHRleHQKICAgICAgICAgICAgIyBQcm9jZXNzIHRoZSB0ZXh0IHRvIHJlY29naW56ZSB0aGUgcGlpIGVudGl0aWVzIGluIGl0CiAgICAgICAgICAgIGFub255bWl6ZWRfdGV4dCwgcmVzdWx0cyA9IF9wcm9jZXNzKAogICAgICAgICAgICAgICAgdGV4dD10ZXh0LAogICAgICAgICAgICAgICAgbW9kZWw9YW5hbHl6ZXIsCiAgICAgICAgICAgICAgICBlbnRpdGllcz1lbnRpdGllcywKICAgICAgICAgICAgICAgIGVudGl0aWVzX29wZXJhdG9yX21hcD1lbnRpdHlfb3BlcmF0b3JfbWFwLAogICAgICAgICAgICAgICAgc2NvcmVfdGhyZXNob2xkPXNjb3JlX3RocmVzaG9sZCwKICAgICAgICAgICAgICAgIGlzX2Z1bGxfdGV4dD1pc19mdWxsX3RleHQsCiAgICAgICAgICAgICkKICAgICAgICAgICAgcmVzX2RpY3Rbc3RyKHR4dF9maWxlKV0gPSByZXN1bHRzCiAgICAgICAgICAgICMgU3RvcmUgdGhlIGFub255bWl6ZWQgdGV4dCBpbiB0aGUgb3V0cHV0IHBhdGgKICAgICAgICAgICAgb3V0cHV0X2ZpbGUgPSBvdXRwdXRfZGlyZWN0b3J5IC8gZiJ7dHh0X2ZpbGUuc3RlbX0udHh0IgogICAgICAgICAgICBvdXRwdXRfZmlsZS5wYXJlbnQubWtkaXIocGFyZW50cz1UcnVlLCBleGlzdF9vaz1UcnVlKQogICAgICAgICAgICB3aXRoIG9wZW4ob3V0cHV0X2ZpbGUsICJ3IikgYXMgZjoKICAgICAgICAgICAgICAgIGYud3JpdGUoYW5vbnltaXplZF90ZXh0KQogICAgICAgICAgICBzdWNjZXNzZXMuYXBwZW5kKFt0eHRfZmlsZS5uYW1lLCBvdXRwdXRfZmlsZS5uYW1lXSkKICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgICAgIGVycm9yc1tzdHIodHh0X2ZpbGUpXSA9IHN0cihlKQogICAgICAgICAgICBsb2dnZXIuZXJyb3IoZiJFcnJvciBwcm9jZXNzaW5nIHt0eHRfZmlsZX06IHtlfSIpCgogICAgc3VjY2Vzc2VzID0gcGQuRGF0YUZyYW1lKAogICAgICAgIHN1Y2Nlc3NlcywKICAgICAgICBjb2x1bW5zPVsib3JpZ2luYWxfZmlsZSIsICJhbm9ueW1pemVkX2ZpbGUiXSwKICAgICkKCiAgICBpZiBnZW5lcmF0ZV9odG1sOgogICAgICAgICMgR2VuZXJhdGUgdGhlIGh0bWwgcmVwb3J0CiAgICAgICAgaHRtbF9yZXMgPSBfZ2V0X2FsbF9odG1sKHR4dF9jb250ZW50LCByZXNfZGljdCwgaXNfZnVsbF9odG1sKQogICAgICAgICMgU3RvcmUgdGhlIGh0bWwgcmVwb3J0IGluIHRoZSBjb250ZXh0CiAgICAgICAgYXJ0aV9odG1sID0gbWxydW4uYXJ0aWZhY3RzLkFydGlmYWN0KGJvZHk9aHRtbF9yZXMsIGZvcm1hdD0iaHRtbCIsIGtleT1odG1sX2tleSkKICAgICAgICBjb250ZXh0LmxvZ19hcnRpZmFjdChhcnRpX2h0bWwpCiAgICBpZiBnZW5lcmF0ZV9qc29uOgogICAgICAgICMgR2VuZXJhdGUgdGhlIGpzb24gcmVwb3J0CiAgICAgICAganNvbl9yZXMgPSBfZ2V0X2FsbF9ycHQocmVzX2RpY3QsIGlzX2Z1bGxfcmVwb3J0KQogICAgICAgIHJldHVybiBzdHIob3V0cHV0X2RpcmVjdG9yeSksIHN1Y2Nlc3NlcywgZXJyb3JzLCBqc29uX3JlcwogICAgcmV0dXJuIHN0cihvdXRwdXRfZGlyZWN0b3J5KSwgc3VjY2Vzc2VzLCBlcnJvcnMK + requirements: + - nltk + - pandas + - presidio-anonymizer + - presidio-analyzer + - torch + - flair@git+https://github.com/flairNLP/flair.git@d4ed67bf663e4066517f00397412510d90043653 + - st-annotated-text + - https://huggingface.co/beki/en_spacy_pii_distilbert/resolve/main/en_spacy_pii_distilbert-any-py3-none-any.whl + code_origin: '' + base_image: mlrun/mlrun + filename: pii_recognizer.py entry_points: analyze: - name: analyze outputs: - doc: The list of Presidio RecognizerResult constructed from the recognized Flair detections. - type: List[pa.RecognizerResult] - has_kwargs: false + type: list[pa.RecognizerResult] parameters: - name: self - name: text type: str doc: The text for analysis. - name: entities - type: List[str] + type: list[str] doc: The list of entities to recognize. - name: nlp_artifacts type: pa.nlp_engine.NlpArtifacts doc: Not used by this recognizer but needed for the interface. default: null - lineno: 381 + name: analyze doc: Analyze text and return the results. + has_kwargs: false has_varargs: false + lineno: 381 recognize_pii: - name: recognize_pii outputs: - doc: 'A tuple of:' - type: Union[Tuple[str, pd.DataFrame, dict, dict], Tuple[str, pd.DataFrame, - dict]] - has_kwargs: false + type: tuple[str, pd.DataFrame, dict, dict] | tuple[str, pd.DataFrame, dict] parameters: - name: context type: MLClientCtx doc: The MLRun context. this is needed for log the artifacts. - name: input_path - type: Union[str, Path] doc: The input path of the text files needs to be analyzed. - name: html_key type: str @@ -49,7 +68,7 @@ spec: doc: The output directory path to store the anonymized text. default: null - name: entities - type: List[str] + type: list[str] doc: The list of entities to recognize. default: null - name: entity_operator_map @@ -81,35 +100,15 @@ spec: type: bool doc: Whether to return the full report or just the score and start, end index default: true - lineno: 845 + name: recognize_pii doc: 'Walk through the input path, recognize PII in text and store the anonymized text in the output path. Generate the html with different colors for each entity, json report of the explanation.' + has_kwargs: false has_varargs: false - build: - base_image: mlrun/mlrun - requirements: - - nltk - - pandas - - presidio-anonymizer - - presidio-analyzer - - torch - - flair@git+https://github.com/flairNLP/flair.git@d4ed67bf663e4066517f00397412510d90043653 - - st-annotated-text - - https://huggingface.co/beki/en_spacy_pii_distilbert/resolve/main/en_spacy_pii_distilbert-any-py3-none-any.whl - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgoKaW1wb3J0IGxvZ2dpbmcKaW1wb3J0IG9zCmltcG9ydCBwYXRobGliCmltcG9ydCB0ZW1wZmlsZQppbXBvcnQgd2FybmluZ3MKZnJvbSB0eXBpbmcgaW1wb3J0IExpc3QsIFNldCwgVHVwbGUsIFVuaW9uCgppbXBvcnQgYW5ub3RhdGVkX3RleHQudXRpbCBhcyBhdF91dGlsCmltcG9ydCBtbHJ1bgppbXBvcnQgbmx0awppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCBwcmVzaWRpb19hbmFseXplciBhcyBwYQppbXBvcnQgcHJlc2lkaW9fYW5vbnltaXplciBhcyBwcmVfYW5veW1pemVyCmZyb20gcHJlc2lkaW9fYW5vbnltaXplci5lbnRpdGllcyBpbXBvcnQgT3BlcmF0b3JDb25maWcKZnJvbSB0cWRtIGltcG9ydCB0cWRtCgp0cnk6CiAgICBpbXBvcnQgZmxhaXIgYXMgZmwKZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3I6CiAgICBwcmludCgiRmxhaXIgaXMgbm90IGluc3RhbGxlZCIpCgojIFRoZXJlIGlzIGEgY29uZmxpY3QgYmV0d2VlbiBSdXN0LWJhc2VkIHRva2VuaXplcnMnIHBhcmFsbGVsIHByb2Nlc3NpbmcKIyBhbmQgUHl0aG9uJ3MgZm9yayBvcGVyYXRpb25zIGR1cmluZyBtdWx0aXByb2Nlc3NpbmcuIFRvIGF2b2lkIHRoaXMsIHdlIG5lZWQKIyB0aGUgZm9sbG93aW5nIHR3byBsaW5lcwoKb3MuZW52aXJvblsiVE9LRU5JWkVSU19QQVJBTExFTElTTSJdID0gImZhbHNlIgp3YXJuaW5ncy5maWx0ZXJ3YXJuaW5ncygiaWdub3JlIikKCmxvZ2dlciA9IGxvZ2dpbmcuZ2V0TG9nZ2VyKCJwaWktcmVjb2duaXplciIpCgoKIyBBZGQgdGhlIGNvbnN0YW50IGNsYXNzZXMgb2YgTW9kZWxzIGFuZCBFbnRpdGllcyB0byBnb3Zlcm4gdGhlIHdob2xlIHBhY2thZ2UKY2xhc3MgTW9kZWxzOgogICAgV0hPTEUgPSAid2hvbGUiCiAgICBQQVRURVJOID0gInBhdHRlcm4iCiAgICBTUEFDWSA9ICJzcGFjeSIKICAgIEZMQUlSID0gImZsYWlyIgoKCmNsYXNzIEVudGl0aWVzOgogICAgQ1JFRElUX0NBUkQgPSAiQ1JFRElUX0NBUkQiCiAgICBTU04gPSAiU1NOIgogICAgUEhPTkUgPSAiUEhPTkUiCiAgICBFTUFJTCA9ICJFTUFJTCIKICAgIExPQ0FUSU9OID0gIkxPQ0FUSU9OIgogICAgUEVSU09OID0gIlBFUlNPTiIKICAgIE5SUCA9ICJOUlAiCiAgICBPUkdBTklaQVRJT04gPSAiT1JHQU5JWkFUSU9OIgogICAgREFURV9USU1FID0gIkRBVEVfVElNRSIKICAgIEdQRSA9ICgiR1BFIiwpCiAgICBNQUNfQUREUkVTUyA9ICJNQUNfQUREUkVTUyIKICAgIFVTX0JBTktfTlVNQkVSID0gIlVTX0JBTktfTlVNQkVSIgogICAgSU1FSSA9ICJJTUVJIgogICAgVElUTEUgPSAiVElUTEUiCiAgICBMSUNFTlNFX1BMQVRFID0gIkxJQ0VOU0VfUExBVEUiCiAgICBVU19QQVNTUE9SVCA9ICJVU19QQVNTUE9SVCIKICAgIENVUlJFTkNZID0gIkNVUlJFTkNZIgogICAgUk9VVElOR19OVU1CRVIgPSAiUk9VVElOR19OVU1CRVIiCiAgICBVU19JVElOID0gIlVTX0lUSU4iCiAgICBVU19CQU5LX05VTUJFUiA9ICJVU19CQU5LX05VTUJFUiIKICAgIFVTX0RSSVZFUl9MSUNFTlNFID0gIlVTX0RSSVZFUl9MSUNFTlNFIgogICAgQUdFID0gIkFHRSIKICAgIFBBU1NXT1JEID0gIlBBU1NXT1JEIgogICAgU1dJRlRfQ09ERSA9ICJTV0lGVF9DT0RFIgoKCmNsYXNzIFBhdHRlcm5SZWNvZ25pemVyRmFjdG9yeToKICAgICIiIgogICAgRmFjdG9yeSBmb3IgY3JlYXRpbmcgcGF0dGVybiByZWNvZ25pemVycywgaXQgY2FuIGJlIGV4dGVuZGVkIGluIHRoZSBmdXR1cmUgdG8KICAgIGFkZCBtb3JlIHJlZ2V4IHBhdHRlcm4gZm9yIGRpZmZlcmVudCBlbnRpdGllcy4gRm9yIHRoZSBwYXR0ZXJuIHJlY29nbml6ZXIgdG8gd29yaywKICAgIHdlIG5lZWQgY29uc3RydWN0IGEgbGlzdCBvZiByZWdleCBwYXR0ZXJucyBmb3IgZWFjaCBlbnRpdHkuCiAgICAiIiIKCiAgICBSRUNPR05JWkFCTEVfRU5USVRJRVMgPSB7CiAgICAgICAgIkNSRURJVF9DQVJEIjogW3BhLlBhdHRlcm4oIkNSRURJVF9DQVJEIiwgciJcYig/OlxkWyAtXSo/KXsxMywxNn1cYiIsIDAuNSldLAogICAgICAgICJTU04iOiBbcGEuUGF0dGVybigiU1NOIiwgciJcYlxkezN9LT9cZHsyfS0/XGR7NH1cYiIsIDAuNSldLAogICAgICAgICJQSE9ORSI6IFtwYS5QYXR0ZXJuKCJQSE9ORSIsIHIiXCg/XGR7M31cKT9bLS5cc10/XGR7M31bLS5cc10/XGR7NH0iLCAwLjUpXSwKICAgICAgICAiRU1BSUwiOiBbcGEuUGF0dGVybigiRU1BSUwiLCByIlxTK0BcUysiLCAwLjUpXSwKICAgIH0KCiAgICAjIGNyZWF0ZSBhIGxpc3Qgb2YgcGF0dGVybiByZWNvZ25pemVycwogICAgQGNsYXNzbWV0aG9kCiAgICBkZWYgX2NyZWF0ZV9wYXR0ZXJuX3JlY29nbml6ZXIoY2xzKToKICAgICAgICAiIiIKICAgICAgICBGb3IgZWFjaCBlbnRpdHksIGNyZWF0ZSBhIGxpc3Qgb2YgcGF0dGVybnMgdG8gcmVjb2duaXplIGl0CgogICAgICAgIDpwYXJhbSBjbHM6IFBhdHRlcm5SZWNvZ25pemVyRmFjdG9yeSBjbGFzcwoKICAgICAgICA6cmV0dXJuczogTGlzdCBvZiBwYXR0ZXJuIHJlY29nbml6ZXJzCiAgICAgICAgIiIiCgogICAgICAgICMgRW50aXRpZXMgdG8gcmVjb2duaXplIGFuZCB0aGVpciByZWdleCBwYXR0ZXJucwoKICAgICAgICByZXR1cm4gWwogICAgICAgICAgICBwYS5QYXR0ZXJuUmVjb2duaXplcihzdXBwb3J0ZWRfZW50aXR5PWVudGl0eSwgcGF0dGVybnM9cGF0dGVybikKICAgICAgICAgICAgZm9yIGVudGl0eSwgcGF0dGVybiBpbiBjbHMuUkVDT0dOSVpBQkxFX0VOVElUSUVTLml0ZW1zKCkKICAgICAgICBdCgoKY2xhc3MgQ3VzdG9tU3BhY3lSZWNvZ25pemVyKHBhLkxvY2FsUmVjb2duaXplcik6CiAgICAiIiIKICAgIEN1c3RvbSBTcGFjeSBSZWNvZ25pemVyIGZyb20gUHJlc2lkaW8gQW5hbHl6ZXIgdHJhaW5lZCBvbiBQcml2eSBkYXRhLgogICAgVGhlIHByaXZ5IGRhdGEgaXMgZ2VuZXJhdGVkIHVzaW5nIHRoaXMgaHR0cHM6Ly9naXRodWIuY29tL3BpeGllLWlvL3BpeGllL3RyZWUvbWFpbi9zcmMvZGF0YWdlbi9waWkvcHJpdnkKICAgIEl0IGNhbiBiZSB1c2VkIHRvIHJlY29nbml6ZSBjdXN0b20gZW50aXRpZXMsIFNpbmNlIHdlIHdhbnQgdG8gdXNlIFByZXNpZGlvJ3MgUmVnaXN0cmllcyB0byBnZW5lcmF0ZSBBbmFseXplckVuZ2luZSwKICAgIGl0IGluaGVyaXRzIGZyb20gUHJlc2lkaW8gQW5hbHl6ZXIncyBMb2NhbFJlY29nbml6ZXIgY2xhc3MuCiAgICAiIiIKCiAgICAjIEVudGl0aWVzIHRvIHJlY29nbml6ZQoKICAgIFJFQ09HTklaQUJMRV9FTlRJVElFUyA9IHsKICAgICAgICAiTE9DQVRJT04iLAogICAgICAgICJQRVJTT04iLAogICAgICAgICJOUlAiLAogICAgICAgICJPUkdBTklaQVRJT04iLAogICAgICAgICJEQVRFX1RJTUUiLAogICAgfQoKICAgICMgRGVmYXVsdCBleHBsYW5hdGlvbiBmb3IgdGhpcyByZWNvZ25pemVyCgogICAgX0RFRkFVTFRfRVhQTEFOQVRJT04gPSAoCiAgICAgICAgIklkZW50aWZpZWQgYXMge30gYnkgU3BhY3kncyBOYW1lZCBFbnRpdHkgUmVjb2duaXRpb24gKFByaXZ5LXRyYWluZWQpIgogICAgKQoKICAgICMgTGFiZWwgZ3JvdXBzIHRvIGNoZWNrCgogICAgX0RFRkFVTFRfQ0hFQ0tfTEFCRUxfR1JPVVBTID0gWwogICAgICAgICh7IkxPQ0FUSU9OIn0sIHsiTE9DIiwgIkxPQ0FUSU9OIiwgIlNUUkVFVF9BRERSRVNTIiwgIkNPT1JESU5BVEUifSksCiAgICAgICAgKHsiUEVSU09OIn0sIHsiUEVSIiwgIlBFUlNPTiJ9KSwKICAgICAgICAoeyJOUlAifSwgeyJOT1JQIiwgIk5SUCJ9KSwKICAgICAgICAoeyJPUkdBTklaQVRJT04ifSwgeyJPUkcifSksCiAgICAgICAgKHsiREFURV9USU1FIn0sIHsiREFURV9USU1FIn0pLAogICAgXQoKICAgICMgcHJldHJhaW5lZCBtb2RlbCBmb3IgdGhpcyByZWNvZ25pemVyCgogICAgX0RFRkFVTFRfTU9ERUxfTEFOR1VBR0VTID0gewogICAgICAgICJlbiI6ICJiZWtpL2VuX3NwYWN5X3BpaV9kaXN0aWxiZXJ0IiwKICAgIH0KCiAgICBfREVGQVVMVF9QUkVTSURJT19FUVVJVkFMRU5DRVMgPSB7CiAgICAgICAgIlBFUiI6ICJQRVJTT04iLAogICAgICAgICJMT0MiOiAiTE9DQVRJT04iLAogICAgICAgICJPUkciOiAiT1JHQU5JWkFUSU9OIiwKICAgICAgICAiTlJPUCI6ICJOUlAiLAogICAgICAgICJEQVRFX1RJTUUiOiAiREFURV9USU1FIiwKICAgIH0KCiAgICBkZWYgX19pbml0X18oCiAgICAgICAgc2VsZiwKICAgICAgICBzdXBwb3J0ZWRfbGFuZ3VhZ2U6IHN0ciA9ICJlbiIsCiAgICAgICAgc3VwcG9ydGVkX2VudGl0aWVzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgICAgIGNoZWNrX2xhYmVsX2dyb3VwczogVHVwbGVbU2V0LCBTZXRdID0gTm9uZSwKICAgICAgICBjb250ZXh0OiBMaXN0W3N0cl0gPSBOb25lLAogICAgICAgIG5lcl9zdHJlbmd0aDogZmxvYXQgPSAxLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIFNwYWN5IFJlY29nbml6ZXIuCgogICAgICAgIDpwYXJhbSBzdXBwb3J0ZWRfbGFuZ3VhZ2U6IExhbmd1YWdlIHRvIHVzZSwgZGVmYXVsdCBpcyBFbmdsaXNoCiAgICAgICAgOnBhcmFtIHN1cHBvcnRlZF9lbnRpdGllczogRW50aXRpZXMgdG8gdXNlIGZvciByZWNvZ25pdGlvbgogICAgICAgIDpwYXJhbSBjaGVja19sYWJlbF9ncm91cHM6IExhYmVsIGdyb3VwcyB0byBjaGVjayBmb3IgdGhlIGVudGl0aWVzCiAgICAgICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgQ29udGV4dCB0byB1c2UgaWYgYW55CiAgICAgICAgOnBhcmFtIG5lcl9zdHJlbmd0aDogICAgICAgRGVmYXVsdCBjb25maWRlbmNlIGZvciBORVIgcHJlZGljdGlvbgoKICAgICAgICA6cmV0dXJuczogU3BhY3lSZWNvZ25pemVyIG9iamVjdAogICAgICAgICIiIgoKICAgICAgICAjIERlZmF1bHQgY29uZmlkZW5jZSBmb3IgTkVSIHByZWRpY3Rpb24KICAgICAgICBzZWxmLm5lcl9zdHJlbmd0aCA9IG5lcl9zdHJlbmd0aAoKICAgICAgICBzZWxmLmNoZWNrX2xhYmVsX2dyb3VwcyA9IGNoZWNrX2xhYmVsX2dyb3VwcyBvciBzZWxmLl9ERUZBVUxUX0NIRUNLX0xBQkVMX0dST1VQUwogICAgICAgIHN1cHBvcnRlZF9lbnRpdGllcyA9IHN1cHBvcnRlZF9lbnRpdGllcyBvciBzZWxmLlJFQ09HTklaQUJMRV9FTlRJVElFUwogICAgICAgIHN1cGVyKCkuX19pbml0X18oCiAgICAgICAgICAgIHN1cHBvcnRlZF9lbnRpdGllcz1zdXBwb3J0ZWRfZW50aXRpZXMsCiAgICAgICAgICAgIHN1cHBvcnRlZF9sYW5ndWFnZT1zdXBwb3J0ZWRfbGFuZ3VhZ2UsCiAgICAgICAgKQoKICAgICMgZ2V0IHRoZSBwcmVzaWRpbyBleHBsYW5hdGlvbiBmb3IgdGhlIHJlc3VsdAoKICAgIGRlZiBfYnVpbGRfc3BhY3lfZXhwbGFuYXRpb24oCiAgICAgICAgc2VsZiwgb3JpZ2luYWxfc2NvcmU6IGZsb2F0LCBleHBsYW5hdGlvbjogc3RyCiAgICApIC0+IHBhLkFuYWx5c2lzRXhwbGFuYXRpb246CiAgICAgICAgIiIiCiAgICAgICAgQ3JlYXRlIGV4cGxhbmF0aW9uIGZvciB3aHkgdGhpcyByZXN1bHQgd2FzIGRldGVjdGVkLgoKICAgICAgICA6cGFyYW0gb3JpZ2luYWxfc2NvcmU6IFNjb3JlIGdpdmVuIGJ5IHRoaXMgcmVjb2duaXplcgogICAgICAgIDpwYXJhbSBleHBsYW5hdGlvbjogICAgRXhwbGFuYXRpb24gc3RyaW5nCgogICAgICAgIDpyZXR1cm5zOiBQcmVzaWRpbyBBbmFseXNpc0V4cGxhbmF0aW9uIG9iamVjdAogICAgICAgICIiIgogICAgICAgIGV4cGxhbmF0aW9uID0gcGEuQW5hbHlzaXNFeHBsYW5hdGlvbigKICAgICAgICAgICAgcmVjb2duaXplcj1zZWxmLl9fY2xhc3NfXy5fX25hbWVfXywKICAgICAgICAgICAgb3JpZ2luYWxfc2NvcmU9b3JpZ2luYWxfc2NvcmUsCiAgICAgICAgICAgIHRleHR1YWxfZXhwbGFuYXRpb249ZXhwbGFuYXRpb24sCiAgICAgICAgKQogICAgICAgIHJldHVybiBleHBsYW5hdGlvbgoKICAgICMgbWFpbiBtZXRob2QgZm9yIHRoZSByZWNvZ25pemVyCiAgICBkZWYgYW5hbHl6ZShzZWxmLCB0ZXh0OiBzdHIsIGVudGl0aWVzOiBMaXN0W3N0cl0sIG5scF9hcnRpZmFjdHM9Tm9uZSk6ICAjIG5vcWEgRDEwMgogICAgICAgICIiIgogICAgICAgIEFuYWx5emUgdGV4dCB1c2luZyBTcGFjeS4KCiAgICAgICAgOnBhcmFtIHRleHQ6ICAgICAgICAgIFRleHQgdG8gYW5hbHl6ZQogICAgICAgIDpwYXJhbSBlbnRpdGllczogICAgICBFbnRpdGllcyB0byBhbmFseXplCiAgICAgICAgOnBhcmFtIG5scF9hcnRpZmFjdHM6IE5MUCBhcnRpZmFjdHMgdG8gdXNlCgogICAgICAgIDpyZXR1cm5zOiBMaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgb2JqZWN0cwogICAgICAgICIiIgogICAgICAgIHJlc3VsdHMgPSBbXQogICAgICAgIGlmIG5vdCBubHBfYXJ0aWZhY3RzOgogICAgICAgICAgICBsb2dnZXIud2FybmluZygiU2tpcHBpbmcgU3BhQ3ksIG5scCBhcnRpZmFjdHMgbm90IHByb3ZpZGVkLi4uIikKICAgICAgICAgICAgcmV0dXJuIHJlc3VsdHMKCiAgICAgICAgbmVyX2VudGl0aWVzID0gbmxwX2FydGlmYWN0cy5lbnRpdGllcwoKICAgICAgICAjIHJlY29nbml6ZSB0aGUgc3VwcG9ydGVkIGVudGl0aWVzCiAgICAgICAgZm9yIGVudGl0eSBpbiBlbnRpdGllczoKICAgICAgICAgICAgaWYgZW50aXR5IG5vdCBpbiBzZWxmLnN1cHBvcnRlZF9lbnRpdGllczoKICAgICAgICAgICAgICAgIGNvbnRpbnVlCiAgICAgICAgICAgIGZvciBlbnQgaW4gbmVyX2VudGl0aWVzOgogICAgICAgICAgICAgICAgaWYgbm90IHNlbGYuX19jaGVja19sYWJlbChlbnRpdHksIGVudC5sYWJlbF8sIHNlbGYuY2hlY2tfbGFiZWxfZ3JvdXBzKToKICAgICAgICAgICAgICAgICAgICBjb250aW51ZQoKICAgICAgICAgICAgICAgICMgc3RyaW5nIG9mIHRoZSBleHBsYW5hdGlvbiBzYXlpbmcgdGhlIGVudGl0eSBpcyByZWNvZ25pemVkIGJ5IHNwYWN5CiAgICAgICAgICAgICAgICB0ZXh0dWFsX2V4cGxhbmF0aW9uID0gc2VsZi5fREVGQVVMVF9FWFBMQU5BVElPTi5mb3JtYXQoZW50LmxhYmVsXykKICAgICAgICAgICAgICAgIGV4cGxhbmF0aW9uID0gc2VsZi5fYnVpbGRfc3BhY3lfZXhwbGFuYXRpb24oCiAgICAgICAgICAgICAgICAgICAgc2VsZi5uZXJfc3RyZW5ndGgsIHRleHR1YWxfZXhwbGFuYXRpb24KICAgICAgICAgICAgICAgICkKCiAgICAgICAgICAgICAgICAjIGNyZWF0ZSB0aGUgc3RhbmRhcmQgcmVzdWx0IHdpdGggdGhlIGVudGl0eSwgc3RhcnQsIGVuZCwgc2NvcmUsIGFuZCBleHBsYW5hdGlvbgogICAgICAgICAgICAgICAgc3BhY3lfcmVzdWx0ID0gcGEuUmVjb2duaXplclJlc3VsdCgKICAgICAgICAgICAgICAgICAgICBlbnRpdHlfdHlwZT1lbnRpdHksCiAgICAgICAgICAgICAgICAgICAgc3RhcnQ9ZW50LnN0YXJ0X2NoYXIsCiAgICAgICAgICAgICAgICAgICAgZW5kPWVudC5lbmRfY2hhciwKICAgICAgICAgICAgICAgICAgICBzY29yZT1zZWxmLm5lcl9zdHJlbmd0aCwKICAgICAgICAgICAgICAgICAgICBhbmFseXNpc19leHBsYW5hdGlvbj1leHBsYW5hdGlvbiwKICAgICAgICAgICAgICAgICAgICByZWNvZ25pdGlvbl9tZXRhZGF0YT17CiAgICAgICAgICAgICAgICAgICAgICAgIHBhLlJlY29nbml6ZXJSZXN1bHQuUkVDT0dOSVpFUl9OQU1FX0tFWTogc2VsZi5uYW1lCiAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJlc3VsdHMuYXBwZW5kKHNwYWN5X3Jlc3VsdCkKCiAgICAgICAgcmV0dXJuIHJlc3VsdHMKCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgX19jaGVja19sYWJlbCgKICAgICAgICBlbnRpdHk6IHN0ciwgbGFiZWw6IHN0ciwgY2hlY2tfbGFiZWxfZ3JvdXBzOiBUdXBsZVtTZXQsIFNldF0KICAgICkgLT4gYm9vbDoKICAgICAgICAiIiIKICAgICAgICBDaGVjayBpZiB0aGUgbGFiZWwgaXMgaW4gdGhlIGxhYmVsIGdyb3VwLgoKICAgICAgICA6cGFyYW0gZW50aXR5OiAgICAgICAgICAgICBFbnRpdHkgdG8gY2hlY2sKICAgICAgICA6cGFyYW0gbGFiZWw6ICAgICAgICAgICAgICBMYWJlbCB0byBjaGVjawogICAgICAgIDpwYXJhbSBjaGVja19sYWJlbF9ncm91cHM6IExhYmVsIGdyb3VwcyB0byBjaGVjawoKICAgICAgICA6cmV0dXJuczogVHJ1ZSBpZiB0aGUgbGFiZWwgaXMgaW4gdGhlIGxhYmVsIGdyb3VwLCBGYWxzZSBvdGhlcndpc2UKICAgICAgICAiIiIKICAgICAgICByZXR1cm4gYW55KAogICAgICAgICAgICBlbnRpdHkgaW4gZWdycCBhbmQgbGFiZWwgaW4gbGdycCBmb3IgZWdycCwgbGdycCBpbiBjaGVja19sYWJlbF9ncm91cHMKICAgICAgICApCgoKIyBDbGFzcyB0byB1c2UgRmxhaXIgd2l0aCBQcmVzaWRpbyBhcyBhbiBleHRlcm5hbCByZWNvZ25pemVyLgpjbGFzcyBGbGFpclJlY29nbml6ZXIocGEuRW50aXR5UmVjb2duaXplcik6CiAgICAiIiIKICAgIFdyYXBwZXIgZm9yIGEgZmxhaXIgbW9kZWwsIGlmIG5lZWRlZCB0byBiZSB1c2VkIHdpdGhpbiBQcmVzaWRpbyBBbmFseXplci4KICAgIFRoaXMgaXMgdG8gbWFrZSBzdXJlIHRoZSByZWNvZ25pemVyIGNhbiBiZSByZWdpc3RlcmVkIHdpdGggUHJlc2lkaW8gcmVnaXN0cnkuCiAgICAiIiIKCiAgICBSRUNPR05JWkFCTEVfRU5USVRJRVMgPSB7CiAgICAgICAgIkxPQ0FUSU9OIiwKICAgICAgICAiUEVSU09OIiwKICAgICAgICAiTlJQIiwKICAgICAgICAiR1BFIiwKICAgICAgICAiT1JHQU5JWkFUSU9OIiwKICAgICAgICAiTUFDX0FERFJFU1MiLAogICAgICAgICJVU19CQU5LX05VTUJFUiIsCiAgICAgICAgIklNRUkiLAogICAgICAgICJUSVRMRSIsCiAgICAgICAgIkxJQ0VOU0VfUExBVEUiLAogICAgICAgICJVU19QQVNTUE9SVCIsCiAgICAgICAgIkNVUlJFTkNZIiwKICAgICAgICAiUk9VVElOR19OVU1CRVIiLAogICAgICAgICJVU19JVElOIiwKICAgICAgICAiVVNfQkFOS19OVU1CRVIiLAogICAgICAgICJVU19EUklWRVJfTElDRU5TRSIsCiAgICAgICAgIkFHRSIsCiAgICAgICAgIlBBU1NXT1JEIiwKICAgICAgICAiU1dJRlRfQ09ERSIsCiAgICB9CgogICAgIyBUaGlzIGlzIHVzZWQgdG8gY29uc3RydWN0IHRoZSBleHBsYW5hdGlvbiBmb3IgdGhlIHJlc3VsdAoKICAgIF9ERUZBVUxUX0VYUExBTkFUSU9OID0gIklkZW50aWZpZWQgYXMge30gYnkgRmxhaXIncyBOYW1lZCBFbnRpdHkgUmVjb2duaXRpb24iCgogICAgX0RFRkFVTFRfQ0hFQ0tfTEFCRUxfR1JPVVBTID0gWwogICAgICAgICh7IkxPQ0FUSU9OIn0sIHsiTE9DIiwgIkxPQ0FUSU9OIiwgIlNUUkVFVF9BRERSRVNTIiwgIkNPT1JESU5BVEUifSksCiAgICAgICAgKHsiUEVSU09OIn0sIHsiUEVSIiwgIlBFUlNPTiJ9KSwKICAgICAgICAoeyJOUlAifSwgeyJOT1JQIiwgIk5SUCJ9KSwKICAgICAgICAoeyJHUEUifSwgeyJHUEUifSksCiAgICAgICAgKHsiT1JHQU5JWkFUSU9OIn0sIHsiT1JHIn0pLAogICAgICAgICh7Ik1BQ19BRERSRVNTIn0sIHsiTUFDX0FERFJFU1MifSksCiAgICAgICAgKHsiVVNfQkFOS19OVU1CRVIifSwgeyJVU19CQU5LX05VTUJFUiJ9KSwKICAgICAgICAoeyJJTUVJIn0sIHsiSU1FSSJ9KSwKICAgICAgICAoeyJUSVRMRSJ9LCB7IlRJVExFIn0pLAogICAgICAgICh7IkxJQ0VOU0VfUExBVEUifSwgeyJMSUNFTlNFX1BMQVRFIn0pLAogICAgICAgICh7IlVTX1BBU1NQT1JUIn0sIHsiVVNfUEFTU1BPUlQifSksCiAgICAgICAgKHsiQ1VSUkVOQ1kifSwgeyJDVVJSRU5DWSJ9KSwKICAgICAgICAoeyJST1VUSU5HX05VTUJFUiJ9LCB7IlJPVVRJTkdfTlVNQkVSIn0pLAogICAgICAgICh7IkFHRSJ9LCB7IkFHRSJ9KSwKICAgICAgICAoeyJDVVJSRU5DWSJ9LCB7IkNVUlJFTkNZIn0pLAogICAgICAgICh7IlNXSUZUX0NPREUifSwgeyJTV0lGVF9DT0RFIn0pLAogICAgICAgICh7IlVTX0lUSU4ifSwgeyJVU19JVElOIn0pLAogICAgICAgICh7IlVTX0JBTktfTlVNQkVSIn0sIHsiVVNfQkFOS19OVU1CRVIifSksCiAgICAgICAgKHsiVVNfRFJJVkVSX0xJQ0VOU0UifSwgeyJVU19EUklWRVJfTElDRU5TRSJ9KSwKICAgIF0KCiAgICBfREVGQVVMVF9NT0RFTF9MQU5HVUFHRVMgPSB7CiAgICAgICAgImVuIjogImJla2kvZmxhaXItcGlpLWRpc3RpbGJlcnQiLAogICAgfQoKICAgIF9ERUZBVUxUX1BSRVNJRElPX0VRVUlWQUxFTkNFUyA9IHsKICAgICAgICAiUEVSIjogIlBFUlNPTiIsCiAgICAgICAgIkxPQyI6ICJMT0NBVElPTiIsCiAgICAgICAgIk9SRyI6ICJPUkdBTklaQVRJT04iLAogICAgICAgICJOUk9QIjogIk5SUCIsCiAgICAgICAgIlVSTCI6ICJVUkwiLAogICAgICAgICJVU19JVElOIjogIlVTX0lUSU4iLAogICAgICAgICJVU19QQVNTUE9SVCI6ICJVU19QQVNTUE9SVCIsCiAgICAgICAgIklCQU5fQ09ERSI6ICJJQkFOX0NPREUiLAogICAgICAgICJJUF9BRERSRVNTIjogIklQX0FERFJFU1MiLAogICAgICAgICJFTUFJTF9BRERSRVNTIjogIkVNQUlMIiwKICAgICAgICAiVVNfRFJJVkVSX0xJQ0VOU0UiOiAiVVNfRFJJVkVSX0xJQ0VOU0UiLAogICAgICAgICJVU19CQU5LX05VTUJFUiI6ICJVU19CQU5LX05VTUJFUiIsCiAgICB9CgogICAgZGVmIF9faW5pdF9fKAogICAgICAgIHNlbGYsCiAgICAgICAgc3VwcG9ydGVkX2xhbmd1YWdlOiBzdHIgPSAiZW4iLAogICAgICAgIHN1cHBvcnRlZF9lbnRpdGllczogTGlzdFtzdHJdID0gTm9uZSwKICAgICAgICBjaGVja19sYWJlbF9ncm91cHM6IFR1cGxlW1NldCwgU2V0XSA9IE5vbmUsCiAgICApOgogICAgICAgICIiIgogICAgICAgIEluaXRpYWxpemUgdGhlIEZsYWlyUmVjb2duaXplci4KCiAgICAgICAgOnBhcmFtIHN1cHBvcnRlZF9sYW5ndWFnZTogTGFuZ3VhZ2UgdG8gdXNlCiAgICAgICAgOnBhcmFtIHN1cHBvcnRlZF9lbnRpdGllczogRW50aXRpZXMgdG8gdXNlCiAgICAgICAgOnBhcmFtIGNoZWNrX2xhYmVsX2dyb3VwczogTGFiZWwgZ3JvdXBzIHRvIGNoZWNrCgogICAgICAgIDpyZXR1cm5zOiBGbGFpclJlY29nbml6ZXIgb2JqZWN0CgogICAgICAgICIiIgogICAgICAgIHNlbGYuY2hlY2tfbGFiZWxfZ3JvdXBzID0gY2hlY2tfbGFiZWxfZ3JvdXBzIG9yIHNlbGYuX0RFRkFVTFRfQ0hFQ0tfTEFCRUxfR1JPVVBTCgogICAgICAgIHN1cHBvcnRlZF9lbnRpdGllcyA9IHN1cHBvcnRlZF9lbnRpdGllcyBvciBzZWxmLlJFQ09HTklaQUJMRV9FTlRJVElFUwogICAgICAgIHNlbGYubW9kZWwgPSBmbC5tb2RlbHMuU2VxdWVuY2VUYWdnZXIubG9hZCgKICAgICAgICAgICAgc2VsZi5fREVGQVVMVF9NT0RFTF9MQU5HVUFHRVMuZ2V0KHN1cHBvcnRlZF9sYW5ndWFnZSkKICAgICAgICApCgogICAgICAgIHN1cGVyKCkuX19pbml0X18oCiAgICAgICAgICAgIHN1cHBvcnRlZF9lbnRpdGllcz1zdXBwb3J0ZWRfZW50aXRpZXMsCiAgICAgICAgICAgIHN1cHBvcnRlZF9sYW5ndWFnZT1zdXBwb3J0ZWRfbGFuZ3VhZ2UsCiAgICAgICAgICAgIG5hbWU9IkZsYWlyIEFuYWx5dGljcyIsCiAgICAgICAgKQoKICAgICMgbWFpbiBtZXRob2QgZm9yIHRoZSByZWNvZ25pemVyCiAgICBkZWYgYW5hbHl6ZSgKICAgICAgICBzZWxmLAogICAgICAgIHRleHQ6IHN0ciwKICAgICAgICBlbnRpdGllczogTGlzdFtzdHJdLAogICAgICAgIG5scF9hcnRpZmFjdHM6IHBhLm5scF9lbmdpbmUuTmxwQXJ0aWZhY3RzID0gTm9uZSwKICAgICkgLT4gTGlzdFtwYS5SZWNvZ25pemVyUmVzdWx0XToKICAgICAgICAiIiIKICAgICAgICBBbmFseXplIHRleHQgYW5kIHJldHVybiB0aGUgcmVzdWx0cy4KCiAgICAgICAgOnBhcmFtIHRleHQ6ICAgICAgICAgIFRoZSB0ZXh0IGZvciBhbmFseXNpcy4KICAgICAgICA6cGFyYW0gZW50aXRpZXM6ICAgICAgVGhlIGxpc3Qgb2YgZW50aXRpZXMgdG8gcmVjb2duaXplLgogICAgICAgIDpwYXJhbSBubHBfYXJ0aWZhY3RzOiBOb3QgdXNlZCBieSB0aGlzIHJlY29nbml6ZXIgYnV0IG5lZWRlZCBmb3IgdGhlIGludGVyZmFjZS4KCiAgICAgICAgOnJldHVybnM6IFRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSB0aGUgcmVjb2duaXplZCBGbGFpciBkZXRlY3Rpb25zLgogICAgICAgICIiIgoKICAgICAgICByZXN1bHRzID0gW10KCiAgICAgICAgc2VudGVuY2VzID0gZmwuZGF0YS5TZW50ZW5jZSh0ZXh0KQogICAgICAgIHNlbGYubW9kZWwucHJlZGljdChzZW50ZW5jZXMpCgogICAgICAgICMgSWYgdGhlcmUgYXJlIG5vIHNwZWNpZmljIGxpc3Qgb2YgZW50aXRpZXMsIHdlIHdpbGwgbG9vayBmb3IgYWxsIG9mIGl0LgogICAgICAgIGlmIG5vdCBlbnRpdGllczoKICAgICAgICAgICAgZW50aXRpZXMgPSBzZWxmLnN1cHBvcnRlZF9lbnRpdGllcwoKICAgICAgICAjIEdvIG92ZXIgdGhlIGVudGl0aWVzIGFuZCBjaGVjayBpZiB0aGV5IGFyZSBpbiB0aGUgc3VwcG9ydGVkIGVudGl0aWVzIGxpc3QuCiAgICAgICAgZm9yIGVudGl0eSBpbiBlbnRpdGllczoKICAgICAgICAgICAgaWYgZW50aXR5IG5vdCBpbiBzZWxmLnN1cHBvcnRlZF9lbnRpdGllczoKICAgICAgICAgICAgICAgIGNvbnRpbnVlCgogICAgICAgICAgICAjIEdvIG92ZXIgdGhlIHNlbnRlbmNlcyBhbmQgY2hlY2sgaWYgdGhlIGVudGl0eSBpcyBpbiB0aGUgc2VudGVuY2UuCiAgICAgICAgICAgIGZvciBlbnQgaW4gc2VudGVuY2VzLmdldF9zcGFucygibmVyIik6CiAgICAgICAgICAgICAgICBpZiBub3Qgc2VsZi5fX2NoZWNrX2xhYmVsKAogICAgICAgICAgICAgICAgICAgIGVudGl0eSwgZW50LmxhYmVsc1swXS52YWx1ZSwgc2VsZi5jaGVja19sYWJlbF9ncm91cHMKICAgICAgICAgICAgICAgICk6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKCiAgICAgICAgICAgICAgICAjIElmIHRoZSBlbnRpdHkgaXMgaW4gdGhlIHNlbnRlbmNlLCB3ZSB3aWxsIGFkZCBpdCB0byB0aGUgcmVzdWx0cy4KICAgICAgICAgICAgICAgIHRleHR1YWxfZXhwbGFuYXRpb24gPSBzZWxmLl9ERUZBVUxUX0VYUExBTkFUSU9OLmZvcm1hdCgKICAgICAgICAgICAgICAgICAgICBlbnQubGFiZWxzWzBdLnZhbHVlCiAgICAgICAgICAgICAgICApCgogICAgICAgICAgICAgICAgIyBCdWlsZCB0aGUgZXhwbGFuYXRpb24gZm9yIHRoZSByZXN1bHQKICAgICAgICAgICAgICAgIGV4cGxhbmF0aW9uID0gc2VsZi5fYnVpbGRfZmxhaXJfZXhwbGFuYXRpb24oCiAgICAgICAgICAgICAgICAgICAgcm91bmQoZW50LnNjb3JlLCAyKSwgdGV4dHVhbF9leHBsYW5hdGlvbgogICAgICAgICAgICAgICAgKQoKICAgICAgICAgICAgICAgIGZsYWlyX3Jlc3VsdCA9IHNlbGYuX2NvbnZlcnRfdG9fcmVjb2duaXplcl9yZXN1bHQoZW50LCBleHBsYW5hdGlvbikKCiAgICAgICAgICAgICAgICByZXN1bHRzLmFwcGVuZChmbGFpcl9yZXN1bHQpCgogICAgICAgIHJldHVybiByZXN1bHRzCgogICAgZGVmIF9jb252ZXJ0X3RvX3JlY29nbml6ZXJfcmVzdWx0KAogICAgICAgIHNlbGYsIGVudGl0eTogZmwuZGF0YS5TcGFuLCBleHBsYW5hdGlvbjogc3RyCiAgICApIC0+IHBhLlJlY29nbml6ZXJSZXN1bHQ6CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCBGbGFpciByZXN1bHQgdG8gUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdC4KCiAgICAgICAgOnBhcmFtIGVudGl0eTogICAgICBGbGFpciBlbnRpdHkgb2YgU3BhbgogICAgICAgIDpwYXJhbSBleHBsYW5hdGlvbjogUHJlc2lkaW8gQW5hbHlzaXNFeHBsYW5hdGlvbgoKICAgICAgICA6cmV0dXJuczogUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdAogICAgICAgICIiIgoKICAgICAgICAjIENvbnZlcnQgdGhlIGVudGl0eSB0eXBlIHRvIFByZXNpZGlvIGVudGl0eSB0eXBlCiAgICAgICAgZW50aXR5X3R5cGUgPSBzZWxmLl9ERUZBVUxUX1BSRVNJRElPX0VRVUlWQUxFTkNFUy5nZXQoZW50aXR5LnRhZywgZW50aXR5LnRhZykKCiAgICAgICAgIyBDb252ZXJ0IHRoZSBzY29yZSB0byBQcmVzaWRpbyBzY29yZQogICAgICAgIGZsYWlyX3Njb3JlID0gcm91bmQoZW50aXR5LnNjb3JlLCAyKQoKICAgICAgICAjIENyZWF0ZSB0aGUgUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdCBmcm9tIHRoZSBGbGFpciBlbnRpdHkKICAgICAgICBmbGFpcl9yZXN1bHRzID0gcGEuUmVjb2duaXplclJlc3VsdCgKICAgICAgICAgICAgZW50aXR5X3R5cGU9ZW50aXR5X3R5cGUsCiAgICAgICAgICAgIHN0YXJ0PWVudGl0eS5zdGFydF9wb3NpdGlvbiwKICAgICAgICAgICAgZW5kPWVudGl0eS5lbmRfcG9zaXRpb24sCiAgICAgICAgICAgIHNjb3JlPWZsYWlyX3Njb3JlLAogICAgICAgICAgICBhbmFseXNpc19leHBsYW5hdGlvbj1leHBsYW5hdGlvbiwKICAgICAgICApCgogICAgICAgIHJldHVybiBmbGFpcl9yZXN1bHRzCgogICAgZGVmIF9idWlsZF9mbGFpcl9leHBsYW5hdGlvbigKICAgICAgICBzZWxmLCBvcmlnaW5hbF9zY29yZTogZmxvYXQsIGV4cGxhbmF0aW9uOiBzdHIKICAgICkgLT4gcGEuQW5hbHlzaXNFeHBsYW5hdGlvbjoKICAgICAgICAiIiIKICAgICAgICBDcmVhdGUgZXhwbGFuYXRpb24gZm9yIHdoeSB0aGlzIHJlc3VsdCB3YXMgZGV0ZWN0ZWQuCgogICAgICAgIDpwYXJhbSBvcmlnaW5hbF9zY29yZTogU2NvcmUgZ2l2ZW4gYnkgdGhpcyByZWNvZ25pemVyCiAgICAgICAgOnBhcmFtIGV4cGxhbmF0aW9uOiAgICBFeHBsYW5hdGlvbiBzdHJpbmcKCiAgICAgICAgOnJldHVybnM6IFByZXNpZGlvIEFuYWx5c2lzRXhwbGFuYXRpb24KICAgICAgICAiIiIKCiAgICAgICAgIyBDcmVhdGUgdGhlIFByZXNpZGlvIEFuYWx5c2lzRXhwbGFuYXRpb24gZm9yIHRoZSByZXN1bHQKICAgICAgICBleHBsYW5hdGlvbiA9IHBhLkFuYWx5c2lzRXhwbGFuYXRpb24oCiAgICAgICAgICAgIHJlY29nbml6ZXI9c2VsZi5fX2NsYXNzX18uX19uYW1lX18sCiAgICAgICAgICAgIG9yaWdpbmFsX3Njb3JlPW9yaWdpbmFsX3Njb3JlLAogICAgICAgICAgICB0ZXh0dWFsX2V4cGxhbmF0aW9uPWV4cGxhbmF0aW9uLAogICAgICAgICkKICAgICAgICByZXR1cm4gZXhwbGFuYXRpb24KCiAgICAjIHNhbml0eSBjaGVjayBvZiB0aGUgZW50aXR5IGFuZCBsYWJlbCBiZWZvcmUgcmVjb2duaXRpb24KICAgIEBzdGF0aWNtZXRob2QKICAgIGRlZiBfX2NoZWNrX2xhYmVsKAogICAgICAgIGVudGl0eTogc3RyLCBsYWJlbDogc3RyLCBjaGVja19sYWJlbF9ncm91cHM6IFR1cGxlW1NldCwgU2V0XQogICAgKSAtPiBib29sOgogICAgICAgIHJldHVybiBhbnkoCiAgICAgICAgICAgIGVudGl0eSBpbiBlZ3JwIGFuZCBsYWJlbCBpbiBsZ3JwIGZvciBlZ3JwLCBsZ3JwIGluIGNoZWNrX2xhYmVsX2dyb3VwcwogICAgICAgICkKCgojIGdldCB0aGUgYW5hbHl6ZXIgZW5naW5lIGJhc2VkIG9uIHRoZSBtb2RlbApkZWYgX2dldF9hbmFseXplcl9lbmdpbmUoCiAgICBtb2RlbDogc3RyID0gTm9uZSwgZW50aXRpZXM6IExpc3Rbc3RyXSA9IE5vbmUKKSAtPiBwYS5BbmFseXplckVuZ2luZToKICAgICIiIgogICAgUmV0dXJuIHBhLkFuYWx5emVyRW5naW5lLgoKICAgIDpwYXJhbSBtb2RlbDogVGhlIG1vZGVsIHRvIHVzZS4gQ2FuIGJlICJzcGFjeSIsICJmbGFpciIsICJwYXR0ZXJuIiBvciAid2hvbGUiLgogICAgOnBhcmFtIGVudGl0aWVzOiBUaGUgbGlzdCBvZiBlbnRpdGllcyB0byB1c2UuCgogICAgOnJldHVybnM6IHBhLkFuYWx5emVyRW5naW5lCiAgICAiIiIKICAgICMgcmVjb2duaXplciByZWdpc3RyeSB0aGF0IGNhbiBzdG9yZSBtdWx0aXBsZSByZWNvZ25pemVycwogICAgcmVnaXN0cnkgPSBwYS5SZWNvZ25pemVyUmVnaXN0cnkoKQogICAgaWYgbW9kZWwgPT0gTW9kZWxzLlNQQUNZOgogICAgICAgICMgY3VzdG9tIHNwYWN5IHJlY29nbml6ZXIKICAgICAgICBzcGFjeV9yZWNvZ25pemVyID0gQ3VzdG9tU3BhY3lSZWNvZ25pemVyKCkKICAgICAgICAjIGFkZCB0aGUgY3VzdG9tIGJ1aWxkIHNwYWN5IHJlY29nbml6ZXIKICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihzcGFjeV9yZWNvZ25pemVyKQogICAgZWxpZiBtb2RlbCA9PSBNb2RlbHMuRkxBSVI6CiAgICAgICAgIyBwcmUtdHJhaW5lZCBmbGFpciByZWNvZ25pemVyCiAgICAgICAgZmxhaXJfcmVjb2duaXplciA9IEZsYWlyUmVjb2duaXplcigpCiAgICAgICAgIyBhZGQgdGhlIGN1c3RvbSBidWlsZCBmbGFpciByZWNvZ25pemVyCiAgICAgICAgcmVnaXN0cnkuYWRkX3JlY29nbml6ZXIoZmxhaXJfcmVjb2duaXplcikKICAgIGVsaWYgbW9kZWwgPT0gTW9kZWxzLlBBVFRFUk46CiAgICAgICAgIyBhZGQgdGhlIHBhdHRlcm4gcmVjb2duaXplcgogICAgICAgIHBhdHRlcm5fcmVjb2duaXplcl9mYWN0b3J5ID0gUGF0dGVyblJlY29nbml6ZXJGYWN0b3J5KCkKICAgICAgICBmb3IgcmVjb2duaXplciBpbiBwYXR0ZXJuX3JlY29nbml6ZXJfZmFjdG9yeS5fY3JlYXRlX3BhdHRlcm5fcmVjb2duaXplcigpOgogICAgICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihyZWNvZ25pemVyKQogICAgZWxpZiBtb2RlbCA9PSBNb2RlbHMuV0hPTEU6CiAgICAgICAgc3BhY3lfcmVjb2duaXplciA9IEN1c3RvbVNwYWN5UmVjb2duaXplcigpCiAgICAgICAgZmxhaXJfcmVjb2duaXplciA9IEZsYWlyUmVjb2duaXplcigpCiAgICAgICAgcmVnaXN0cnkuYWRkX3JlY29nbml6ZXIoc3BhY3lfcmVjb2duaXplcikKICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihmbGFpcl9yZWNvZ25pemVyKQogICAgICAgICMgYWRkIHRoZSBwYXR0ZXJuIHJlY29nbml6ZXIKICAgICAgICBwYXR0ZXJuX3JlY29nbml6ZXJfZmFjdG9yeSA9IFBhdHRlcm5SZWNvZ25pemVyRmFjdG9yeSgpCiAgICAgICAgZm9yIHJlY29nbml6ZXIgaW4gcGF0dGVybl9yZWNvZ25pemVyX2ZhY3RvcnkuX2NyZWF0ZV9wYXR0ZXJuX3JlY29nbml6ZXIoKToKICAgICAgICAgICAgcmVnaXN0cnkuYWRkX3JlY29nbml6ZXIocmVjb2duaXplcikKICAgIGVsaWYgbm90IG1vZGVsIGFuZCBlbnRpdGllczoKICAgICAgICBpZiBzZXQoZW50aXRpZXMpICYgQ3VzdG9tU3BhY3lSZWNvZ25pemVyLlJFQ09HTklaQUJMRV9FTlRJVElFUzoKICAgICAgICAgICAgc3BhY3lfcmVjb2duaXplciA9IEN1c3RvbVNwYWN5UmVjb2duaXplcigpCiAgICAgICAgICAgIHJlZ2lzdHJ5LmFkZF9yZWNvZ25pemVyKHNwYWN5X3JlY29nbml6ZXIpCiAgICAgICAgaWYgc2V0KGVudGl0aWVzKSAmIEZsYWlyUmVjb2duaXplci5SRUNPR05JWkFCTEVfRU5USVRJRVM6CiAgICAgICAgICAgIGZsYWlyX3JlY29nbml6ZXIgPSBGbGFpclJlY29nbml6ZXIoKQogICAgICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihmbGFpcl9yZWNvZ25pemVyKQogICAgICAgICMgYWRkIHRoZSBwYXR0ZXJuIHJlY29nbml6ZXIKICAgICAgICBpZiBzZXQoZW50aXRpZXMpICYgKHNldChQYXR0ZXJuUmVjb2duaXplckZhY3RvcnkuUkVDT0dOSVpBQkxFX0VOVElUSUVTLmtleXMoKSkpOgogICAgICAgICAgICBwYXR0ZXJuX3JlY29nbml6ZXJfZmFjdG9yeSA9IFBhdHRlcm5SZWNvZ25pemVyRmFjdG9yeSgpCiAgICAgICAgICAgIGZvciByZWNvZ25pemVyIGluIHBhdHRlcm5fcmVjb2duaXplcl9mYWN0b3J5Ll9jcmVhdGVfcGF0dGVybl9yZWNvZ25pemVyKCk6CiAgICAgICAgICAgICAgICByZWdpc3RyeS5hZGRfcmVjb2duaXplcihyZWNvZ25pemVyKQogICAgZWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmImFyZ3VtZW50IG9mIG1vZGVsIGFuZCBlbnRpdGllcyBjYW4gbm90IGJlIE5vbmUgYXQgdGhlIHNhbWUgdGltZSIKICAgICAgICApCiAgICBhbmFseXplciA9IHBhLkFuYWx5emVyRW5naW5lKAogICAgICAgIHJlZ2lzdHJ5PXJlZ2lzdHJ5LAogICAgICAgIHN1cHBvcnRlZF9sYW5ndWFnZXM9WyJlbiJdLAogICAgKQoKICAgIHN1cHBvcnRlZF9lbnRpdGllcyA9IGFuYWx5emVyLmdldF9zdXBwb3J0ZWRfZW50aXRpZXMoKQoKICAgIGlmIGVudGl0aWVzIGFuZCBub3QgYWxsKGl0ZW0gaW4gc3VwcG9ydGVkX2VudGl0aWVzIGZvciBpdGVtIGluIGVudGl0aWVzKToKICAgICAgICBub3Rfc3VwcG9ydGVkX2VudGl0aWVzID0gWwogICAgICAgICAgICBpdGVtIGZvciBpdGVtIGluIGVudGl0aWVzIGlmIGl0ZW0gbm90IGluIHN1cHBvcnRlZF9lbnRpdGllcwogICAgICAgIF0KICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlRoZSBjdXJyZW50IG1vZGVsIHttb2RlbH0gZG9lc24ndCBzdXBwb3J0IHRoZSBmb2xsb3dpbmcgZW50aXRpZXM6IHtub3Rfc3VwcG9ydGVkX2VudGl0aWVzfS4gIgogICAgICAgICAgICBmIlN1cHBvcnRlZCBlbnRpdGllcyBhcmU6IHtzdXBwb3J0ZWRfZW50aXRpZXN9IgogICAgICAgICkKICAgIHJldHVybiBhbmFseXplcgoKCmRlZiBfZ2V0X2Fub255bWl6ZXJfZW5naW5lKCkgLT4gcHJlX2Fub3ltaXplci5Bbm9ueW1pemVyRW5naW5lOgogICAgIiIiCiAgICBSZXR1cm4gQW5vbnltaXplckVuZ2luZS4KCiAgICA6cmV0dXJuczogVGhlIEFub255bWl6ZXJFbmdpbmUuCiAgICAiIiIKICAgIHJldHVybiBwcmVfYW5veW1pemVyLkFub255bWl6ZXJFbmdpbmUoKQoKCmRlZiBfYW5vbnltaXplKAogICAgdGV4dDogc3RyLAogICAgYW5hbHl6ZV9yZXN1bHRzOiBMaXN0W3BhLlJlY29nbml6ZXJSZXN1bHRdLAogICAgZW50aXR5X29wZXJhdG9yX21hcDogZGljdCA9IE5vbmUsCiAgICBpc19mdWxsX3RleHQ6IGJvb2wgPSBUcnVlLAopIC0+IHN0cjoKICAgICIiIgogICAgQW5vbnltaXplIGlkZW50aWZpZWQgaW5wdXQgdXNpbmcgUHJlc2lkaW8gQWJvbnltaXplci4KCiAgICA6cGFyYW0gdGV4dDogICAgICAgICAgICAgICAgVGhlIHRleHQgZm9yIGFuYWx5c2lzLgogICAgOnBhcmFtIGFuYWx5emVfcmVzdWx0czogICAgIFRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbQogICAgOnBhcmFtIGVudGl0eV9vcGVyYXRvcl9tYXA6IFRoZSBlbnRpdHlfb3BlcmF0b3JfbWFwIGlzIGEgZGljdGlvbmFyeSB0aGF0IG1hcHMgZW50aXR5IHRvIG9wZXJhdG9yIG5hbWUgYW5kIG9wZXJhdG9yIHBhcmFtcy4KICAgIDpwYXJhbSBpc19mdWxsX3RleHQ6ICAgICAgICBXaGV0aGVyIHRoZSB0ZXh0IGlzIGZ1bGwgdGV4dCBvciBub3QuCgogICAgOnJldHVybnM6IFRoZSBhbm9ueW1pemVkIHRleHQuCiAgICAiIiIKICAgIGlmIG5vdCB0ZXh0OgogICAgICAgIHJldHVybiAiIgoKICAgIGFub255bWl6ZXJfZW5naW5lID0gX2dldF9hbm9ueW1pemVyX2VuZ2luZSgpCiAgICBpZiBub3QgZW50aXR5X29wZXJhdG9yX21hcDoKICAgICAgICBvcGVyYXRvcnMgPSBOb25lCiAgICBlbHNlOgogICAgICAgICMgQ3JlYXRlIE9wZXJhdG9yQ29uZmlnIGJhc2VkIG9uIHRoZSBlbnRpdHlfb3BlcmF0b3JfbWFwCiAgICAgICAgb3BlcmF0b3JzID0gewogICAgICAgICAgICBlbnRpdHk6IE9wZXJhdG9yQ29uZmlnKG9wZXJhdG9yX25hbWUsIG9wZXJhdG9yX3BhcmFtcykKICAgICAgICAgICAgZm9yIGVudGl0eSwgKG9wZXJhdG9yX25hbWUsIG9wZXJhdG9yX3BhcmFtcykgaW4gZW50aXR5X29wZXJhdG9yX21hcC5pdGVtcygpCiAgICAgICAgfQoKICAgIGlmIGlzX2Z1bGxfdGV4dDoKICAgICAgICAjIEFub255bWl6ZSB0aGUgZW50aXJlIHRleHQKICAgICAgICByZXR1cm4gYW5vbnltaXplcl9lbmdpbmUuYW5vbnltaXplKAogICAgICAgICAgICB0ZXh0PXRleHQsIGFuYWx5emVyX3Jlc3VsdHM9YW5hbHl6ZV9yZXN1bHRzLCBvcGVyYXRvcnM9b3BlcmF0b3JzCiAgICAgICAgKS50ZXh0CiAgICAjIFRva2VuaXplIHRoZSB0ZXh0IHRvIHNlbnRlbmNlcwogICAgc2VudGVuY2VzID0gbmx0ay5zZW50X3Rva2VuaXplKHRleHQpCiAgICBhbm9ueW1pemVkX3NlbnRlbmNlcyA9IFtdCiAgICBjdXJyZW50X2lkeCA9IDAKCiAgICAjIEZpbmQgdGhlIHNlbnRlbmNlIHRoYXQgaGFzIHBpaSBlbnRpdHkKICAgIGZvciBzZW50ZW5jZSBpbiBzZW50ZW5jZXM6CiAgICAgICAgc3RhcnRfaWR4ID0gY3VycmVudF9pZHgKICAgICAgICBlbmRfaWR4ID0gc3RhcnRfaWR4ICsgbGVuKHNlbnRlbmNlKQoKICAgICAgICAjIEdldCB0aGUgZW50aXRpZXMgdGhhdCBhcmUgaW4gdGhlIHNlbnRlbmNlLCB1cGRhdGUgaHRlIHN0YXJ0X2lkeCBhbmQgZW5kX2lkeAogICAgICAgIHNlbnRlbmNlX3Jlc3VsdHMgPSBbCiAgICAgICAgICAgIHBhLlJlY29nbml6ZXJSZXN1bHQoCiAgICAgICAgICAgICAgICByZXN1bHQuZW50aXR5X3R5cGUsCiAgICAgICAgICAgICAgICBzdGFydD1yZXN1bHQuc3RhcnQgLSBzdGFydF9pZHgsCiAgICAgICAgICAgICAgICBlbmQ9cmVzdWx0LmVuZCAtIHN0YXJ0X2lkeCwKICAgICAgICAgICAgICAgIHNjb3JlPXJlc3VsdC5zY29yZSwKICAgICAgICAgICAgKQogICAgICAgICAgICBmb3IgcmVzdWx0IGluIGFuYWx5emVfcmVzdWx0cwogICAgICAgICAgICBpZiByZXN1bHQuc3RhcnQgPj0gc3RhcnRfaWR4IGFuZCByZXN1bHQuZW5kIDw9IGVuZF9pZHgKICAgICAgICBdCgogICAgICAgICMgSWYgUElJIGlzIGRldGVjdGVkCiAgICAgICAgaWYgc2VudGVuY2VfcmVzdWx0czoKICAgICAgICAgICAgYW5vbnltaXplZF9zZW50ZW5jZSA9IGFub255bWl6ZXJfZW5naW5lLmFub255bWl6ZSgKICAgICAgICAgICAgICAgIHRleHQ9c2VudGVuY2UsIGFuYWx5emVyX3Jlc3VsdHM9c2VudGVuY2VfcmVzdWx0cywgb3BlcmF0b3JzPW9wZXJhdG9ycwogICAgICAgICAgICApLnRleHQKICAgICAgICAgICAgYW5vbnltaXplZF9zZW50ZW5jZXMuYXBwZW5kKGFub255bWl6ZWRfc2VudGVuY2UpCgogICAgICAgIGN1cnJlbnRfaWR4ID0gZW5kX2lkeAoKICAgIHJldHVybiAiICIuam9pbihhbm9ueW1pemVkX3NlbnRlbmNlcykKCgpkZWYgX2dldF90b2tlbnMoCiAgICB0ZXh0OiBzdHIsIGFuYWx5emVfcmVzdWx0czogTGlzdFtwYS5SZWNvZ25pemVyUmVzdWx0XSwgaXNfZnVsbDogYm9vbCA9IFRydWUKKSAtPiBMaXN0W3N0cl06CiAgICAiIiIKICAgIEdldCB0aGUgZnVsbCB0b2tlbnMgb3Igb25seSBjb250YWlucyB0aGUgZW50aXRpZXMgdGhhdCBjYW4gZm9ybSBhIHNlbnRlbmNlLgoKICAgIDpwYXJhbSB0ZXh0OiAgICAgICAgICAgIFRoZSB0ZXh0IGZvciBhbmFseXNpcy4KICAgIDpwYXJhbSBhbmFseXplX3Jlc3VsdHM6IFRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbQogICAgOnBhcmFtIGlzX2Z1bGw6ICAgICAgICAgV2hldGhlciByZXR1cm4gZnVsbCB0b2tlbnMgb3IganVzdCB0aGUgdG9rZW5zIHRoYXQgb25seSBjb250YWlucyB0aGUgZW50aXRpZXMgdGhhdCBjYW4gZm9ybSBhIHNlbnRlbmNlLgoKICAgIDpyZXR1cm5zOiBUaGUgdG9rZW5zLgogICAgIiIiCgogICAgdG9rZW5zID0gW10KICAgICMgc29ydCBieSBzdGFydCBpbmRleAogICAgcmVzdWx0cyA9IHNvcnRlZChhbmFseXplX3Jlc3VsdHMsIGtleT1sYW1iZGEgeDogeC5zdGFydCkKICAgIGZvciBpLCByZXMgaW4gZW51bWVyYXRlKHJlc3VsdHMpOgogICAgICAgIGlmIGkgPT0gMDoKICAgICAgICAgICAgdG9rZW5zLmFwcGVuZCh0ZXh0WzogcmVzLnN0YXJ0XSkKCiAgICAgICAgIyBhcHBlbmQgZW50aXR5IHRleHQgYW5kIGVudGl0eSB0eXBlCiAgICAgICAgdG9rZW5zLmFwcGVuZCgodGV4dFtyZXMuc3RhcnQgOiByZXMuZW5kXSwgcmVzLmVudGl0eV90eXBlKSkKCiAgICAgICAgIyBpZiBhbm90aGVyIGVudGl0eSBjb21pbmcgaS5lLiB3ZSdyZSBub3QgYXQgdGhlIGxhc3QgcmVzdWx0cyBlbGVtZW50LAogICAgICAgICMgYWRkIHRleHQgdXAgdG8gbmV4dCBlbnRpdHkKICAgICAgICBpZiBpICE9IGxlbihyZXN1bHRzKSAtIDE6CiAgICAgICAgICAgIHRva2Vucy5hcHBlbmQodGV4dFtyZXMuZW5kIDogcmVzdWx0c1tpICsgMV0uc3RhcnRdKQogICAgICAgICMgaWYgbm8gbW9yZSBlbnRpdGllcyBjb21pbmcsIGFkZCBhbGwgcmVtYWluaW5nIHRleHQKICAgICAgICBlbHNlOgogICAgICAgICAgICB0b2tlbnMuYXBwZW5kKHRleHRbcmVzLmVuZCA6XSkKCiAgICAjIGdldCB0aGUgdG9rZW5zIHRoYXQgb25seSBjb250YWlucyB0aGUgZW50aXRpZXMgdGhhdCBjYW4gZm9ybSBhIHNlbnRlbmNlCiAgICBwYXJ0X2Fubm9udGF0ZWRfdG9rZW5zID0gW10KICAgIGlmIG5vdCBpc19mdWxsOgogICAgICAgIGxhc3RfZW5kX3NlbnRlbmNlID0gMAogICAgICAgIGZvciBpLCB0b2tlbiBpbiBlbnVtZXJhdGUodG9rZW5zKToKICAgICAgICAgICAgaWYgYW55KGl0ZW0gaW4gdG9rZW4gZm9yIGl0ZW0gaW4gWyIuIiwgIiEiLCAiPyJdKSBhbmQgYW55KAogICAgICAgICAgICAgICAgdHlwZShpdGVtKSBpcyB0dXBsZSBmb3IgaXRlbSBpbiB0b2tlbnNbbGFzdF9lbmRfc2VudGVuY2U6aV0KICAgICAgICAgICAgKToKICAgICAgICAgICAgICAgIHBhcnRfYW5ub250YXRlZF90b2tlbnMuYXBwZW5kKHRva2Vuc1tsYXN0X2VuZF9zZW50ZW5jZTppXSkKICAgICAgICAgICAgICAgIGxhc3RfZW5kX3NlbnRlbmNlID0gaQogICAgICAgIHJldHVybiBwYXJ0X2Fubm9udGF0ZWRfdG9rZW5zCiAgICByZXR1cm4gdG9rZW5zCgoKZGVmIF9hbm5vdGF0ZSgKICAgIHRleHQ6IHN0ciwgc3RfYW5hbHl6ZV9yZXN1bHRzOiBMaXN0W3BhLlJlY29nbml6ZXJSZXN1bHRdLCBpc19mdWxsX2h0bWw6IGJvb2wgPSBUcnVlCikgLT4gTGlzdFtzdHJdOgogICAgIiIiCiAgICBBbm5vdGF0ZSBpZGVudGlmaWVkIGlucHV0IHVzaW5nIFByZXNpZGlvIEFub255bWl6ZXIuCgogICAgOnBhcmFtIHRleHQ6ICAgICAgICAgICAgICAgVGhlIHRleHQgZm9yIGFuYWx5c2lzLgogICAgOnBhcmFtIHN0X2FuYWx5emVfcmVzdWx0czogVGhlIGxpc3Qgb2YgUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdCBjb25zdHJ1Y3RlZCBmcm9tIGFuYWx5c2lzLgogICAgOnBhcmFtIGlzX2Z1bGxfaHRtbDogICAgICAgV2hldGhlciBnZW5lcmF0ZSBmdWxsIGh0bWwgb3Igbm90LgoKICAgIDpyZXR1cm5zOiBUaGUgbGlzdCBvZiB0b2tlbnMgd2l0aCB0aGUgaWRlbnRpZmllZCBlbnRpdGllcy4KCiAgICAiIiIKICAgIHJldHVybiBfZ2V0X3Rva2Vucyh0ZXh0LCBzdF9hbmFseXplX3Jlc3VsdHMsIGlzX2Z1bGxfaHRtbCkKCgpkZWYgX3Byb2Nlc3MoCiAgICB0ZXh0OiBzdHIsCiAgICBtb2RlbDogcGEuQW5hbHl6ZXJFbmdpbmUsCiAgICBzY29yZV90aHJlc2hvbGQ6IGZsb2F0LAogICAgZW50aXRpZXM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBlbnRpdGllc19vcGVyYXRvcl9tYXA6IGRpY3QgPSBOb25lLAogICAgaXNfZnVsbF90ZXh0OiBib29sID0gVHJ1ZSwKKSAtPiBUdXBsZVtzdHIsIGxpc3RdOgogICAgIiIiCiAgICBQcm9jZXNzIHRoZSB0ZXh0IG9mIHN0ciB1c2luZyB0aGUgbW9kZWwuCgogICAgOnBhcmFtIHRleHQ6ICAgICAgICAgICAgICAgICAgVGV4dCB0byBwcm9jZXNzCiAgICA6cGFyYW0gbW9kZWw6ICAgICAgICAgICAgICAgICBNb2RlbCB0byB1c2UgZm9yIHByb2Nlc3NpbmcKICAgIDpwYXJhbSBlbnRpdGllczogICAgICAgICAgICAgIEVudGl0aWVzIHRvIHJlY29nbml6ZQogICAgOnBhcmFtIGVudGl0aWVzX29wZXJhdG9yX21hcDogVGhlIGVudGl0eV9vcGVyYXRvcl9tYXAgaXMgYSBkaWN0aW9uYXJ5IHRoYXQgbWFwcyBlbnRpdHkgdG8gb3BlcmF0b3IgbmFtZSBhbmQgb3BlcmF0b3IgcGFyYW1zLgogICAgOnBhcmFtIHNjb3JlX3RocmVzaG9sZDogICAgICAgVGhlIHNjb3JlIHRocmVzaG9sZCB0byB1c2UgZm9yIHJlY29nbml0aW9uCiAgICA6cGFyYW0gaXNfZnVsbF90ZXh0OiAgICAgICAgICBXaGV0aGVyIHRvIHJldHVybiB0aGUgZnVsbCB0ZXh0IG9yIGp1c3QgdGhlIGFubm90YXRlZCB0ZXh0CgogICAgOnJldHVybnM6IEEgdHVwbGUgb2Y6CgogICAgICAgICAgICAgICogdGhlIGFub255bWl6ZWQgdGV4dAogICAgICAgICAgICAgICogdGhlIGxpc3Qgb2YgUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdCBjb25zdHJ1Y3RlZCBmcm9tIGFuYWx5c2lzCiAgICAiIiIKCiAgICAjIGdldCB0aGUgYW5hbHl6ZXIgZW5naW5lCiAgICBhbmFseXplciA9IG1vZGVsCgogICAgIyBhbmFseXplIHRoZSB0ZXh0IHRoYXQgY2FuIGJlIHVzZWQgZm9yIGFub255bWl6YXRpb24KICAgIHJlc3VsdHMgPSBhbmFseXplci5hbmFseXplKAogICAgICAgIHRleHQ9dGV4dCwKICAgICAgICBsYW5ndWFnZT0iZW4iLAogICAgICAgIGVudGl0aWVzPWVudGl0aWVzLAogICAgICAgIHNjb3JlX3RocmVzaG9sZD1zY29yZV90aHJlc2hvbGQsCiAgICAgICAgcmV0dXJuX2RlY2lzaW9uX3Byb2Nlc3M9VHJ1ZSwKICAgICkKCiAgICAjIGFub255bWl6ZSB0aGUgdGV4dCwgcmVwbGFjZSB0aGUgcGlpIGVudGl0aWVzIHdpdGggdGhlIGxhYmVscwogICAgYW5vbnltaXplZF90ZXh0ID0gX2Fub255bWl6ZSh0ZXh0LCByZXN1bHRzLCBlbnRpdGllc19vcGVyYXRvcl9tYXAsIGlzX2Z1bGxfdGV4dCkKCiAgICByZXR1cm4gYW5vbnltaXplZF90ZXh0LCByZXN1bHRzCgoKZGVmIF9nZXRfc2luZ2xlX2h0bWwoCiAgICB0ZXh0OiBzdHIsIHJlc3VsdHM6IExpc3RbcGEuUmVjb2duaXplclJlc3VsdF0sIGlzX2Z1bGxfaHRtbDogYm9vbCA9IFRydWUKKToKICAgICIiIgogICAgR2VuZXJhdGUgdGhlIGh0bWwgZm9yIGEgc2luZ2xlIHR4dCBmaWxlLgoKICAgIDpwYXJhbSB0ZXh0OiAgICAgICAgIFRoZSB0ZXh0IGZvciBhbmFseXNpcy4KICAgIDpwYXJhbSByZXN1bHRzOiAgICAgIFRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcy4KICAgIDpwYXJhbSBpc19mdWxsX2h0bWw6IFdoZXRoZXIgZ2VuZXJhdGUgZnVsbCBodG1sIG9yIG5vdC4KCiAgICA6cmV0dXJuczogVGhlIGh0bWwgc3RyaW5nIGZvciBhIHNpbmdsZSB0eHQgZmlsZS4KICAgICIiIgogICAgIyBjb252ZXJ0IHRoZSByZXN1bHRzIHRvIHRva2VucyB0byBnZW5lcmF0ZSB0aGUgaHRtbAogICAgdG9rZW5zID0gX2Fubm90YXRlKHRleHQsIHJlc3VsdHMsIGlzX2Z1bGxfaHRtbCkKICAgIGh0bWwgPSBhdF91dGlsLmdldF9hbm5vdGF0ZWRfaHRtbCgqdG9rZW5zKQoKICAgICMgYXZvaWQgdGhlIGVycm9yIGR1cmluZyByZW5kZXJpbmcgb2YgdGhlIFxuIGluIHRoZSBodG1sCiAgICBiYWNrc2xhc2hfY2hhciA9ICJcXCIKCiAgICBodG1sX3N0ciA9IGYiPHA+e2h0bWwucmVwbGFjZSgne2JhY2tzbGFzaF9jaGFyfW4nLCAnPGJyPicpfTwvcD4iCgogICAgcmV0dXJuIGh0bWxfc3RyCgoKZGVmIF9nZXRfc2luZ2xlX2pzb24ocmVzdWx0czogTGlzdFtwYS5SZWNvZ25pemVyUmVzdWx0XSwgaXNfZnVsbF9yZXBvcnQ6IGJvb2wgPSBUcnVlKToKICAgICIiIgogICAgR2VuZXJhdGUgdGhlIGpzb24gZm9yIGEgc2luZ2xlIHR4dCBmaWxlLgoKICAgIDpwYXJhbSByZXN1bHRzOiAgICAgICAgVGhlIGxpc3Qgb2YgUHJlc2lkaW8gUmVjb2duaXplclJlc3VsdCBjb25zdHJ1Y3RlZCBmcm9tIGFuYWx5c2lzLgogICAgOnBhcmFtIGlzX2Z1bGxfcmVwb3J0OiBXaGV0aGVyIGdlbmVyYXRlIGZ1bGwganNvbiBvciBub3QuCgogICAgOnJldHVybnM6IFRoZSBqc29uIHN0cmluZyBmb3IgYSBzaW5nbGUgdHh0IGZpbGUuCiAgICAiIiIKICAgICMgZ2VuZXJhdGUgdGhlIHN0YXRzIHJlcG9ydCBpZiBuZWVkZWQKICAgIGlmIG5vdCBpc19mdWxsX3JlcG9ydDoKICAgICAgICBzdGF0cyA9IFtdCiAgICAgICAgIyBhZGQgdGhlIHNpbXBsaWZ5IHN0YXRzIGxvZ2ljIGhlcmUKICAgICAgICBmb3IgaXRlbSBpbiByZXN1bHRzOgogICAgICAgICAgICBpdGVtLmFuYWx5c2lzX2V4cGxhbmF0aW9uID0gTm9uZQogICAgICAgICAgICBzdGF0cy5hcHBlbmQoaXRlbSkKICAgIGVsc2U6CiAgICAgICAgc3RhdHMgPSByZXN1bHRzCgogICAgcmV0dXJuIHN0YXRzCgoKZGVmIF9nZXRfYWxsX2h0bWwoCiAgICB0eHRfY29udGVudDogZGljdCwKICAgIHJlc19kaWN0OiBkaWN0LAogICAgaXNfZnVsbF9odG1sOiBib29sID0gVHJ1ZSwKKToKICAgICIiIgogICAgR2VuZXJhdGUgdGhlIGh0bWwgZm9yIGFsbCB0eHQgZmlsZXMuCgogICAgOnBhcmFtIHR4dF9jb250ZW50OiAgVGhlIGRpY3Rpb25hcnkgb2YgdHh0IGZpbGUgbmFtZSBhbmQgY29udGVudC4KICAgIDpwYXJhbSByZXNfZGljdDogICAgIFRoZSBkaWN0aW9uYXJ5IG9mIHR4dCBmaWxlIG5hbWUgYW5kIHRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcy4KICAgIDpwYXJhbSBpc19mdWxsX2h0bWw6IFdoZXRoZXIgZ2VuZXJhdGUgZnVsbCBodG1sIG9yIG5vdC4KCiAgICA6cmV0dXJuczogVGhlIGh0bWwgc3RyaW5nIGZvciBhbGwgdHh0IGZpbGVzLgoKICAgICIiIgogICAgIyBUaGVzZSBhcmUgcGxhY2Vob2xkZXIgZm9yIHRoZSBodG1sIHN0cmluZwogICAgaHRtbF9pbmRleCA9ICI8aHRtbD48aGVhZD48dGl0bGU+SGlnaGxpZ2h0ZWQgUGlpIEVudGl0aWVzPC90aXRsZT48L2hlYWQ+PGJvZHk+PGgxPkhpZ2hsaWdodGVkIFBpaSBFbnRpdGllczwvaDE+PHVsPiIKICAgIGh0bWxfY29udGVudCA9ICIiCiAgICBmb3IgdHh0X2ZpbGUsIHJlc3VsdHMgaW4gcmVzX2RpY3QuaXRlbXMoKToKICAgICAgICB0eHQgPSB0eHRfY29udGVudFt0eHRfZmlsZV0KICAgICAgICBodG1sX2luZGV4ICs9IGYiPGxpPjxhIGhyZWY9JyN7dHh0X2ZpbGV9Jz57dHh0X2ZpbGV9PC9hPjwvbGk+IgogICAgICAgIGh0bWxfY29udGVudCArPSBmIjxsaT48aDI+e3R4dF9maWxlfTwvaDI+PHA+e19nZXRfc2luZ2xlX2h0bWwodHh0LCByZXN1bHRzLCBpc19mdWxsX2h0bWwpfTwvcD48L2xpPiIKICAgIGh0bWxfaW5kZXggKz0gIjwvdWw+IgogICAgaHRtbF9yZXMgPSBmIntodG1sX2luZGV4fXtodG1sX2NvbnRlbnR9PC9ib2R5PjwvaHRtbD4iCgogICAgcmV0dXJuIGh0bWxfcmVzCgoKZGVmIF9nZXRfYWxsX3JwdChyZXNfZGljdDogZGljdCwgaXNfZnVsbF9yZXBvcnQ6IGJvb2wgPSBUcnVlKToKICAgICIiIgogICAgR2VuZXJhdGUgdGhlIHN0YXRzIHJlcG9ydCBmb3IgYWxsIHR4dCBmaWxlcy4KCiAgICA6cGFyYW0gcmVzX2RpY3Q6ICAgICAgIFRoZSBkaWN0aW9uYXJ5IG9mIHR4dCBmaWxlIG5hbWUgYW5kIHRoZSBsaXN0IG9mIFByZXNpZGlvIFJlY29nbml6ZXJSZXN1bHQgY29uc3RydWN0ZWQgZnJvbSBhbmFseXNpcy4KICAgIDpwYXJhbSBpc19mdWxsX3JlcG9ydDogV2hldGhlciBnZW5lcmF0ZSBmdWxsIHJlcG9ydCBvciBub3QuCgogICAgOnJldHVybnM6IFRoZSBzdGF0cyByZXBvcnQgZm9yIGFsbCB0eHQgZmlsZXMuCiAgICAiIiIKICAgICMgVGhlc2UgYXJlIHBsYWNlaG9sZGVyIGZvciB0aGUganNvbiByZXBvcnQKICAgIHN0YXRzX2RpY3QgPSB7fQogICAgZm9yIHR4dF9maWxlLCByZXN1bHRzIGluIHJlc19kaWN0Lml0ZW1zKCk6CiAgICAgICAgbmV3X3N0YXRzID0gW10KICAgICAgICBmb3IgaXRlbSBpbiBfZ2V0X3NpbmdsZV9qc29uKHJlc3VsdHMsIGlzX2Z1bGxfcmVwb3J0KToKICAgICAgICAgICAgaWYgaXNfZnVsbF9yZXBvcnQ6CiAgICAgICAgICAgICAgICBpdGVtLmFuYWx5c2lzX2V4cGxhbmF0aW9uID0gaXRlbS5hbmFseXNpc19leHBsYW5hdGlvbi50b19kaWN0KCkKICAgICAgICAgICAgICAgIG5ld19zdGF0cy5hcHBlbmQoaXRlbS50b19kaWN0KCkpCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICB0bXBfZGljdCA9IGl0ZW0udG9fZGljdCgpCiAgICAgICAgICAgICAgICB0bXBfZGljdC5wb3AoImFuYWx5c2lzX2V4cGxhbmF0aW9uIikKICAgICAgICAgICAgICAgIHRtcF9kaWN0LnBvcCgicmVjb2duaXRpb25fbWV0YWRhdGEiKQogICAgICAgICAgICAgICAgbmV3X3N0YXRzLmFwcGVuZCh0bXBfZGljdCkKICAgICAgICBzdGF0c19kaWN0W3R4dF9maWxlXSA9IG5ld19zdGF0cwogICAgcmV0dXJuIHN0YXRzX2RpY3QKCgpkZWYgcmVjb2duaXplX3BpaSgKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgaW5wdXRfcGF0aDogVW5pb25bc3RyLCBwYXRobGliLlBhdGhdLAogICAgaHRtbF9rZXk6IHN0ciwKICAgIHNjb3JlX3RocmVzaG9sZDogZmxvYXQsCiAgICBvdXRwdXRfZGlyZWN0b3J5OiBzdHIgPSBOb25lLAogICAgZW50aXRpZXM6IExpc3RbCiAgICAgICAgc3RyCiAgICBdID0gTm9uZSwgICMgTGlzdCBvZiBlbnRpdGllcyB0byByZWNvZ25pemUsIGRlZmF1bHQgaXMgcmVjb2duaXppbmcgYWxsCiAgICBlbnRpdHlfb3BlcmF0b3JfbWFwOiBkaWN0ID0gTm9uZSwKICAgIG1vZGVsOiBzdHIgPSBOb25lLAogICAgZ2VuZXJhdGVfanNvbjogYm9vbCA9IFRydWUsCiAgICBnZW5lcmF0ZV9odG1sOiBib29sID0gVHJ1ZSwKICAgIGlzX2Z1bGxfdGV4dDogYm9vbCA9IFRydWUsCiAgICBpc19mdWxsX2h0bWw6IGJvb2wgPSBUcnVlLAogICAgaXNfZnVsbF9yZXBvcnQ6IGJvb2wgPSBUcnVlLAopIC0+IFVuaW9uW1R1cGxlW3N0ciwgcGQuRGF0YUZyYW1lLCBkaWN0LCBkaWN0XSwgVHVwbGVbc3RyLCBwZC5EYXRhRnJhbWUsIGRpY3RdXToKICAgICIiIgogICAgV2FsayB0aHJvdWdoIHRoZSBpbnB1dCBwYXRoLCByZWNvZ25pemUgUElJIGluIHRleHQgYW5kIHN0b3JlIHRoZSBhbm9ueW1pemVkIHRleHQgaW4gdGhlIG91dHB1dCBwYXRoLgogICAgR2VuZXJhdGUgdGhlIGh0bWwgd2l0aCBkaWZmZXJlbnQgY29sb3JzIGZvciBlYWNoIGVudGl0eSwganNvbiByZXBvcnQgb2YgdGhlIGV4cGxhbmF0aW9uLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgVGhlIE1MUnVuIGNvbnRleHQuIHRoaXMgaXMgbmVlZGVkIGZvciBsb2cgdGhlIGFydGlmYWN0cy4KICAgIDpwYXJhbSBpbnB1dF9wYXRoOiAgICAgICAgICAgVGhlIGlucHV0IHBhdGggb2YgdGhlIHRleHQgZmlsZXMgbmVlZHMgdG8gYmUgYW5hbHl6ZWQuCiAgICA6cGFyYW0gaHRtbF9rZXk6ICAgICAgICAgICAgIFRoZSBodG1sIGtleSBmb3IgdGhlIGFydGlmYWN0LgogICAgOnBhcmFtIHNjb3JlX3RocmVzaG9sZDogICAgICBUaGUgc2NvcmUgdGhyZXNob2xkIHRvIG1hcmsgdGhlIHJlY29nbml0aW9uIGFzIHRydXN0ZWQuCiAgICA6cGFyYW0gb3V0cHV0X2RpcmVjdG9yeTogICAgIFRoZSBvdXRwdXQgZGlyZWN0b3J5IHBhdGggdG8gc3RvcmUgdGhlIGFub255bWl6ZWQgdGV4dC4KICAgIDpwYXJhbSBlbnRpdGllczogICAgICAgICAgICAgVGhlIGxpc3Qgb2YgZW50aXRpZXMgdG8gcmVjb2duaXplLgogICAgOnBhcmFtIGVudGl0eV9vcGVyYXRvcl9tYXA6ICBUaGUgbWFwIG9mIGVudGl0eSB0byBvcGVyYXRvciAobWFzaywgcmVkYWN0LCByZXBsYWNlLCBrZWVwLCBoYXNoLCBhbmQgaXRzIHBhcmFtcykKICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgVGhlIG1vZGVsIHRvIHVzZS4gQ2FuIGJlICJzcGFjeSIsICJmbGFpciIsICJwYXR0ZXJuIiBvciAid2hvbGUiLgogICAgOnBhcmFtIGdlbmVyYXRlX2pzb246ICAgICAgICBXaGV0aGVyIHRvIGdlbmVyYXRlIHRoZSBqc29uIHJlcG9ydCBvZiB0aGUgZXhwbGFuYXRpb24uCiAgICA6cGFyYW0gZ2VuZXJhdGVfaHRtbDogICAgICAgIFdoZXRoZXIgdG8gZ2VuZXJhdGUgdGhlIGh0bWwgcmVwb3J0IG9mIHRoZSBleHBsYW5hdGlvbi4KICAgIDpwYXJhbSBpc19mdWxsX3RleHQ6ICAgICAgICAgV2hldGhlciB0byByZXR1cm4gdGhlIGZ1bGwgdGV4dCBvciBvbmx5IHRoZSBtYXNrZWQgdGV4dC4KICAgIDpwYXJhbSBpc19mdWxsX2h0bWw6ICAgICAgICAgV2hldGhlciB0byByZXR1cm4gdGhlIGZ1bGwgaHRtbCBvciBqdXN0IHRoZSBhbm5vdGF0ZWQgdGV4dAogICAgOnBhcmFtIGlzX2Z1bGxfcmVwb3J0OiAgICAgICBXaGV0aGVyIHRvIHJldHVybiB0aGUgZnVsbCByZXBvcnQgb3IganVzdCB0aGUgc2NvcmUgYW5kIHN0YXJ0LCBlbmQgaW5kZXgKCiAgICA6cmV0dXJuczogQSB0dXBsZSBvZjoKCiAgICAgICAgICAgICAgKiBQYXRoIHRvIHRoZSBvdXRwdXQgZGlyZWN0b3J5CiAgICAgICAgICAgICAgKiBUaGUganNvbiByZXBvcnQgb2YgdGhlIGV4cGxhbmF0aW9uIChpZiBnZW5lcmF0ZV9qc29uIGlzIFRydWUpCiAgICAgICAgICAgICAgKiBBIGRpY3Rpb25hcnkgb2YgZXJyb3JzIGZpbGVzIHRoYXQgd2VyZSBub3QgcHJvY2Vzc2VkCgogICAgIiIiCgogICAgIyBTZXQgb3V0cHV0IGRpcmVjdG9yeQogICAgaWYgb3V0cHV0X2RpcmVjdG9yeSBpcyBOb25lOgogICAgICAgIG91dHB1dF9kaXJlY3RvcnkgPSB0ZW1wZmlsZS5ta2R0ZW1wKCkKCiAgICAjIENyZWF0ZSB0aGUgb3V0cHV0IGRpcmVjdG9yeToKICAgIG91dHB1dF9kaXJlY3RvcnkgPSBwYXRobGliLlBhdGgob3V0cHV0X2RpcmVjdG9yeSkKICAgIGlmIG5vdCBvdXRwdXRfZGlyZWN0b3J5LmV4aXN0cygpOgogICAgICAgIG91dHB1dF9kaXJlY3RvcnkubWtkaXIocGFyZW50cz1UcnVlLCBleGlzdF9vaz1UcnVlKQoKICAgIHR4dF9maWxlc19kaXJlY3RvcnkgPSBwYXRobGliLlBhdGgoaW5wdXRfcGF0aCkKICAgIHN1Y2Nlc3NlcyA9IFtdCiAgICBlcnJvcnMgPSB7fQoKICAgIHJlc19kaWN0ID0ge30KICAgIHR4dF9jb250ZW50ID0ge30KICAgICMgTG9hZCB0aGUgbW9kZWw6CiAgICBhbmFseXplciA9IF9nZXRfYW5hbHl6ZXJfZW5naW5lKG1vZGVsLCBlbnRpdGllcykKICAgIGxvZ2dlci5pbmZvKCJNb2RlbCBsb2FkZWQiKQogICAgIyBHbyBvdmVyIHRoZSB0ZXh0IGZpbGVzIGluIHRoZSBpbnB1dCBwYXRoLCBhbmFseXplIGFuZCBhbm9ueW1pemUgdGhlbToKICAgIGZvciB0eHRfZmlsZSBpbiB0cWRtKAogICAgICAgIGxpc3QodHh0X2ZpbGVzX2RpcmVjdG9yeS5nbG9iKCIqLnR4dCIpKSwKICAgICAgICBkZXNjPSJQcm9jZXNzaW5nIGZpbGVzIiwKICAgICAgICB1bml0PSJmaWxlIiwKICAgICk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICAjIExvYWQgdGhlIHN0ciBmcm9tIHRoZSB0ZXh0IGZpbGUKICAgICAgICAgICAgdGV4dCA9IHR4dF9maWxlLnJlYWRfdGV4dCgpCiAgICAgICAgICAgIHR4dF9jb250ZW50W3N0cih0eHRfZmlsZSldID0gdGV4dAogICAgICAgICAgICAjIFByb2Nlc3MgdGhlIHRleHQgdG8gcmVjb2dpbnplIHRoZSBwaWkgZW50aXRpZXMgaW4gaXQKICAgICAgICAgICAgYW5vbnltaXplZF90ZXh0LCByZXN1bHRzID0gX3Byb2Nlc3MoCiAgICAgICAgICAgICAgICB0ZXh0PXRleHQsCiAgICAgICAgICAgICAgICBtb2RlbD1hbmFseXplciwKICAgICAgICAgICAgICAgIGVudGl0aWVzPWVudGl0aWVzLAogICAgICAgICAgICAgICAgZW50aXRpZXNfb3BlcmF0b3JfbWFwPWVudGl0eV9vcGVyYXRvcl9tYXAsCiAgICAgICAgICAgICAgICBzY29yZV90aHJlc2hvbGQ9c2NvcmVfdGhyZXNob2xkLAogICAgICAgICAgICAgICAgaXNfZnVsbF90ZXh0PWlzX2Z1bGxfdGV4dCwKICAgICAgICAgICAgKQogICAgICAgICAgICByZXNfZGljdFtzdHIodHh0X2ZpbGUpXSA9IHJlc3VsdHMKICAgICAgICAgICAgIyBTdG9yZSB0aGUgYW5vbnltaXplZCB0ZXh0IGluIHRoZSBvdXRwdXQgcGF0aAogICAgICAgICAgICBvdXRwdXRfZmlsZSA9IG91dHB1dF9kaXJlY3RvcnkgLyBmInt0eHRfZmlsZS5zdGVtfS50eHQiCiAgICAgICAgICAgIG91dHB1dF9maWxlLnBhcmVudC5ta2RpcihwYXJlbnRzPVRydWUsIGV4aXN0X29rPVRydWUpCiAgICAgICAgICAgIHdpdGggb3BlbihvdXRwdXRfZmlsZSwgInciKSBhcyBmOgogICAgICAgICAgICAgICAgZi53cml0ZShhbm9ueW1pemVkX3RleHQpCiAgICAgICAgICAgIHN1Y2Nlc3Nlcy5hcHBlbmQoW3R4dF9maWxlLm5hbWUsIG91dHB1dF9maWxlLm5hbWVdKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZToKICAgICAgICAgICAgZXJyb3JzW3N0cih0eHRfZmlsZSldID0gc3RyKGUpCiAgICAgICAgICAgIGxvZ2dlci5lcnJvcihmIkVycm9yIHByb2Nlc3Npbmcge3R4dF9maWxlfToge2V9IikKCiAgICBzdWNjZXNzZXMgPSBwZC5EYXRhRnJhbWUoCiAgICAgICAgc3VjY2Vzc2VzLAogICAgICAgIGNvbHVtbnM9WyJvcmlnaW5hbF9maWxlIiwgImFub255bWl6ZWRfZmlsZSJdLAogICAgKQoKICAgIGlmIGdlbmVyYXRlX2h0bWw6CiAgICAgICAgIyBHZW5lcmF0ZSB0aGUgaHRtbCByZXBvcnQKICAgICAgICBodG1sX3JlcyA9IF9nZXRfYWxsX2h0bWwodHh0X2NvbnRlbnQsIHJlc19kaWN0LCBpc19mdWxsX2h0bWwpCiAgICAgICAgIyBTdG9yZSB0aGUgaHRtbCByZXBvcnQgaW4gdGhlIGNvbnRleHQKICAgICAgICBhcnRpX2h0bWwgPSBtbHJ1bi5hcnRpZmFjdHMuQXJ0aWZhY3QoYm9keT1odG1sX3JlcywgZm9ybWF0PSJodG1sIiwga2V5PWh0bWxfa2V5KQogICAgICAgIGNvbnRleHQubG9nX2FydGlmYWN0KGFydGlfaHRtbCkKICAgIGlmIGdlbmVyYXRlX2pzb246CiAgICAgICAgIyBHZW5lcmF0ZSB0aGUganNvbiByZXBvcnQKICAgICAgICBqc29uX3JlcyA9IF9nZXRfYWxsX3JwdChyZXNfZGljdCwgaXNfZnVsbF9yZXBvcnQpCiAgICAgICAgcmV0dXJuIHN0cihvdXRwdXRfZGlyZWN0b3J5KSwgc3VjY2Vzc2VzLCBlcnJvcnMsIGpzb25fcmVzCiAgICByZXR1cm4gc3RyKG91dHB1dF9kaXJlY3RvcnkpLCBzdWNjZXNzZXMsIGVycm9ycwo= - code_origin: '' - origin_filename: '' - description: This function is used to recognize PII in a directory of text files - image: '' + lineno: 845 command: '' - disable_auto_mount: false -kind: job -metadata: - name: pii-recognizer - tag: '' - categories: - - data-preparation - - NLP + description: This function is used to recognize PII in a directory of text files + default_handler: recognize_pii diff --git a/functions/src/pii_recognizer/pii_recognizer.py b/functions/src/pii_recognizer/pii_recognizer.py index 0acc55dcb..3a5366635 100644 --- a/functions/src/pii_recognizer/pii_recognizer.py +++ b/functions/src/pii_recognizer/pii_recognizer.py @@ -17,7 +17,7 @@ import pathlib import tempfile import warnings -from typing import List, Set, Tuple, Union +from typing import List import annotated_text.util as at_util import mlrun @@ -162,9 +162,9 @@ class CustomSpacyRecognizer(pa.LocalRecognizer): def __init__( self, supported_language: str = "en", - supported_entities: List[str] = None, - check_label_groups: Tuple[Set, Set] = None, - context: List[str] = None, + supported_entities: list[str] = None, + check_label_groups: tuple[set, set] = None, + context: list[str] = None, ner_strength: float = 1, ): """ @@ -258,7 +258,7 @@ def analyze(self, text: str, entities: List[str], nlp_artifacts=None): # noqa D @staticmethod def __check_label( - entity: str, label: str, check_label_groups: Tuple[Set, Set] + entity: str, label: str, check_label_groups: tuple[set, set] ) -> bool: """ Check if the label is in the label group. @@ -351,8 +351,8 @@ class FlairRecognizer(pa.EntityRecognizer): def __init__( self, supported_language: str = "en", - supported_entities: List[str] = None, - check_label_groups: Tuple[Set, Set] = None, + supported_entities: list[str] = None, + check_label_groups: tuple[set, set] = None, ): """ Initialize the FlairRecognizer. @@ -381,9 +381,9 @@ def __init__( def analyze( self, text: str, - entities: List[str], + entities: list[str], nlp_artifacts: pa.nlp_engine.NlpArtifacts = None, - ) -> List[pa.RecognizerResult]: + ) -> list[pa.RecognizerResult]: """ Analyze text and return the results. @@ -483,7 +483,7 @@ def _build_flair_explanation( # sanity check of the entity and label before recognition @staticmethod def __check_label( - entity: str, label: str, check_label_groups: Tuple[Set, Set] + entity: str, label: str, check_label_groups: tuple[set, set] ) -> bool: return any( entity in egrp and label in lgrp for egrp, lgrp in check_label_groups @@ -492,7 +492,7 @@ def __check_label( # get the analyzer engine based on the model def _get_analyzer_engine( - model: str = None, entities: List[str] = None + model: str = None, entities: list[str] = None ) -> pa.AnalyzerEngine: """ Return pa.AnalyzerEngine. @@ -542,7 +542,7 @@ def _get_analyzer_engine( registry.add_recognizer(recognizer) else: raise ValueError( - f"argument of model and entities can not be None at the same time" + "argument of model and entities can not be None at the same time" ) analyzer = pa.AnalyzerEngine( registry=registry, @@ -573,7 +573,7 @@ def _get_anonymizer_engine() -> pre_anoymizer.AnonymizerEngine: def _anonymize( text: str, - analyze_results: List[pa.RecognizerResult], + analyze_results: list[pa.RecognizerResult], entity_operator_map: dict = None, is_full_text: bool = True, ) -> str: @@ -640,8 +640,8 @@ def _anonymize( def _get_tokens( - text: str, analyze_results: List[pa.RecognizerResult], is_full: bool = True -) -> List[str]: + text: str, analyze_results: list[pa.RecognizerResult], is_full: bool = True +) -> list[str]: """ Get the full tokens or only contains the entities that can form a sentence. @@ -685,8 +685,8 @@ def _get_tokens( def _annotate( - text: str, st_analyze_results: List[pa.RecognizerResult], is_full_html: bool = True -) -> List[str]: + text: str, st_analyze_results: list[pa.RecognizerResult], is_full_html: bool = True +) -> list[str]: """ Annotate identified input using Presidio Anonymizer. @@ -704,10 +704,10 @@ def _process( text: str, model: pa.AnalyzerEngine, score_threshold: float, - entities: List[str] = None, + entities: list[str] = None, entities_operator_map: dict = None, is_full_text: bool = True, -) -> Tuple[str, list]: +) -> tuple[str, list]: """ Process the text of str using the model. @@ -743,7 +743,7 @@ def _process( def _get_single_html( - text: str, results: List[pa.RecognizerResult], is_full_html: bool = True + text: str, results: list[pa.RecognizerResult], is_full_html: bool = True ): """ Generate the html for a single txt file. @@ -766,7 +766,7 @@ def _get_single_html( return html_str -def _get_single_json(results: List[pa.RecognizerResult], is_full_report: bool = True): +def _get_single_json(results: list[pa.RecognizerResult], is_full_report: bool = True): """ Generate the json for a single txt file. @@ -844,11 +844,11 @@ def _get_all_rpt(res_dict: dict, is_full_report: bool = True): def recognize_pii( context: mlrun.MLClientCtx, - input_path: Union[str, pathlib.Path], + input_path: str | pathlib.Path, html_key: str, score_threshold: float, output_directory: str = None, - entities: List[ + entities: list[ str ] = None, # List of entities to recognize, default is recognizing all entity_operator_map: dict = None, @@ -858,7 +858,7 @@ def recognize_pii( is_full_text: bool = True, is_full_html: bool = True, is_full_report: bool = True, -) -> Union[Tuple[str, pd.DataFrame, dict, dict], Tuple[str, pd.DataFrame, dict]]: +) -> tuple[str, pd.DataFrame, dict, dict] | tuple[str, pd.DataFrame, dict]: """ Walk through the input path, recognize PII in text and store the anonymized text in the output path. Generate the html with different colors for each entity, json report of the explanation. diff --git a/functions/src/pii_recognizer/test_pii_recognizer.py b/functions/src/pii_recognizer/test_pii_recognizer.py index 81a16611f..080a5367a 100644 --- a/functions/src/pii_recognizer/test_pii_recognizer.py +++ b/functions/src/pii_recognizer/test_pii_recognizer.py @@ -13,16 +13,14 @@ # limitations under the License. # -import os -import pytest import random -from faker import Faker + import mlrun +import pytest +from faker import Faker from pii_recognizer import ( - _process, _get_analyzer_engine, - _anonymize, - _annotate, + _process, recognize_pii_parallel, ) diff --git a/functions/src/pyannote_audio/function.yaml b/functions/src/pyannote_audio/function.yaml index b4cd9ad93..78bfaf1a6 100644 --- a/functions/src/pyannote_audio/function.yaml +++ b/functions/src/pyannote_audio/function.yaml @@ -1,56 +1,58 @@ +metadata: + tag: '' + name: pyannote-audio + categories: + - deep-learning + - audio +verbose: false kind: job spec: - command: '' - disable_auto_mount: false image: '' + disable_auto_mount: false build: - code_origin: '' + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgoKaW1wb3J0IGhlYXBxCmltcG9ydCBsb2dnaW5nCmltcG9ydCBvcGVyYXRvcgppbXBvcnQgb3MKaW1wb3J0IHBhdGhsaWIKZnJvbSBmdW5jdG9vbHMgaW1wb3J0IHJlZHVjZSwgd3JhcHMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueQoKaW1wb3J0IHBhbmRhcyBhcyBwZAppbXBvcnQgcHlhbm5vdGUuYXVkaW8KaW1wb3J0IHB5YW5ub3RlLmNvcmUKaW1wb3J0IHRvcmNoCmltcG9ydCB0b3JjaGF1ZGlvCmZyb20gdHFkbSBpbXBvcnQgdHFkbQoKIyBHZXQgdGhlIGdsb2JhbCBsb2dnZXI6Cl9MT0dHRVIgPSBsb2dnaW5nLmdldExvZ2dlcigpCgoKZGVmIF9jaGVja19tbHJ1bl9hbmRfb3Blbl9tcGkoKSAtPiB0dXBsZVsibWxydW4uTUxDbGllbnRDdHgiLCAibXBpNHB5Lk1QSS5JbnRyYWNvbW0iXToKICAgIGlzX21waSA9IEZhbHNlCiAgICB0cnk6CiAgICAgICAgaW1wb3J0IG1scnVuCgogICAgICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJtbHJ1biIpCiAgICAgICAgaXNfbXBpID0gY29udGV4dC5sYWJlbHMuZ2V0KCJraW5kIiwgImpvYiIpID09ICJtcGlqb2IiCgogICAgICAgIGlmIGlzX21waToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnJvbSBtcGk0cHkgaW1wb3J0IE1QSQoKICAgICAgICAgICAgICAgIHJldHVybiBjb250ZXh0LCBNUEkuQ09NTV9XT1JMRAogICAgICAgICAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvciBhcyBtcGk0cHlfbm90X2ZvdW5kOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgIlRvIGRpc3RyaWJ1dGUgdGhlIGZ1bmN0aW9uIHVzaW5nIE1MUnVuJ3MgJ21waWpvYicgeW91IG5lZWQgdG8gaGF2ZSBgbXBpNHB5YCBwYWNrYWdlIGluIHlvdXIgIgogICAgICAgICAgICAgICAgICAgICJpbnRlcnByZXRlci4gUGxlYXNlIHJ1biBgcGlwIGluc3RhbGwgbXBpNHB5YCBhbmQgbWFrZSBzdXJlIHlvdSBoYXZlIG9wZW4tbXBpLiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIG1waTRweV9ub3RfZm91bmQKICAgICAgICBlbHNlOgogICAgICAgICAgICByZXR1cm4gY29udGV4dCwgTm9uZQogICAgZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3IgYXMgbW9kdWxlX25vdF9mb3VuZDoKICAgICAgICBpZiBpc19tcGk6CiAgICAgICAgICAgIHJhaXNlIG1vZHVsZV9ub3RfZm91bmQKICAgIHJldHVybiBOb25lLCBOb25lCgoKZGVmIG9wZW5fbXBpX2hhbmRsZXIoCiAgICB3b3JrZXJfaW5wdXRzOiBsaXN0W3N0cl0sIHJvb3Rfd29ya2VyX2lucHV0czogZGljdFtzdHIsIEFueV0gPSBOb25lCik6CiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgQ2hlY2sgZm9yIE1MUnVuIGFuZCBPcGVuTVBJIGF2YWlsYWJpbGl0eToKICAgIGNvbnRleHQsIGNvbW0gPSBfY2hlY2tfbWxydW5fYW5kX29wZW5fbXBpKCkKCiAgICAjIENoZWNrIGlmIE1MUnVuIGlzIGF2YWlsYWJsZSwgc2V0IHRoZSBnbG9iYWwgbG9nZ2VyIHRvIE1MUnVuJ3M6CiAgICBpZiBjb250ZXh0OgogICAgICAgIF9MT0dHRVIgPSBjb250ZXh0LmxvZ2dlcgoKICAgIGRlZiBkZWNvcmF0b3IoaGFuZGxlcik6CiAgICAgICAgaWYgY29tbSBpcyBOb25lIG9yIGNvbW0uR2V0X3NpemUoKSA9PSAxOgogICAgICAgICAgICByZXR1cm4gaGFuZGxlcgoKICAgICAgICBAd3JhcHMoaGFuZGxlcikKICAgICAgICBkZWYgd3JhcHBlcigqKmt3YXJncyk6CiAgICAgICAgICAgICMgR2V0IHRoZSBvcGVuIG1waSBlbnZpcm9ubWVudCBwcm9wZXJ0aWVzOgogICAgICAgICAgICBzaXplID0gY29tbS5HZXRfc2l6ZSgpCiAgICAgICAgICAgIHJhbmsgPSBjb21tLkdldF9yYW5rKCkKCiAgICAgICAgICAgICMgR2l2ZSB0aGUgY29ycmVjdCBjaHVuayBvZiB0aGUgd29ya2VycyBpbnB1dHM6CiAgICAgICAgICAgIGZvciB3b3JrZXJfaW5wdXQgaW4gd29ya2VyX2lucHV0czoKICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0ga3dhcmdzW3dvcmtlcl9pbnB1dF0KICAgICAgICAgICAgICAgIGlmIGlucHV0X2FyZ3VtZW50IGlzIE5vbmU6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIHN0cik6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBfZ2V0X2F1ZGlvX2ZpbGVzKAogICAgICAgICAgICAgICAgICAgICAgICBkYXRhX3BhdGg9cGF0aGxpYi5QYXRoKGlucHV0X2FyZ3VtZW50KS5hYnNvbHV0ZSgpCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgaWYgbGVuKGlucHV0X2FyZ3VtZW50KSA8IHNpemU6CiAgICAgICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAgICAgZiJDYW5ub3Qgc3BsaXQgdGhlIGlucHV0ICd7d29ya2VyX2lucHV0fScgb2YgbGVuZ3RoIHtsZW4oaW5wdXRfYXJndW1lbnQpfSB0byB7c2l6ZX0gd29ya2Vycy4gIgogICAgICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSByZWR1Y2UgdGhlIGFtb3VudCBvZiB3b3JrZXJzIGZvciB0aGlzIGlucHV0LiIKICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBldmVuX2NodW5rX3NpemUgPSBsZW4oaW5wdXRfYXJndW1lbnQpIC8vIHNpemUKICAgICAgICAgICAgICAgIGNodW5rX3N0YXJ0ID0gcmFuayAqIGV2ZW5fY2h1bmtfc2l6ZQogICAgICAgICAgICAgICAgY2h1bmtfZW5kID0gKAogICAgICAgICAgICAgICAgICAgIChyYW5rICsgMSkgKiBldmVuX2NodW5rX3NpemUKICAgICAgICAgICAgICAgICAgICBpZiByYW5rICsgMSA8IHNpemUKICAgICAgICAgICAgICAgICAgICBlbHNlIGxlbihpbnB1dF9hcmd1bWVudCkKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSYW5rICN7cmFua306IFByb2Nlc3NpbmcgaW5wdXQgY2h1bmsgb2YgJ3t3b3JrZXJfaW5wdXR9JyAiCiAgICAgICAgICAgICAgICAgICAgZiJmcm9tIGluZGV4IHtjaHVua19zdGFydH0gdG8ge2NodW5rX2VuZH0uIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgbGlzdCk6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBpbnB1dF9hcmd1bWVudFtjaHVua19zdGFydDpjaHVua19lbmRdCiAgICAgICAgICAgICAgICBlbGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIHBkLkRhdGFGcmFtZSk6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBpbnB1dF9hcmd1bWVudC5pbG9jW2NodW5rX3N0YXJ0OmNodW5rX2VuZDosIDpdCiAgICAgICAgICAgICAgICBrd2FyZ3Nbd29ya2VyX2lucHV0XSA9IGlucHV0X2FyZ3VtZW50CgogICAgICAgICAgICAjIFNldCB0aGUgcm9vdCB3b3JrZXIgb25seSBhcmd1bWVudHM6CiAgICAgICAgICAgIGlmIHJhbmsgPT0gMCBhbmQgcm9vdF93b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAga3dhcmdzLnVwZGF0ZShyb290X3dvcmtlcl9pbnB1dHMpCgogICAgICAgICAgICAjIFJ1biB0aGUgd29ya2VyOgogICAgICAgICAgICBvdXRwdXQgPSBoYW5kbGVyKCoqa3dhcmdzKQoKICAgICAgICAgICAgIyBTZW5kIHRoZSBvdXRwdXQgdG8gdGhlIHJvb3QgcmFuayAocmFuayAjMCk6CiAgICAgICAgICAgIG91dHB1dCA9IGNvbW0uZ2F0aGVyKG91dHB1dCwgcm9vdD0wKQogICAgICAgICAgICBpZiByYW5rID09IDA6CiAgICAgICAgICAgICAgICAjIEpvaW4gdGhlIG91dHB1dHM6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJDb2xsZWN0aW5nIGRhdGEgZnJvbSB3b3JrZXJzIHRvIHJvb3Qgd29ya2VyLiIpCiAgICAgICAgICAgICAgICBkaWFyaXphdGlvbl9kaWN0aW9uYXJ5ID0gcmVkdWNlKAogICAgICAgICAgICAgICAgICAgIG9wZXJhdG9yLmlvciwgW2RpYSBmb3IgZGlhLCBfIGluIG91dHB1dF0sIHt9CiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBlcnJvcnNfZGljdGlvbmFyeSA9IHJlZHVjZShvcGVyYXRvci5pb3IsIFtlcnIgZm9yIF8sIGVyciBpbiBvdXRwdXRdLCB7fSkKICAgICAgICAgICAgICAgIHJldHVybiBkaWFyaXphdGlvbl9kaWN0aW9uYXJ5LCBlcnJvcnNfZGljdGlvbmFyeQogICAgICAgICAgICByZXR1cm4gTm9uZQoKICAgICAgICByZXR1cm4gd3JhcHBlcgoKICAgIHJldHVybiBkZWNvcmF0b3IKCgpAb3Blbl9tcGlfaGFuZGxlcih3b3JrZXJfaW5wdXRzPVsiZGF0YV9wYXRoIl0sIHJvb3Rfd29ya2VyX2lucHV0cz17InZlcmJvc2UiOiBUcnVlfSkKZGVmIGRpYXJpemUoCiAgICBkYXRhX3BhdGg6IHN0ciB8IGxpc3Rbc3RyXSwKICAgIG1vZGVsX25hbWU6IHN0ciA9ICJweWFubm90ZS9zcGVha2VyLWRpYXJpemF0aW9uLTMuMCIsCiAgICBhY2Nlc3NfdG9rZW46IHN0ciA9IE5vbmUsCiAgICBkZXZpY2U6IHN0ciA9IE5vbmUsCiAgICBzcGVha2Vyc19sYWJlbHM6IGxpc3Rbc3RyXSA9IE5vbmUsCiAgICBzcGVha2VyX3ByZWZpeDogc3RyID0gInNwZWFrZXJfIiwKICAgIHNlcGFyYXRlX2J5X2NoYW5uZWxzOiBib29sID0gRmFsc2UsCiAgICBtaW5pbXVtX3NwZWFrZXJzOiBpbnQgPSBOb25lLAogICAgbWF4aW11bV9zcGVha2VyczogaW50ID0gTm9uZSwKICAgIHZlcmJvc2U6IGJvb2wgPSBGYWxzZSwKKSAtPiB0dXBsZVtkaWN0W3N0ciwgbGlzdFt0dXBsZVtmbG9hdCwgZmxvYXQsIHN0cl1dXSwgZGljdFtzdHIsIHN0cl1dOgogICAgIiIiCiAgICBQZXJmb3JtIHNwZWVjaCBkaWFyaXphdGlvbiBvbiBnaXZlbiBhdWRpbyBmaWxlcyB1c2luZyBweWFubm90ZS1hdWRpbyAoaHR0cHM6Ly9naXRodWIuY29tL3B5YW5ub3RlL3B5YW5ub3RlLWF1ZGlvKS4KICAgIFRoZSBlbmQgcmVzdWx0IGlzIGEgZGljdGlvbmFyeSB3aXRoIHRoZSBmaWxlIG5hbWVzIGFzIGtleXMgYW5kIHRoZWlyIGRpYXJpemF0aW9uIGFzIHZhbHVlLiBBIGRpYXJpemF0aW9uIGlzIGEgbGlzdAogICAgb2YgdHVwbGVzOiAoc3RhcnQsIGVuZCwgc3BlYWtlcl9sYWJlbCkuCgogICAgVG8gdXNlIHRoZSBgcHlhbm5vdGUuYXVkaW9gIG1vZGVscyB5b3UgbXVzdCBwYXNzIGEgSHVnZ2luZ2ZhY2UgdG9rZW4gYW5kIGdldCBhY2Nlc3MgdG8gdGhlIHJlcXVpcmVkIG1vZGVscy4gVGhlCiAgICB0b2tlbiBjYW4gYmUgcGFzc2VkIGluIG9uZSBvZiB0aGUgZm9sbG93aW5nIG9wdGlvbnM6CgogICAgKiBVc2UgdGhlIHBhcmFtZXRlciBgYWNjZXNzX3Rva2VuYC4KICAgICogU2V0IGFuIGVudmlyb25tZW50IHZhcmlhYmxlIG5hbWVkICJIVUdHSU5HX0ZBQ0VfSFVCX1RPS0VOIi4KICAgICogSWYgdXNpbmcgTUxSdW4sIHlvdSBjYW4gcGFzcyBpdCBhcyBhIHNlY3JldCBuYW1lZCAiSFVHR0lOR19GQUNFX0hVQl9UT0tFTiIuCgogICAgVG8gZ2V0IGFjY2VzcyB0byB0aGUgbW9kZWxzIG9uIEh1Z2dpbmdmYWNlLCB2aXNpdCB0aGVpciBwYWdlLiBGb3IgZXhhbXBsZSwgdG8gdXNlIHRoZSBkZWZhdWx0IGRpYXJpemF0aW9uIG1vZGVsIHNldAogICAgaW4gdGhpcyBmdW5jdGlvbiAoInB5YW5ub3RlL3NwZWFrZXItZGlhcml6YXRpb24tMy4wIiksIHlvdSBuZWVkIGFjY2VzcyBmb3IgdGhlc2UgdHdvIG1vZGVsczoKCiAgICAqIGh0dHBzOi8vaHVnZ2luZ2ZhY2UuY28vcHlhbm5vdGUvc2VnbWVudGF0aW9uLTMuMAogICAgKiBodHRwczovL2h1Z2dpbmdmYWNlLmNvL3B5YW5ub3RlL3NwZWFrZXItZGlhcml6YXRpb24tMy4wCgogICAgTm90ZTogVG8gY29udHJvbCB0aGUgcmVjb2duaXplZCBzcGVha2VycyBpbiB0aGUgZGlhcml6YXRpb24gb3V0cHV0IHlvdSBjYW4gY2hvb3NlIG9uZSBvZiB0aGUgZm9sbG93aW5nIG1ldGhvZHM6CgogICAgKiBGb3IgYSBrbm93biBzcGVha2VycyBhbW91bnQsIHlvdSBtYXkgc2V0IHNwZWFrZXIgbGFiZWxzIHZpYSB0aGUgYHNwZWFrZXJzX2xhYmVsc2AgcGFyYW1ldGVyIHRoYXQgd2lsbCBiZSB1c2VkIGluCiAgICAgIHRoZSBvcmRlciBvZiBzcGVha2luZyBpbiB0aGUgYXVkaW8gKGZpcnN0IHBlcnNvbiBzcGVha2luZyBiZSB0aGUgZmlyc3QgbGFiZWwgaW4gdGhlIGxpc3QpLiBJbiBhZGRpdGlvbiwgeW91IGNhbiBkbwogICAgICBkaWFyaXphdGlvbiBwZXIgY2hhbm5lbCAoc2V0dGluZyB0aGUgcGFyYW1ldGVyIGBzZXBhcmF0ZV9ieV9jaGFubmVsc2AgdG8gVHJ1ZSkuIEVhY2ggbGFiZWwgd2lsbCBiZSBhc3NpZ25lZCB0byBhCiAgICAgIHNwZWNpZmljIGNoYW5uZWwgYnkgb3JkZXIgKGZpcnN0IGxhYmVsIHRvIGNoYW5uZWwgMCwgc2Vjb25kIGxhYmVsIHRvIGNoYW5uZWwgMSBhbmQgc28gb24pLiBOb3RpY2UsIHRoaXMgd2lsbAogICAgICBpbmNyZWFzZSBydW50aW1lLgogICAgKiBGb3IgdW5rbm93biBzcGVha2VycyBhbW91bnQsIHlvdSBjYW4gc2V0IHRoZSBgc3BlYWtlcl9wcmVmaXhgIHBhcmFtZXRlciB0byBhZGQgYSBwcmVmaXggZm9yIGVhY2ggc3BlYWtlciBudW1iZXIuCiAgICAgIFlvdSBjYW4gYWxzbyBoZWxwIHRoZSBkaWFyaXphdGlvbiBieSBzZXR0aW5nIHRoZSBzcGVha2VycyByYW5nZSB2aWEgdGhlIGBzcGVha2Vyc19hbW91bnRfcmFuZ2VgIHBhcmFtZXRlci4KCiAgICA6cGFyYW0gZGF0YV9wYXRoOiAgICAgICAgICAgIEEgZGlyZWN0b3J5IG9mIHRoZSBhdWRpbyBmaWxlcywgYSBzaW5nbGUgZmlsZSBvciBhIGxpc3Qgb2YgZmlsZXMgdG8gdHJhbnNjcmliZS4KICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgICAgICAgICAgT25lIG9mIHRoZSBvZmZpY2lhbCBkaWFyaXphdGlvbiBtb2RlbCBuYW1lcyAocmVmZXJyZWQgYXMgZGlhcml6YXRpb24gcGlwZWxpbmVzKSBvZgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBgcHlhbm5vdGUuYXVkaW9gIEh1Z2dpbmdmYWNlIHBhZ2UuIERlZmF1bHQ6ICJweWFubm90ZS9zcGVha2VyLWRpYXJpemF0aW9uLTMuMCIuCiAgICA6cGFyYW0gYWNjZXNzX3Rva2VuOiAgICAgICAgIEFuIGFjY2VzcyB0b2tlbiB0byBwYXNzIGZvciB1c2luZyB0aGUgYHB5YW5ub3RlLmF1ZGlvYCBtb2RlbHMuIElmIG5vdCBwcm92aWRlZCwgaXQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSBsb29raW5nIGZvciB0aGUgZW52aXJvbm1lbnQgdmFyaWFibGUgIkhVR0dJTkdfRkFDRV9IVUJfVE9LRU4iLiBJZiBNTFJ1biBpcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhdmFpbGFibGUsIGl0IHdpbGwgbG9vayBmb3IgYSBzZWNyZXQgIkhVR0dJTkdfRkFDRV9IVUJfVE9LRU4iLgogICAgOnBhcmFtIGRldmljZTogICAgICAgICAgICAgICBEZXZpY2UgdG8gbG9hZCB0aGUgbW9kZWwuIENhbiBiZSBvbmUgb2YgeyJjdWRhIiwgImNwdSJ9LiBEZWZhdWx0IHdpbGwgcHJlZmVyICJjdWRhIiBpZgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhdmFpbGFibGUuCiAgICA6cGFyYW0gc3BlYWtlcnNfbGFiZWxzOiAgICAgIExhYmVscyB0byB1c2UgZm9yIHRoZSByZWNvZ25pemVkIHNwZWFrZXJzLiBEZWZhdWx0OiBudW1lcmljIGxhYmVscyAoMCwgMSwgLi4uKS4KICAgIDpwYXJhbSBzZXBhcmF0ZV9ieV9jaGFubmVsczogSWYgZWFjaCBzcGVha2VyIGlzIHNwZWFraW5nIGluIGEgc2VwYXJhdGUgY2hhbm5lbCwgeW91IGNhbiBkaWFyaXplIGVhY2ggY2hhbm5lbCBhbmQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgY29tYmluZSB0aGUgcmVzdWx0IGludG8gYSBzaW5nbGUgZGlhcml6YXRpb24uIEVhY2ggbGFiZWwgc2V0IGluIHRoZSBgc3BlYWtlcnNfbGFiZWxzYAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBwYXJhbWV0ZXIgd2lsbCBiZSBhc3NpZ25lZCB0byBhIHNwZWNpZmljIGNoYW5uZWwgYnkgb3JkZXIuCiAgICA6cGFyYW0gc3BlYWtlcl9wcmVmaXg6ICAgICAgIEEgcHJlZml4IHRvIGFkZCBmb3IgdGhlIHNwZWFrZXJzIGxhYmVscy4gVGhpcyBwYXJhbWV0ZXIgaXMgaWdub3JlZCBpZgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBgc3BlYWtlcnNfbGFiZWxzYCBpcyBub3QgTm9uZS4gRGVmYXVsdDogInNwZWFrZXIiLgogICAgOnBhcmFtIG1pbmltdW1fc3BlYWtlcnM6ICAgICBTZXQgdGhlIG1pbmltdW0gZXhwZWN0ZWQgYW1vdW50IG9mIHNwZWFrZXJzIHRvIGJlIGluIHRoZSBhdWRpbyBmaWxlcy4gVGhpcyBwYXJhbWV0ZXIgaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaWdub3JlZCBpZiBgc3BlYWtlcnNfbGFiZWxzYCBpcyBub3QgTm9uZS4KICAgIDpwYXJhbSBtYXhpbXVtX3NwZWFrZXJzOiAgICAgU2V0IHRoZSBtYXhpbXVtIGV4cGVjdGVkIGFtb3VudCBvZiBzcGVha2VycyB0byBiZSBpbiB0aGUgYXVkaW8gZmlsZXMuIFRoaXMgcGFyYW1ldGVyIGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGlnbm9yZWQgaWYgYHNwZWFrZXJzX2xhYmVsc2AgaXMgbm90IE5vbmUuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICAgIFdoZXRoZXIgdG8gcHJlc2VudCBsb2dzIG9mIGEgcHJvZ3Jlc3MgYmFyIGFuZCBlcnJvcnMuIERlZmF1bHQ6IFRydWUuCgogICAgOnJldHVybnM6IEEgdHVwbGUgb2Y6CgogICAgICAgICAgICAgICogU3BlZWNoIGRpYXJpemF0aW9uIGRpY3Rpb25hcnkuCiAgICAgICAgICAgICAgKiBBIGRpY3Rpb25hcnkgb2YgZXJyb3JlZCBmaWxlcyB0aGF0IHdlcmUgbm90IHRyYW5zY3JpYmVkLgogICAgIiIiCiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgR2V0IHRoZSBpbnB1dCBhdWRpbyBmaWxlcyB0byBkaWFyaXplOgogICAgaWYgaXNpbnN0YW5jZShkYXRhX3BhdGgsIHN0cik6CiAgICAgICAgZGF0YV9wYXRoID0gcGF0aGxpYi5QYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQogICAgICAgIGF1ZGlvX2ZpbGVzID0gX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9ZGF0YV9wYXRoKQogICAgZWxzZTogICMgU2hvdWxkIGJlIGEgbGlzdCBvZiBmaWxlcy4KICAgICAgICBhdWRpb19maWxlcyA9IGRhdGFfcGF0aAoKICAgICMgR2V0IHRoZSBIdWdnaW5nZmFjZSBhY2Nlc3MgdG9rZW46CiAgICBhY2Nlc3NfdG9rZW4gPSBfZ2V0X2FjY2Vzc190b2tlbihwYXJhbWV0ZXI9YWNjZXNzX3Rva2VuKQogICAgaWYgYWNjZXNzX3Rva2VuIGlzIE5vbmU6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgIkEgSHVnZ2luZ2ZhY2UgYWNjZXNzIHRva2VuIG11c3QgYmUgcHJvdmlkZWQgdG8gdXNlIGBweWFubm90ZS5hdWRpb2AgbW9kZWxzLiBBY2Nlc3MgdG9rZW4gY2FuIGJlIHBhc3NlZCAiCiAgICAgICAgICAgICJ2aWEgb25lIG9mIHRoZSBmb2xsb3dpbmcgb3B0aW9uczpcbiIKICAgICAgICAgICAgIiogVXNlIHRoZSBwYXJhbWV0ZXIgYGFjY2Vzc190b2tlbmAuXG4iCiAgICAgICAgICAgICIqIFNldCBhbiBlbnZpcm9ubWVudCB2YXJpYWJsZSBuYW1lZCAnSFVHR0lOR19GQUNFX0hVQl9UT0tFTicuXG4iCiAgICAgICAgICAgICIqIElmIHVzaW5nIE1MUnVuLCB5b3UgY2FuIHBhc3MgaXQgYXMgYSBzZWNyZXQgbmFtZWQgJ0hVR0dJTkdfRkFDRV9IVUJfVE9LRU4nLiIKICAgICAgICApCgogICAgIyBMb2FkIHRoZSBkaWFyaXphdGlvbiBwaXBlbGluZToKICAgIHBpcGVsaW5lID0gcHlhbm5vdGUuYXVkaW8uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKAogICAgICAgIGNoZWNrcG9pbnRfcGF0aD1tb2RlbF9uYW1lLCB1c2VfYXV0aF90b2tlbj1hY2Nlc3NfdG9rZW4KICAgICkKCiAgICAjIFNldCB0aGUgZGV2aWNlOgogICAgZGV2aWNlID0gZGV2aWNlIG9yICgiY3VkYSIgaWYgdG9yY2guY3VkYS5pc19hdmFpbGFibGUoKSBlbHNlICJjcHUiKQogICAgaWYgZGV2aWNlICE9ICJjcHUiOgogICAgICAgIHBpcGVsaW5lLnRvKHRvcmNoLmRldmljZShkZXZpY2UpKQoKICAgICMgUHJlcGFyZSB0aGUgc3VjY2Vzc2VzIGRhdGFmcmFtZSBhbmQgZXJyb3JzIGRpY3Rpb25hcnkgdG8gYmUgcmV0dXJuZWQ6CiAgICBkaWFyaXphdGlvbnMgPSB7fQogICAgZXJyb3JzID0ge30KCiAgICAjIFByZXBhcmUgdGhlIGRpYXJpemF0aW9uIGtleXdvcmQgYXJndW1lbnRzOgogICAgZGlhcml6ZV9rd2FyZ3MgPSB7fQogICAgaWYgc3BlYWtlcnNfbGFiZWxzOgogICAgICAgIGRpYXJpemVfa3dhcmdzWyJudW1fc3BlYWtlcnMiXSA9IGxlbihzcGVha2Vyc19sYWJlbHMpCiAgICBlbHNlOgogICAgICAgIGlmIG1pbmltdW1fc3BlYWtlcnM6CiAgICAgICAgICAgIGRpYXJpemVfa3dhcmdzWyJtaW5fc3BlYWtlcnMiXSA9IG1pbmltdW1fc3BlYWtlcnMKICAgICAgICBpZiBtYXhpbXVtX3NwZWFrZXJzOgogICAgICAgICAgICBkaWFyaXplX2t3YXJnc1sibWF4X3NwZWFrZXJzIl0gPSBtYXhpbXVtX3NwZWFrZXJzCgogICAgIyBHbyBvdmVyIHRoZSBhdWRpbyBmaWxlcyBhbmQgZGlhcml6ZToKICAgIGZvciBhdWRpb19maWxlIGluIHRxZG0oCiAgICAgICAgYXVkaW9fZmlsZXMsIGRlc2M9IkRpYXJpemluZyIsIHVuaXQ9ImZpbGUiLCBkaXNhYmxlPW5vdCB2ZXJib3NlCiAgICApOgogICAgICAgIHRyeToKICAgICAgICAgICAgIyBMb2FkIGF1ZGlvIGZpbGU6CiAgICAgICAgICAgIGF1ZGlvLCBzYW1wbGVfcmF0ZSA9IHRvcmNoYXVkaW8ubG9hZCh1cmk9YXVkaW9fZmlsZSwgY2hhbm5lbHNfZmlyc3Q9VHJ1ZSkKICAgICAgICAgICAgIyBHZXQgdGhlIGRpYXJpemF0aW9uIChpZiBwcm92aWRlZCk6CiAgICAgICAgICAgIGRpYXJpemF0aW9uc1thdWRpb19maWxlLm5hbWVdID0gX2RpYXJpemUoCiAgICAgICAgICAgICAgICBhdWRpbz1hdWRpbywKICAgICAgICAgICAgICAgIHNhbXBsZV9yYXRlPXNhbXBsZV9yYXRlLAogICAgICAgICAgICAgICAgcGlwZWxpbmU9cGlwZWxpbmUsCiAgICAgICAgICAgICAgICBzcGVha2Vyc19sYWJlbHM9c3BlYWtlcnNfbGFiZWxzLAogICAgICAgICAgICAgICAgc2VwYXJhdGVfYnlfY2hhbm5lbHM9c2VwYXJhdGVfYnlfY2hhbm5lbHMsCiAgICAgICAgICAgICAgICBzcGVha2VyX3ByZWZpeD1zcGVha2VyX3ByZWZpeCwKICAgICAgICAgICAgICAgIGRpYXJpemVfa3dhcmdzPWRpYXJpemVfa3dhcmdzLAogICAgICAgICAgICApCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBleGNlcHRpb246CiAgICAgICAgICAgICMgTm90ZSB0aGUgZXhjZXB0aW9uIGFzIGVycm9yIGluIHRoZSBkaWN0aW9uYXJ5OgogICAgICAgICAgICBpZiB2ZXJib3NlOgogICAgICAgICAgICAgICAgX0xPR0dFUi53YXJuaW5nKGYiRXJyb3IgaW4gZmlsZTogJ3thdWRpb19maWxlLm5hbWV9JyIpCiAgICAgICAgICAgIGVycm9yc1tzdHIoYXVkaW9fZmlsZS5uYW1lKV0gPSBzdHIoZXhjZXB0aW9uKQogICAgICAgICAgICBjb250aW51ZQoKICAgICMgUHJpbnQgdGhlIGhlYWQgb2YgdGhlIHByb2R1Y2VkIGRhdGFmcmFtZSBhbmQgcmV0dXJuOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJEb25lICh7bGVuKGRpYXJpemF0aW9ucyl9L3tsZW4oYXVkaW9fZmlsZXMpfSlcbiIpCiAgICByZXR1cm4gZGlhcml6YXRpb25zLCBlcnJvcnMKCgpkZWYgX2dldF9hdWRpb19maWxlcygKICAgIGRhdGFfcGF0aDogcGF0aGxpYi5QYXRoLAopIC0+IGxpc3RbcGF0aGxpYi5QYXRoXToKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICBhdWRpb19maWxlcyA9IGxpc3QoZGF0YV9wYXRoLmdsb2IoIiouKiIpKQogICAgZWxpZiBkYXRhX3BhdGguaXNfZmlsZSgpOgogICAgICAgIGF1ZGlvX2ZpbGVzID0gW2RhdGFfcGF0aF0KICAgIGVsc2U6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJVbnJlY29nbml6ZWQgZGF0YSBwYXRoLiBUaGUgcGFyYW1ldGVyIGBkYXRhX3BhdGhgIG11c3QgYmUgZWl0aGVyIGEgZGlyZWN0b3J5IHBhdGggb3IgYSBmaWxlIHBhdGguICIKICAgICAgICAgICAgZiJHaXZlbjoge3N0cihkYXRhX3BhdGgpfSAiCiAgICAgICAgKQoKICAgIHJldHVybiBhdWRpb19maWxlcwoKCmRlZiBfZ2V0X2FjY2Vzc190b2tlbihwYXJhbWV0ZXI6IHN0cikgLT4gc3RyOgogICAgIyBJZiBnaXZlbiBhcyBhIHBhcmFtZXRlciwgcmV0dXJuIGl0OgogICAgaWYgcGFyYW1ldGVyOgogICAgICAgIHJldHVybiBwYXJhbWV0ZXIKCiAgICAjIE90aGVyd2lzZSwgbG9vayBhdCB0aGUgZW52aXJvbm1lbnQgdmFyaWFibGU6CiAgICBlbnZpcm9ubWVudF92YXJpYWJsZSA9IG9zLmVudmlyb24uZ2V0KCJIVUdHSU5HX0ZBQ0VfSFVCX1RPS0VOIikKICAgIGlmIGVudmlyb25tZW50X3ZhcmlhYmxlOgogICAgICAgIHJldHVybiBlbnZpcm9ubWVudF92YXJpYWJsZQoKICAgICMgTGFzdGx5LCB0cnkgbG9vayBpbiB0aGUgc2V0IHNlY3JldHMgaW4gTUxSdW46CiAgICBzZWNyZXQgPSBOb25lCiAgICB0cnk6CiAgICAgICAgaW1wb3J0IG1scnVuCgogICAgICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJtbHJ1biIpCiAgICAgICAgc2VjcmV0ID0gY29udGV4dC5nZXRfc2VjcmV0KGtleT0iSFVHR0lOR19GQUNFX0hVQl9UT0tFTiIpCiAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvcjoKICAgICAgICBwYXNzCgogICAgcmV0dXJuIHNlY3JldAoKCmRlZiBfZGlhcml6ZSgKICAgIGF1ZGlvOiB0b3JjaC5UZW5zb3IsCiAgICBzYW1wbGVfcmF0ZTogaW50LAogICAgcGlwZWxpbmU6IHB5YW5ub3RlLmF1ZGlvLlBpcGVsaW5lLAogICAgc3BlYWtlcnNfbGFiZWxzOiBsaXN0W3N0cl0sCiAgICBzZXBhcmF0ZV9ieV9jaGFubmVsczogYm9vbCwKICAgIHNwZWFrZXJfcHJlZml4OiBzdHIsCiAgICBkaWFyaXplX2t3YXJnczogZGljdCwKKSAtPiBsaXN0W3R1cGxlW2Zsb2F0LCBmbG9hdCwgc3RyXV06CiAgICAjIElmIHRoZXJlIGlzIG5vIG5lZWQgZm9yIHNlcGFyYXRpb24gYnkgY2hhbm5lbHMsIHdlIGRpYXJpemUgYW5kIHJldHVybjoKICAgIGlmIG5vdCBzZXBhcmF0ZV9ieV9jaGFubmVsczoKICAgICAgICAjIERpYXJpemU6CiAgICAgICAgZGlhcml6YXRpb246IHB5YW5ub3RlLmNvcmUuQW5ub3RhdGlvbiA9IHBpcGVsaW5lKAogICAgICAgICAgICBmaWxlPXsid2F2ZWZvcm0iOiBhdWRpbywgInNhbXBsZV9yYXRlIjogc2FtcGxlX3JhdGV9LCAqKmRpYXJpemVfa3dhcmdzCiAgICAgICAgKQogICAgICAgICMgVmVyaWZ5IHNwZWFrZXJzIGxhYmVscyAoc2hvdWxkIG5vdCBmYWlsIGhlcmUgYXMgd2Ugc2V0IGBudW1fc3BlYWtlcnM9bGVuKHNwZWFrZXJzX2xhYmVscylgIHdoZW4gaW5mZXJyaW5nCiAgICAgICAgIyB0aHJvdWdoIHRoZSBwaXBlbGluZSk6CiAgICAgICAgaWYgc3BlYWtlcnNfbGFiZWxzOgogICAgICAgICAgICBnaXZlbl9zcGVha2VycyA9IGxlbihzcGVha2Vyc19sYWJlbHMpCiAgICAgICAgICAgIGZvdW5kX3NwZWFrZXJzID0gbGVuKHNldChkaWFyaXphdGlvbi5sYWJlbHMoKSkpCiAgICAgICAgICAgIGlmIGdpdmVuX3NwZWFrZXJzIDwgZm91bmRfc3BlYWtlcnM6CiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgICAgIGYiTm90IGVub3VnaCBgc3BlYWtlcnNfbGFiZWxzYCB3ZXJlIGdpdmVuLiBHb3Qge2dpdmVuX3NwZWFrZXJzfSBsYWJlbHMgYnV0IHRoZSBkaWFyaXphdGlvbiAiCiAgICAgICAgICAgICAgICAgICAgZiJyZWNvZ25pemVkIHtmb3VuZF9zcGVha2Vyc30gc3BlYWtlcnMuIgogICAgICAgICAgICAgICAgKQogICAgICAgICMgUmV0dXJuIGFzIGEgZGlhcml6YXRpb24gbGlzdCAtIGEgc29ydGVkIGxpc3Qgb2YgdHVwbGVzIG9mIHN0YXJ0IHRpbWUsIGVuZCB0aW1lIGFuZCBhIGxhYmVsICh0aGUgZGVmYXVsdCBsYWJlbAogICAgICAgICMgcmV0dXJuZWQgaXMgIlNQRUFLRVJfaSIgc28gd2UgdGFrZSBvbmx5IHRoZSBpbmRleCBvdXQgb2YgaXQpOgogICAgICAgIHJldHVybiBbCiAgICAgICAgICAgICgKICAgICAgICAgICAgICAgIHNlZ21lbnQuc3RhcnQsCiAgICAgICAgICAgICAgICBzZWdtZW50LmVuZCwKICAgICAgICAgICAgICAgIHNwZWFrZXJzX2xhYmVsc1tpbnQobGFiZWwuc3BsaXQoIl8iKVsxXSldCiAgICAgICAgICAgICAgICBpZiBzcGVha2Vyc19sYWJlbHMKICAgICAgICAgICAgICAgIGVsc2UgZiJ7c3BlYWtlcl9wcmVmaXh9e2ludChsYWJlbC5zcGxpdCgnXycpWzFdKX0iLAogICAgICAgICAgICApCiAgICAgICAgICAgIGZvciBzZWdtZW50LCB0cmFjaywgbGFiZWwgaW4gZGlhcml6YXRpb24uaXRlcnRyYWNrcyh5aWVsZF9sYWJlbD1UcnVlKQogICAgICAgIF0KCiAgICAjIFNlcGFyYXRlIHRvIGNoYW5uZWxzIGFuZCBkaWFyaXplICh3ZSBleHBlY3Qgb25seSBvbmUgc3BlYWtlciBwZXIgY2hhbm5lbCk6CiAgICBjaGFubmVsX2RpYXJpemF0aW9ucyA9IFsKICAgICAgICBfZGlhcml6ZSgKICAgICAgICAgICAgYXVkaW89YXVkaW9bY2hhbm5lbF0udW5zcXVlZXplKAogICAgICAgICAgICAgICAgMAogICAgICAgICAgICApLCAgIyBUYWtlIGNoYW5uZWwgYW5kIGFkZCBhIGNoYW5uZWwgZGltZW5zaW9uIHRvIGl0LgogICAgICAgICAgICBzYW1wbGVfcmF0ZT1zYW1wbGVfcmF0ZSwKICAgICAgICAgICAgcGlwZWxpbmU9cGlwZWxpbmUsCiAgICAgICAgICAgIHNwZWFrZXJzX2xhYmVscz1bCiAgICAgICAgICAgICAgICBzcGVha2Vyc19sYWJlbHNbY2hhbm5lbF0KICAgICAgICAgICAgXSwgICMgVGFrZSB0aGUgY2hhbm5lbCdzIGxhYmVsIG9ubHkuCiAgICAgICAgICAgIHNlcGFyYXRlX2J5X2NoYW5uZWxzPUZhbHNlLAogICAgICAgICAgICBzcGVha2VyX3ByZWZpeD1zcGVha2VyX3ByZWZpeCwKICAgICAgICAgICAgZGlhcml6ZV9rd2FyZ3M9eyJudW1fc3BlYWtlcnMiOiAxfSwgICMgU2V0IHRvIG9uZSBzcGVha2VyLgogICAgICAgICkKICAgICAgICBmb3IgY2hhbm5lbCBpbiByYW5nZShhdWRpby5zaGFwZVswXSkKICAgIF0KCiAgICAjIE1lcmdlIHRoZSBjaGFubmVsIGRpYXJpemF0aW9ucyBpbnRvIGEgc2luZ2xlIHNvcnRlZCBsaXN0OgogICAgcmV0dXJuIGxpc3QoaGVhcHEubWVyZ2UoKmNoYW5uZWxfZGlhcml6YXRpb25zKSkK requirements: - pyannote.audio - pyannote.core - torchaudio - tqdm + code_origin: '' base_image: mlrun/mlrun-gpu - origin_filename: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgoKaW1wb3J0IGhlYXBxCmltcG9ydCBsb2dnaW5nCmltcG9ydCBvcGVyYXRvcgppbXBvcnQgb3MKaW1wb3J0IHBhdGhsaWIKZnJvbSBmdW5jdG9vbHMgaW1wb3J0IHJlZHVjZSwgd3JhcHMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgRGljdCwgTGlzdCwgVHVwbGUsIFVuaW9uCgppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCBweWFubm90ZS5hdWRpbwppbXBvcnQgcHlhbm5vdGUuY29yZQppbXBvcnQgdG9yY2gKaW1wb3J0IHRvcmNoYXVkaW8KZnJvbSB0cWRtIGltcG9ydCB0cWRtCgojIEdldCB0aGUgZ2xvYmFsIGxvZ2dlcjoKX0xPR0dFUiA9IGxvZ2dpbmcuZ2V0TG9nZ2VyKCkKCgpkZWYgX2NoZWNrX21scnVuX2FuZF9vcGVuX21waSgpIC0+IFR1cGxlWyJtbHJ1bi5NTENsaWVudEN0eCIsICJtcGk0cHkuTVBJLkludHJhY29tbSJdOgogICAgaXNfbXBpID0gRmFsc2UKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KCiAgICAgICAgY29udGV4dCA9IG1scnVuLmdldF9vcl9jcmVhdGVfY3R4KG5hbWU9Im1scnVuIikKICAgICAgICBpc19tcGkgPSBjb250ZXh0LmxhYmVscy5nZXQoImtpbmQiLCAiam9iIikgPT0gIm1waWpvYiIKCiAgICAgICAgaWYgaXNfbXBpOgogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICBmcm9tIG1waTRweSBpbXBvcnQgTVBJCgogICAgICAgICAgICAgICAgcmV0dXJuIGNvbnRleHQsIE1QSS5DT01NX1dPUkxECiAgICAgICAgICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yIGFzIG1waTRweV9ub3RfZm91bmQ6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgICAgICAgICAiVG8gZGlzdHJpYnV0ZSB0aGUgZnVuY3Rpb24gdXNpbmcgTUxSdW4ncyAnbXBpam9iJyB5b3UgbmVlZCB0byBoYXZlIGBtcGk0cHlgIHBhY2thZ2UgaW4geW91ciAiCiAgICAgICAgICAgICAgICAgICAgImludGVycHJldGVyLiBQbGVhc2UgcnVuIGBwaXAgaW5zdGFsbCBtcGk0cHlgIGFuZCBtYWtlIHN1cmUgeW91IGhhdmUgb3Blbi1tcGkuIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgcmFpc2UgbXBpNHB5X25vdF9mb3VuZAogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHJldHVybiBjb250ZXh0LCBOb25lCiAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvciBhcyBtb2R1bGVfbm90X2ZvdW5kOgogICAgICAgIGlmIGlzX21waToKICAgICAgICAgICAgcmFpc2UgbW9kdWxlX25vdF9mb3VuZAogICAgcmV0dXJuIE5vbmUsIE5vbmUKCgpkZWYgb3Blbl9tcGlfaGFuZGxlcigKICAgIHdvcmtlcl9pbnB1dHM6IExpc3Rbc3RyXSwgcm9vdF93b3JrZXJfaW5wdXRzOiBEaWN0W3N0ciwgQW55XSA9IE5vbmUKKToKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBDaGVjayBmb3IgTUxSdW4gYW5kIE9wZW5NUEkgYXZhaWxhYmlsaXR5OgogICAgY29udGV4dCwgY29tbSA9IF9jaGVja19tbHJ1bl9hbmRfb3Blbl9tcGkoKQoKICAgICMgQ2hlY2sgaWYgTUxSdW4gaXMgYXZhaWxhYmxlLCBzZXQgdGhlIGdsb2JhbCBsb2dnZXIgdG8gTUxSdW4nczoKICAgIGlmIGNvbnRleHQ6CiAgICAgICAgX0xPR0dFUiA9IGNvbnRleHQubG9nZ2VyCgogICAgZGVmIGRlY29yYXRvcihoYW5kbGVyKToKICAgICAgICBpZiBjb21tIGlzIE5vbmUgb3IgY29tbS5HZXRfc2l6ZSgpID09IDE6CiAgICAgICAgICAgIHJldHVybiBoYW5kbGVyCgogICAgICAgIEB3cmFwcyhoYW5kbGVyKQogICAgICAgIGRlZiB3cmFwcGVyKCoqa3dhcmdzKToKICAgICAgICAgICAgIyBHZXQgdGhlIG9wZW4gbXBpIGVudmlyb25tZW50IHByb3BlcnRpZXM6CiAgICAgICAgICAgIHNpemUgPSBjb21tLkdldF9zaXplKCkKICAgICAgICAgICAgcmFuayA9IGNvbW0uR2V0X3JhbmsoKQoKICAgICAgICAgICAgIyBHaXZlIHRoZSBjb3JyZWN0IGNodW5rIG9mIHRoZSB3b3JrZXJzIGlucHV0czoKICAgICAgICAgICAgZm9yIHdvcmtlcl9pbnB1dCBpbiB3b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBrd2FyZ3Nbd29ya2VyX2lucHV0XQogICAgICAgICAgICAgICAgaWYgaW5wdXRfYXJndW1lbnQgaXMgTm9uZToKICAgICAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgc3RyKToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IF9nZXRfYXVkaW9fZmlsZXMoCiAgICAgICAgICAgICAgICAgICAgICAgIGRhdGFfcGF0aD1wYXRobGliLlBhdGgoaW5wdXRfYXJndW1lbnQpLmFic29sdXRlKCkKICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBpZiBsZW4oaW5wdXRfYXJndW1lbnQpIDwgc2l6ZToKICAgICAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgICAgICAgICBmIkNhbm5vdCBzcGxpdCB0aGUgaW5wdXQgJ3t3b3JrZXJfaW5wdXR9JyBvZiBsZW5ndGgge2xlbihpbnB1dF9hcmd1bWVudCl9IHRvIHtzaXplfSB3b3JrZXJzLiAiCiAgICAgICAgICAgICAgICAgICAgICAgIGYiUGxlYXNlIHJlZHVjZSB0aGUgYW1vdW50IG9mIHdvcmtlcnMgZm9yIHRoaXMgaW5wdXQuIgogICAgICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGV2ZW5fY2h1bmtfc2l6ZSA9IGxlbihpbnB1dF9hcmd1bWVudCkgLy8gc2l6ZQogICAgICAgICAgICAgICAgY2h1bmtfc3RhcnQgPSByYW5rICogZXZlbl9jaHVua19zaXplCiAgICAgICAgICAgICAgICBjaHVua19lbmQgPSAoCiAgICAgICAgICAgICAgICAgICAgKHJhbmsgKyAxKSAqIGV2ZW5fY2h1bmtfc2l6ZQogICAgICAgICAgICAgICAgICAgIGlmIHJhbmsgKyAxIDwgc2l6ZQogICAgICAgICAgICAgICAgICAgIGVsc2UgbGVuKGlucHV0X2FyZ3VtZW50KQogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICBmIlJhbmsgI3tyYW5rfTogUHJvY2Vzc2luZyBpbnB1dCBjaHVuayBvZiAne3dvcmtlcl9pbnB1dH0nICIKICAgICAgICAgICAgICAgICAgICBmImZyb20gaW5kZXgge2NodW5rX3N0YXJ0fSB0byB7Y2h1bmtfZW5kfS4iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBpZiBpc2luc3RhbmNlKGlucHV0X2FyZ3VtZW50LCBsaXN0KToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IGlucHV0X2FyZ3VtZW50W2NodW5rX3N0YXJ0OmNodW5rX2VuZF0KICAgICAgICAgICAgICAgIGVsaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgcGQuRGF0YUZyYW1lKToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IGlucHV0X2FyZ3VtZW50Lmlsb2NbY2h1bmtfc3RhcnQ6Y2h1bmtfZW5kOiwgOl0KICAgICAgICAgICAgICAgIGt3YXJnc1t3b3JrZXJfaW5wdXRdID0gaW5wdXRfYXJndW1lbnQKCiAgICAgICAgICAgICMgU2V0IHRoZSByb290IHdvcmtlciBvbmx5IGFyZ3VtZW50czoKICAgICAgICAgICAgaWYgcmFuayA9PSAwIGFuZCByb290X3dvcmtlcl9pbnB1dHM6CiAgICAgICAgICAgICAgICBrd2FyZ3MudXBkYXRlKHJvb3Rfd29ya2VyX2lucHV0cykKCiAgICAgICAgICAgICMgUnVuIHRoZSB3b3JrZXI6CiAgICAgICAgICAgIG91dHB1dCA9IGhhbmRsZXIoKiprd2FyZ3MpCgogICAgICAgICAgICAjIFNlbmQgdGhlIG91dHB1dCB0byB0aGUgcm9vdCByYW5rIChyYW5rICMwKToKICAgICAgICAgICAgb3V0cHV0ID0gY29tbS5nYXRoZXIob3V0cHV0LCByb290PTApCiAgICAgICAgICAgIGlmIHJhbmsgPT0gMDoKICAgICAgICAgICAgICAgICMgSm9pbiB0aGUgb3V0cHV0czoKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkNvbGxlY3RpbmcgZGF0YSBmcm9tIHdvcmtlcnMgdG8gcm9vdCB3b3JrZXIuIikKICAgICAgICAgICAgICAgIGRpYXJpemF0aW9uX2RpY3Rpb25hcnkgPSByZWR1Y2UoCiAgICAgICAgICAgICAgICAgICAgb3BlcmF0b3IuaW9yLCBbZGlhIGZvciBkaWEsIF8gaW4gb3V0cHV0XSwge30KICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGVycm9yc19kaWN0aW9uYXJ5ID0gcmVkdWNlKG9wZXJhdG9yLmlvciwgW2VyciBmb3IgXywgZXJyIGluIG91dHB1dF0sIHt9KQogICAgICAgICAgICAgICAgcmV0dXJuIGRpYXJpemF0aW9uX2RpY3Rpb25hcnksIGVycm9yc19kaWN0aW9uYXJ5CiAgICAgICAgICAgIHJldHVybiBOb25lCgogICAgICAgIHJldHVybiB3cmFwcGVyCgogICAgcmV0dXJuIGRlY29yYXRvcgoKCkBvcGVuX21waV9oYW5kbGVyKHdvcmtlcl9pbnB1dHM9WyJkYXRhX3BhdGgiXSwgcm9vdF93b3JrZXJfaW5wdXRzPXsidmVyYm9zZSI6IFRydWV9KQpkZWYgZGlhcml6ZSgKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBMaXN0W3N0cl1dLAogICAgbW9kZWxfbmFtZTogc3RyID0gInB5YW5ub3RlL3NwZWFrZXItZGlhcml6YXRpb24tMy4wIiwKICAgIGFjY2Vzc190b2tlbjogc3RyID0gTm9uZSwKICAgIGRldmljZTogc3RyID0gTm9uZSwKICAgIHNwZWFrZXJzX2xhYmVsczogTGlzdFtzdHJdID0gTm9uZSwKICAgIHNwZWFrZXJfcHJlZml4OiBzdHIgPSAic3BlYWtlcl8iLAogICAgc2VwYXJhdGVfYnlfY2hhbm5lbHM6IGJvb2wgPSBGYWxzZSwKICAgIG1pbmltdW1fc3BlYWtlcnM6IGludCA9IE5vbmUsCiAgICBtYXhpbXVtX3NwZWFrZXJzOiBpbnQgPSBOb25lLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopIC0+IFR1cGxlW0RpY3Rbc3RyLCBMaXN0W1R1cGxlW2Zsb2F0LCBmbG9hdCwgc3RyXV1dLCBEaWN0W3N0ciwgc3RyXV06CiAgICAiIiIKICAgIFBlcmZvcm0gc3BlZWNoIGRpYXJpemF0aW9uIG9uIGdpdmVuIGF1ZGlvIGZpbGVzIHVzaW5nIHB5YW5ub3RlLWF1ZGlvIChodHRwczovL2dpdGh1Yi5jb20vcHlhbm5vdGUvcHlhbm5vdGUtYXVkaW8pLgogICAgVGhlIGVuZCByZXN1bHQgaXMgYSBkaWN0aW9uYXJ5IHdpdGggdGhlIGZpbGUgbmFtZXMgYXMga2V5cyBhbmQgdGhlaXIgZGlhcml6YXRpb24gYXMgdmFsdWUuIEEgZGlhcml6YXRpb24gaXMgYSBsaXN0CiAgICBvZiB0dXBsZXM6IChzdGFydCwgZW5kLCBzcGVha2VyX2xhYmVsKS4KCiAgICBUbyB1c2UgdGhlIGBweWFubm90ZS5hdWRpb2AgbW9kZWxzIHlvdSBtdXN0IHBhc3MgYSBIdWdnaW5nZmFjZSB0b2tlbiBhbmQgZ2V0IGFjY2VzcyB0byB0aGUgcmVxdWlyZWQgbW9kZWxzLiBUaGUKICAgIHRva2VuIGNhbiBiZSBwYXNzZWQgaW4gb25lIG9mIHRoZSBmb2xsb3dpbmcgb3B0aW9uczoKCiAgICAqIFVzZSB0aGUgcGFyYW1ldGVyIGBhY2Nlc3NfdG9rZW5gLgogICAgKiBTZXQgYW4gZW52aXJvbm1lbnQgdmFyaWFibGUgbmFtZWQgIkhVR0dJTkdfRkFDRV9IVUJfVE9LRU4iLgogICAgKiBJZiB1c2luZyBNTFJ1biwgeW91IGNhbiBwYXNzIGl0IGFzIGEgc2VjcmV0IG5hbWVkICJIVUdHSU5HX0ZBQ0VfSFVCX1RPS0VOIi4KCiAgICBUbyBnZXQgYWNjZXNzIHRvIHRoZSBtb2RlbHMgb24gSHVnZ2luZ2ZhY2UsIHZpc2l0IHRoZWlyIHBhZ2UuIEZvciBleGFtcGxlLCB0byB1c2UgdGhlIGRlZmF1bHQgZGlhcml6YXRpb24gbW9kZWwgc2V0CiAgICBpbiB0aGlzIGZ1bmN0aW9uICgicHlhbm5vdGUvc3BlYWtlci1kaWFyaXphdGlvbi0zLjAiKSwgeW91IG5lZWQgYWNjZXNzIGZvciB0aGVzZSB0d28gbW9kZWxzOgoKICAgICogaHR0cHM6Ly9odWdnaW5nZmFjZS5jby9weWFubm90ZS9zZWdtZW50YXRpb24tMy4wCiAgICAqIGh0dHBzOi8vaHVnZ2luZ2ZhY2UuY28vcHlhbm5vdGUvc3BlYWtlci1kaWFyaXphdGlvbi0zLjAKCiAgICBOb3RlOiBUbyBjb250cm9sIHRoZSByZWNvZ25pemVkIHNwZWFrZXJzIGluIHRoZSBkaWFyaXphdGlvbiBvdXRwdXQgeW91IGNhbiBjaG9vc2Ugb25lIG9mIHRoZSBmb2xsb3dpbmcgbWV0aG9kczoKCiAgICAqIEZvciBhIGtub3duIHNwZWFrZXJzIGFtb3VudCwgeW91IG1heSBzZXQgc3BlYWtlciBsYWJlbHMgdmlhIHRoZSBgc3BlYWtlcnNfbGFiZWxzYCBwYXJhbWV0ZXIgdGhhdCB3aWxsIGJlIHVzZWQgaW4KICAgICAgdGhlIG9yZGVyIG9mIHNwZWFraW5nIGluIHRoZSBhdWRpbyAoZmlyc3QgcGVyc29uIHNwZWFraW5nIGJlIHRoZSBmaXJzdCBsYWJlbCBpbiB0aGUgbGlzdCkuIEluIGFkZGl0aW9uLCB5b3UgY2FuIGRvCiAgICAgIGRpYXJpemF0aW9uIHBlciBjaGFubmVsIChzZXR0aW5nIHRoZSBwYXJhbWV0ZXIgYHNlcGFyYXRlX2J5X2NoYW5uZWxzYCB0byBUcnVlKS4gRWFjaCBsYWJlbCB3aWxsIGJlIGFzc2lnbmVkIHRvIGEKICAgICAgc3BlY2lmaWMgY2hhbm5lbCBieSBvcmRlciAoZmlyc3QgbGFiZWwgdG8gY2hhbm5lbCAwLCBzZWNvbmQgbGFiZWwgdG8gY2hhbm5lbCAxIGFuZCBzbyBvbikuIE5vdGljZSwgdGhpcyB3aWxsCiAgICAgIGluY3JlYXNlIHJ1bnRpbWUuCiAgICAqIEZvciB1bmtub3duIHNwZWFrZXJzIGFtb3VudCwgeW91IGNhbiBzZXQgdGhlIGBzcGVha2VyX3ByZWZpeGAgcGFyYW1ldGVyIHRvIGFkZCBhIHByZWZpeCBmb3IgZWFjaCBzcGVha2VyIG51bWJlci4KICAgICAgWW91IGNhbiBhbHNvIGhlbHAgdGhlIGRpYXJpemF0aW9uIGJ5IHNldHRpbmcgdGhlIHNwZWFrZXJzIHJhbmdlIHZpYSB0aGUgYHNwZWFrZXJzX2Ftb3VudF9yYW5nZWAgcGFyYW1ldGVyLgoKICAgIDpwYXJhbSBkYXRhX3BhdGg6ICAgICAgICAgICAgQSBkaXJlY3Rvcnkgb2YgdGhlIGF1ZGlvIGZpbGVzLCBhIHNpbmdsZSBmaWxlIG9yIGEgbGlzdCBvZiBmaWxlcyB0byB0cmFuc2NyaWJlLgogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICBPbmUgb2YgdGhlIG9mZmljaWFsIGRpYXJpemF0aW9uIG1vZGVsIG5hbWVzIChyZWZlcnJlZCBhcyBkaWFyaXphdGlvbiBwaXBlbGluZXMpIG9mCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBweWFubm90ZS5hdWRpb2AgSHVnZ2luZ2ZhY2UgcGFnZS4gRGVmYXVsdDogInB5YW5ub3RlL3NwZWFrZXItZGlhcml6YXRpb24tMy4wIi4KICAgIDpwYXJhbSBhY2Nlc3NfdG9rZW46ICAgICAgICAgQW4gYWNjZXNzIHRva2VuIHRvIHBhc3MgZm9yIHVzaW5nIHRoZSBgcHlhbm5vdGUuYXVkaW9gIG1vZGVscy4gSWYgbm90IHByb3ZpZGVkLCBpdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIGxvb2tpbmcgZm9yIHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZSAiSFVHR0lOR19GQUNFX0hVQl9UT0tFTiIuIElmIE1MUnVuIGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGF2YWlsYWJsZSwgaXQgd2lsbCBsb29rIGZvciBhIHNlY3JldCAiSFVHR0lOR19GQUNFX0hVQl9UT0tFTiIuCiAgICA6cGFyYW0gZGV2aWNlOiAgICAgICAgICAgICAgIERldmljZSB0byBsb2FkIHRoZSBtb2RlbC4gQ2FuIGJlIG9uZSBvZiB7ImN1ZGEiLCAiY3B1In0uIERlZmF1bHQgd2lsbCBwcmVmZXIgImN1ZGEiIGlmCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGF2YWlsYWJsZS4KICAgIDpwYXJhbSBzcGVha2Vyc19sYWJlbHM6ICAgICAgTGFiZWxzIHRvIHVzZSBmb3IgdGhlIHJlY29nbml6ZWQgc3BlYWtlcnMuIERlZmF1bHQ6IG51bWVyaWMgbGFiZWxzICgwLCAxLCAuLi4pLgogICAgOnBhcmFtIHNlcGFyYXRlX2J5X2NoYW5uZWxzOiBJZiBlYWNoIHNwZWFrZXIgaXMgc3BlYWtpbmcgaW4gYSBzZXBhcmF0ZSBjaGFubmVsLCB5b3UgY2FuIGRpYXJpemUgZWFjaCBjaGFubmVsIGFuZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBjb21iaW5lIHRoZSByZXN1bHQgaW50byBhIHNpbmdsZSBkaWFyaXphdGlvbi4gRWFjaCBsYWJlbCBzZXQgaW4gdGhlIGBzcGVha2Vyc19sYWJlbHNgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHBhcmFtZXRlciB3aWxsIGJlIGFzc2lnbmVkIHRvIGEgc3BlY2lmaWMgY2hhbm5lbCBieSBvcmRlci4KICAgIDpwYXJhbSBzcGVha2VyX3ByZWZpeDogICAgICAgQSBwcmVmaXggdG8gYWRkIGZvciB0aGUgc3BlYWtlcnMgbGFiZWxzLiBUaGlzIHBhcmFtZXRlciBpcyBpZ25vcmVkIGlmCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBzcGVha2Vyc19sYWJlbHNgIGlzIG5vdCBOb25lLiBEZWZhdWx0OiAic3BlYWtlciIuCiAgICA6cGFyYW0gbWluaW11bV9zcGVha2VyczogICAgIFNldCB0aGUgbWluaW11bSBleHBlY3RlZCBhbW91bnQgb2Ygc3BlYWtlcnMgdG8gYmUgaW4gdGhlIGF1ZGlvIGZpbGVzLiBUaGlzIHBhcmFtZXRlciBpcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpZ25vcmVkIGlmIGBzcGVha2Vyc19sYWJlbHNgIGlzIG5vdCBOb25lLgogICAgOnBhcmFtIG1heGltdW1fc3BlYWtlcnM6ICAgICBTZXQgdGhlIG1heGltdW0gZXhwZWN0ZWQgYW1vdW50IG9mIHNwZWFrZXJzIHRvIGJlIGluIHRoZSBhdWRpbyBmaWxlcy4gVGhpcyBwYXJhbWV0ZXIgaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaWdub3JlZCBpZiBgc3BlYWtlcnNfbGFiZWxzYCBpcyBub3QgTm9uZS4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICAgV2hldGhlciB0byBwcmVzZW50IGxvZ3Mgb2YgYSBwcm9ncmVzcyBiYXIgYW5kIGVycm9ycy4gRGVmYXVsdDogVHJ1ZS4KCiAgICA6cmV0dXJuczogQSB0dXBsZSBvZjoKCiAgICAgICAgICAgICAgKiBTcGVlY2ggZGlhcml6YXRpb24gZGljdGlvbmFyeS4KICAgICAgICAgICAgICAqIEEgZGljdGlvbmFyeSBvZiBlcnJvcmVkIGZpbGVzIHRoYXQgd2VyZSBub3QgdHJhbnNjcmliZWQuCiAgICAiIiIKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBHZXQgdGhlIGlucHV0IGF1ZGlvIGZpbGVzIHRvIGRpYXJpemU6CiAgICBpZiBpc2luc3RhbmNlKGRhdGFfcGF0aCwgc3RyKToKICAgICAgICBkYXRhX3BhdGggPSBwYXRobGliLlBhdGgoZGF0YV9wYXRoKS5hYnNvbHV0ZSgpCiAgICAgICAgYXVkaW9fZmlsZXMgPSBfZ2V0X2F1ZGlvX2ZpbGVzKGRhdGFfcGF0aD1kYXRhX3BhdGgpCiAgICBlbHNlOiAgIyBTaG91bGQgYmUgYSBsaXN0IG9mIGZpbGVzLgogICAgICAgIGF1ZGlvX2ZpbGVzID0gZGF0YV9wYXRoCgogICAgIyBHZXQgdGhlIEh1Z2dpbmdmYWNlIGFjY2VzcyB0b2tlbjoKICAgIGFjY2Vzc190b2tlbiA9IF9nZXRfYWNjZXNzX3Rva2VuKHBhcmFtZXRlcj1hY2Nlc3NfdG9rZW4pCiAgICBpZiBhY2Nlc3NfdG9rZW4gaXMgTm9uZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAiQSBIdWdnaW5nZmFjZSBhY2Nlc3MgdG9rZW4gbXVzdCBiZSBwcm92aWRlZCB0byB1c2UgYHB5YW5ub3RlLmF1ZGlvYCBtb2RlbHMuIEFjY2VzcyB0b2tlbiBjYW4gYmUgcGFzc2VkICIKICAgICAgICAgICAgInZpYSBvbmUgb2YgdGhlIGZvbGxvd2luZyBvcHRpb25zOlxuIgogICAgICAgICAgICAiKiBVc2UgdGhlIHBhcmFtZXRlciBgYWNjZXNzX3Rva2VuYC5cbiIKICAgICAgICAgICAgIiogU2V0IGFuIGVudmlyb25tZW50IHZhcmlhYmxlIG5hbWVkICdIVUdHSU5HX0ZBQ0VfSFVCX1RPS0VOJy5cbiIKICAgICAgICAgICAgIiogSWYgdXNpbmcgTUxSdW4sIHlvdSBjYW4gcGFzcyBpdCBhcyBhIHNlY3JldCBuYW1lZCAnSFVHR0lOR19GQUNFX0hVQl9UT0tFTicuIgogICAgICAgICkKCiAgICAjIExvYWQgdGhlIGRpYXJpemF0aW9uIHBpcGVsaW5lOgogICAgcGlwZWxpbmUgPSBweWFubm90ZS5hdWRpby5QaXBlbGluZS5mcm9tX3ByZXRyYWluZWQoCiAgICAgICAgY2hlY2twb2ludF9wYXRoPW1vZGVsX25hbWUsIHVzZV9hdXRoX3Rva2VuPWFjY2Vzc190b2tlbgogICAgKQoKICAgICMgU2V0IHRoZSBkZXZpY2U6CiAgICBkZXZpY2UgPSBkZXZpY2Ugb3IgKCJjdWRhIiBpZiB0b3JjaC5jdWRhLmlzX2F2YWlsYWJsZSgpIGVsc2UgImNwdSIpCiAgICBpZiBkZXZpY2UgIT0gImNwdSI6CiAgICAgICAgcGlwZWxpbmUudG8odG9yY2guZGV2aWNlKGRldmljZSkpCgogICAgIyBQcmVwYXJlIHRoZSBzdWNjZXNzZXMgZGF0YWZyYW1lIGFuZCBlcnJvcnMgZGljdGlvbmFyeSB0byBiZSByZXR1cm5lZDoKICAgIGRpYXJpemF0aW9ucyA9IHt9CiAgICBlcnJvcnMgPSB7fQoKICAgICMgUHJlcGFyZSB0aGUgZGlhcml6YXRpb24ga2V5d29yZCBhcmd1bWVudHM6CiAgICBkaWFyaXplX2t3YXJncyA9IHt9CiAgICBpZiBzcGVha2Vyc19sYWJlbHM6CiAgICAgICAgZGlhcml6ZV9rd2FyZ3NbIm51bV9zcGVha2VycyJdID0gbGVuKHNwZWFrZXJzX2xhYmVscykKICAgIGVsc2U6CiAgICAgICAgaWYgbWluaW11bV9zcGVha2VyczoKICAgICAgICAgICAgZGlhcml6ZV9rd2FyZ3NbIm1pbl9zcGVha2VycyJdID0gbWluaW11bV9zcGVha2VycwogICAgICAgIGlmIG1heGltdW1fc3BlYWtlcnM6CiAgICAgICAgICAgIGRpYXJpemVfa3dhcmdzWyJtYXhfc3BlYWtlcnMiXSA9IG1heGltdW1fc3BlYWtlcnMKCiAgICAjIEdvIG92ZXIgdGhlIGF1ZGlvIGZpbGVzIGFuZCBkaWFyaXplOgogICAgZm9yIGF1ZGlvX2ZpbGUgaW4gdHFkbSgKICAgICAgICBhdWRpb19maWxlcywgZGVzYz0iRGlhcml6aW5nIiwgdW5pdD0iZmlsZSIsIGRpc2FibGU9bm90IHZlcmJvc2UKICAgICk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICAjIExvYWQgYXVkaW8gZmlsZToKICAgICAgICAgICAgYXVkaW8sIHNhbXBsZV9yYXRlID0gdG9yY2hhdWRpby5sb2FkKHVyaT1hdWRpb19maWxlLCBjaGFubmVsc19maXJzdD1UcnVlKQogICAgICAgICAgICAjIEdldCB0aGUgZGlhcml6YXRpb24gKGlmIHByb3ZpZGVkKToKICAgICAgICAgICAgZGlhcml6YXRpb25zW2F1ZGlvX2ZpbGUubmFtZV0gPSBfZGlhcml6ZSgKICAgICAgICAgICAgICAgIGF1ZGlvPWF1ZGlvLAogICAgICAgICAgICAgICAgc2FtcGxlX3JhdGU9c2FtcGxlX3JhdGUsCiAgICAgICAgICAgICAgICBwaXBlbGluZT1waXBlbGluZSwKICAgICAgICAgICAgICAgIHNwZWFrZXJzX2xhYmVscz1zcGVha2Vyc19sYWJlbHMsCiAgICAgICAgICAgICAgICBzZXBhcmF0ZV9ieV9jaGFubmVscz1zZXBhcmF0ZV9ieV9jaGFubmVscywKICAgICAgICAgICAgICAgIHNwZWFrZXJfcHJlZml4PXNwZWFrZXJfcHJlZml4LAogICAgICAgICAgICAgICAgZGlhcml6ZV9rd2FyZ3M9ZGlhcml6ZV9rd2FyZ3MsCiAgICAgICAgICAgICkKICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGV4Y2VwdGlvbjoKICAgICAgICAgICAgIyBOb3RlIHRoZSBleGNlcHRpb24gYXMgZXJyb3IgaW4gdGhlIGRpY3Rpb25hcnk6CiAgICAgICAgICAgIGlmIHZlcmJvc2U6CiAgICAgICAgICAgICAgICBfTE9HR0VSLndhcm5pbmcoZiJFcnJvciBpbiBmaWxlOiAne2F1ZGlvX2ZpbGUubmFtZX0nIikKICAgICAgICAgICAgZXJyb3JzW3N0cihhdWRpb19maWxlLm5hbWUpXSA9IHN0cihleGNlcHRpb24pCiAgICAgICAgICAgIGNvbnRpbnVlCgogICAgIyBQcmludCB0aGUgaGVhZCBvZiB0aGUgcHJvZHVjZWQgZGF0YWZyYW1lIGFuZCByZXR1cm46CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkRvbmUgKHtsZW4oZGlhcml6YXRpb25zKX0ve2xlbihhdWRpb19maWxlcyl9KVxuIikKICAgIHJldHVybiBkaWFyaXphdGlvbnMsIGVycm9ycwoKCmRlZiBfZ2V0X2F1ZGlvX2ZpbGVzKAogICAgZGF0YV9wYXRoOiBwYXRobGliLlBhdGgsCikgLT4gTGlzdFtwYXRobGliLlBhdGhdOgogICAgIyBDaGVjayBpZiB0aGUgcGF0aCBpcyBvZiBhIGRpcmVjdG9yeSBvciBhIGZpbGU6CiAgICBpZiBkYXRhX3BhdGguaXNfZGlyKCk6CiAgICAgICAgIyBHZXQgYWxsIGZpbGVzIGluc2lkZSB0aGUgZGlyZWN0b3J5OgogICAgICAgIGF1ZGlvX2ZpbGVzID0gbGlzdChkYXRhX3BhdGguZ2xvYigiKi4qIikpCiAgICBlbGlmIGRhdGFfcGF0aC5pc19maWxlKCk6CiAgICAgICAgYXVkaW9fZmlsZXMgPSBbZGF0YV9wYXRoXQogICAgZWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlVucmVjb2duaXplZCBkYXRhIHBhdGguIFRoZSBwYXJhbWV0ZXIgYGRhdGFfcGF0aGAgbXVzdCBiZSBlaXRoZXIgYSBkaXJlY3RvcnkgcGF0aCBvciBhIGZpbGUgcGF0aC4gIgogICAgICAgICAgICBmIkdpdmVuOiB7c3RyKGRhdGFfcGF0aCl9ICIKICAgICAgICApCgogICAgcmV0dXJuIGF1ZGlvX2ZpbGVzCgoKZGVmIF9nZXRfYWNjZXNzX3Rva2VuKHBhcmFtZXRlcjogc3RyKSAtPiBzdHI6CiAgICAjIElmIGdpdmVuIGFzIGEgcGFyYW1ldGVyLCByZXR1cm4gaXQ6CiAgICBpZiBwYXJhbWV0ZXI6CiAgICAgICAgcmV0dXJuIHBhcmFtZXRlcgoKICAgICMgT3RoZXJ3aXNlLCBsb29rIGF0IHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZToKICAgIGVudmlyb25tZW50X3ZhcmlhYmxlID0gb3MuZW52aXJvbi5nZXQoIkhVR0dJTkdfRkFDRV9IVUJfVE9LRU4iKQogICAgaWYgZW52aXJvbm1lbnRfdmFyaWFibGU6CiAgICAgICAgcmV0dXJuIGVudmlyb25tZW50X3ZhcmlhYmxlCgogICAgIyBMYXN0bHksIHRyeSBsb29rIGluIHRoZSBzZXQgc2VjcmV0cyBpbiBNTFJ1bjoKICAgIHNlY3JldCA9IE5vbmUKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KCiAgICAgICAgY29udGV4dCA9IG1scnVuLmdldF9vcl9jcmVhdGVfY3R4KG5hbWU9Im1scnVuIikKICAgICAgICBzZWNyZXQgPSBjb250ZXh0LmdldF9zZWNyZXQoa2V5PSJIVUdHSU5HX0ZBQ0VfSFVCX1RPS0VOIikKICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgICAgIHBhc3MKCiAgICByZXR1cm4gc2VjcmV0CgoKZGVmIF9kaWFyaXplKAogICAgYXVkaW86IHRvcmNoLlRlbnNvciwKICAgIHNhbXBsZV9yYXRlOiBpbnQsCiAgICBwaXBlbGluZTogcHlhbm5vdGUuYXVkaW8uUGlwZWxpbmUsCiAgICBzcGVha2Vyc19sYWJlbHM6IExpc3Rbc3RyXSwKICAgIHNlcGFyYXRlX2J5X2NoYW5uZWxzOiBib29sLAogICAgc3BlYWtlcl9wcmVmaXg6IHN0ciwKICAgIGRpYXJpemVfa3dhcmdzOiBkaWN0LAopIC0+IExpc3RbVHVwbGVbZmxvYXQsIGZsb2F0LCBzdHJdXToKICAgICMgSWYgdGhlcmUgaXMgbm8gbmVlZCBmb3Igc2VwYXJhdGlvbiBieSBjaGFubmVscywgd2UgZGlhcml6ZSBhbmQgcmV0dXJuOgogICAgaWYgbm90IHNlcGFyYXRlX2J5X2NoYW5uZWxzOgogICAgICAgICMgRGlhcml6ZToKICAgICAgICBkaWFyaXphdGlvbjogcHlhbm5vdGUuY29yZS5Bbm5vdGF0aW9uID0gcGlwZWxpbmUoCiAgICAgICAgICAgIGZpbGU9eyJ3YXZlZm9ybSI6IGF1ZGlvLCAic2FtcGxlX3JhdGUiOiBzYW1wbGVfcmF0ZX0sICoqZGlhcml6ZV9rd2FyZ3MKICAgICAgICApCiAgICAgICAgIyBWZXJpZnkgc3BlYWtlcnMgbGFiZWxzIChzaG91bGQgbm90IGZhaWwgaGVyZSBhcyB3ZSBzZXQgYG51bV9zcGVha2Vycz1sZW4oc3BlYWtlcnNfbGFiZWxzKWAgd2hlbiBpbmZlcnJpbmcKICAgICAgICAjIHRocm91Z2ggdGhlIHBpcGVsaW5lKToKICAgICAgICBpZiBzcGVha2Vyc19sYWJlbHM6CiAgICAgICAgICAgIGdpdmVuX3NwZWFrZXJzID0gbGVuKHNwZWFrZXJzX2xhYmVscykKICAgICAgICAgICAgZm91bmRfc3BlYWtlcnMgPSBsZW4oc2V0KGRpYXJpemF0aW9uLmxhYmVscygpKSkKICAgICAgICAgICAgaWYgZ2l2ZW5fc3BlYWtlcnMgPCBmb3VuZF9zcGVha2VyczoKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICAgICAgICAgZiJOb3QgZW5vdWdoIGBzcGVha2Vyc19sYWJlbHNgIHdlcmUgZ2l2ZW4uIEdvdCB7Z2l2ZW5fc3BlYWtlcnN9IGxhYmVscyBidXQgdGhlIGRpYXJpemF0aW9uICIKICAgICAgICAgICAgICAgICAgICBmInJlY29nbml6ZWQge2ZvdW5kX3NwZWFrZXJzfSBzcGVha2Vycy4iCiAgICAgICAgICAgICAgICApCiAgICAgICAgIyBSZXR1cm4gYXMgYSBkaWFyaXphdGlvbiBsaXN0IC0gYSBzb3J0ZWQgbGlzdCBvZiB0dXBsZXMgb2Ygc3RhcnQgdGltZSwgZW5kIHRpbWUgYW5kIGEgbGFiZWwgKHRoZSBkZWZhdWx0IGxhYmVsCiAgICAgICAgIyByZXR1cm5lZCBpcyAiU1BFQUtFUl9pIiBzbyB3ZSB0YWtlIG9ubHkgdGhlIGluZGV4IG91dCBvZiBpdCk6CiAgICAgICAgcmV0dXJuIFsKICAgICAgICAgICAgKAogICAgICAgICAgICAgICAgc2VnbWVudC5zdGFydCwKICAgICAgICAgICAgICAgIHNlZ21lbnQuZW5kLAogICAgICAgICAgICAgICAgc3BlYWtlcnNfbGFiZWxzW2ludChsYWJlbC5zcGxpdCgiXyIpWzFdKV0KICAgICAgICAgICAgICAgIGlmIHNwZWFrZXJzX2xhYmVscwogICAgICAgICAgICAgICAgZWxzZSBmIntzcGVha2VyX3ByZWZpeH17aW50KGxhYmVsLnNwbGl0KCdfJylbMV0pfSIsCiAgICAgICAgICAgICkKICAgICAgICAgICAgZm9yIHNlZ21lbnQsIHRyYWNrLCBsYWJlbCBpbiBkaWFyaXphdGlvbi5pdGVydHJhY2tzKHlpZWxkX2xhYmVsPVRydWUpCiAgICAgICAgXQoKICAgICMgU2VwYXJhdGUgdG8gY2hhbm5lbHMgYW5kIGRpYXJpemUgKHdlIGV4cGVjdCBvbmx5IG9uZSBzcGVha2VyIHBlciBjaGFubmVsKToKICAgIGNoYW5uZWxfZGlhcml6YXRpb25zID0gWwogICAgICAgIF9kaWFyaXplKAogICAgICAgICAgICBhdWRpbz1hdWRpb1tjaGFubmVsXS51bnNxdWVlemUoCiAgICAgICAgICAgICAgICAwCiAgICAgICAgICAgICksICAjIFRha2UgY2hhbm5lbCBhbmQgYWRkIGEgY2hhbm5lbCBkaW1lbnNpb24gdG8gaXQuCiAgICAgICAgICAgIHNhbXBsZV9yYXRlPXNhbXBsZV9yYXRlLAogICAgICAgICAgICBwaXBlbGluZT1waXBlbGluZSwKICAgICAgICAgICAgc3BlYWtlcnNfbGFiZWxzPVsKICAgICAgICAgICAgICAgIHNwZWFrZXJzX2xhYmVsc1tjaGFubmVsXQogICAgICAgICAgICBdLCAgIyBUYWtlIHRoZSBjaGFubmVsJ3MgbGFiZWwgb25seS4KICAgICAgICAgICAgc2VwYXJhdGVfYnlfY2hhbm5lbHM9RmFsc2UsCiAgICAgICAgICAgIHNwZWFrZXJfcHJlZml4PXNwZWFrZXJfcHJlZml4LAogICAgICAgICAgICBkaWFyaXplX2t3YXJncz17Im51bV9zcGVha2VycyI6IDF9LCAgIyBTZXQgdG8gb25lIHNwZWFrZXIuCiAgICAgICAgKQogICAgICAgIGZvciBjaGFubmVsIGluIHJhbmdlKGF1ZGlvLnNoYXBlWzBdKQogICAgXQoKICAgICMgTWVyZ2UgdGhlIGNoYW5uZWwgZGlhcml6YXRpb25zIGludG8gYSBzaW5nbGUgc29ydGVkIGxpc3Q6CiAgICByZXR1cm4gbGlzdChoZWFwcS5tZXJnZSgqY2hhbm5lbF9kaWFyaXphdGlvbnMpKQo= - default_handler: diarize + filename: pyannote_audio.py entry_points: open_mpi_handler: - name: open_mpi_handler - has_varargs: false - lineno: 61 parameters: - name: worker_inputs - type: List[str] + type: list[str] - name: root_worker_inputs - type: Dict[str, Any] + type: dict[str, Any] default: null - has_kwargs: false + name: open_mpi_handler doc: '' - decorator: - name: decorator + has_kwargs: false has_varargs: false - lineno: 73 + lineno: 61 + decorator: parameters: - name: handler - has_kwargs: false + name: decorator doc: '' + has_kwargs: false + has_varargs: false + lineno: 73 wrapper: name: wrapper + doc: '' + has_kwargs: true has_varargs: false lineno: 78 - has_kwargs: true - doc: '' diarize: - name: diarize - has_varargs: false - lineno: 139 outputs: - doc: 'A tuple of:' - type: Tuple[Dict[str, List[Tuple[float, float, str]]], Dict[str, str]] + type: tuple[dict[str, list[tuple[float, float, str]]], dict[str, str]] parameters: - name: data_path - type: Union[str, List[str]] doc: A directory of the audio files, a single file or a list of files to transcribe. - name: model_name type: str @@ -69,7 +71,7 @@ spec: prefer "cuda" if available. default: null - name: speakers_labels - type: List[str] + type: list[str] doc: 'Labels to use for the recognized speakers. Default: numeric labels (0, 1, ...).' default: null @@ -99,7 +101,7 @@ spec: type: bool doc: 'Whether to present logs of a progress bar and errors. Default: True.' default: false - has_kwargs: false + name: diarize doc: "Perform speech diarization on given audio files using pyannote-audio (https://github.com/pyannote/pyannote-audio).\n\ The end result is a dictionary with the file names as keys and their diarization\ \ as value. A diarization is a list\nof tuples: (start, end, speaker_label).\n\ @@ -123,11 +125,9 @@ spec: \ you can set the `speaker_prefix` parameter to add a prefix for each speaker\ \ number.\n You can also help the diarization by setting the speakers range\ \ via the `speakers_amount_range` parameter." + has_kwargs: false + has_varargs: false + lineno: 139 + command: '' description: pyannote's speech diarization of audio files -metadata: - name: pyannote-audio - tag: '' - categories: - - deep-learning - - audio -verbose: false + default_handler: diarize diff --git a/functions/src/pyannote_audio/pyannote_audio.py b/functions/src/pyannote_audio/pyannote_audio.py index 6271da6ae..bb097a750 100644 --- a/functions/src/pyannote_audio/pyannote_audio.py +++ b/functions/src/pyannote_audio/pyannote_audio.py @@ -18,7 +18,7 @@ import os import pathlib from functools import reduce, wraps -from typing import Any, Dict, List, Tuple, Union +from typing import Any import pandas as pd import pyannote.audio @@ -31,7 +31,7 @@ _LOGGER = logging.getLogger() -def _check_mlrun_and_open_mpi() -> Tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intracomm"]: +def _check_mlrun_and_open_mpi() -> tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intracomm"]: is_mpi = False try: import mlrun @@ -59,7 +59,7 @@ def _check_mlrun_and_open_mpi() -> Tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intrac def open_mpi_handler( - worker_inputs: List[str], root_worker_inputs: Dict[str, Any] = None + worker_inputs: list[str], root_worker_inputs: dict[str, Any] = None ): global _LOGGER @@ -137,17 +137,17 @@ def wrapper(**kwargs): @open_mpi_handler(worker_inputs=["data_path"], root_worker_inputs={"verbose": True}) def diarize( - data_path: Union[str, List[str]], + data_path: str | list[str], model_name: str = "pyannote/speaker-diarization-3.0", access_token: str = None, device: str = None, - speakers_labels: List[str] = None, + speakers_labels: list[str] = None, speaker_prefix: str = "speaker_", separate_by_channels: bool = False, minimum_speakers: int = None, maximum_speakers: int = None, verbose: bool = False, -) -> Tuple[Dict[str, List[Tuple[float, float, str]]], Dict[str, str]]: +) -> tuple[dict[str, list[tuple[float, float, str]]], dict[str, str]]: """ Perform speech diarization on given audio files using pyannote-audio (https://github.com/pyannote/pyannote-audio). The end result is a dictionary with the file names as keys and their diarization as value. A diarization is a list @@ -277,7 +277,7 @@ def diarize( def _get_audio_files( data_path: pathlib.Path, -) -> List[pathlib.Path]: +) -> list[pathlib.Path]: # Check if the path is of a directory or a file: if data_path.is_dir(): # Get all files inside the directory: @@ -320,11 +320,11 @@ def _diarize( audio: torch.Tensor, sample_rate: int, pipeline: pyannote.audio.Pipeline, - speakers_labels: List[str], + speakers_labels: list[str], separate_by_channels: bool, speaker_prefix: str, diarize_kwargs: dict, -) -> List[Tuple[float, float, str]]: +) -> list[tuple[float, float, str]]: # If there is no need for separation by channels, we diarize and return: if not separate_by_channels: # Diarize: diff --git a/functions/src/question_answering/function.yaml b/functions/src/question_answering/function.yaml index 21f741aa8..afcf893a2 100644 --- a/functions/src/question_answering/function.yaml +++ b/functions/src/question_answering/function.yaml @@ -1,83 +1,56 @@ metadata: - name: question-answering tag: '' + name: question-answering categories: - genai verbose: false kind: job spec: - command: '' - default_handler: answer_questions + image: '' + disable_auto_mount: false build: origin_filename: '' - base_image: mlrun/mlrun + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgZW51bQppbXBvcnQgbG9nZ2luZwppbXBvcnQgb3BlcmF0b3IKaW1wb3J0IHBhdGhsaWIKZnJvbSBjb2xsZWN0aW9ucyBpbXBvcnQgQ291bnRlcgpmcm9tIGZ1bmN0b29scyBpbXBvcnQgcmVkdWNlLCB3cmFwcwpmcm9tIHR5cGluZyBpbXBvcnQgQW55CgppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCB0cmFuc2Zvcm1lcnMKZnJvbSB0cWRtIGltcG9ydCB0cWRtCgojIEdldCB0aGUgZ2xvYmFsIGxvZ2dlcjoKX0xPR0dFUiA9IGxvZ2dpbmcuZ2V0TG9nZ2VyKCkKCgpkZWYgX2NoZWNrX21scnVuX2FuZF9vcGVuX21waSgpIC0+IHR1cGxlWyJtbHJ1bi5NTENsaWVudEN0eCIsICJtcGk0cHkuTVBJLkludHJhY29tbSJdOgogICAgZ2xvYmFsIF9MT0dHRVIKCiAgICBpc19tcGkgPSBGYWxzZQogICAgdHJ5OgogICAgICAgIGltcG9ydCBtbHJ1bgoKICAgICAgICBjb250ZXh0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9jdHgobmFtZT0ibWxydW4iKQogICAgICAgIF9MT0dHRVIgPSBjb250ZXh0LmxvZ2dlcgogICAgICAgIGlzX21waSA9IGNvbnRleHQubGFiZWxzLmdldCgia2luZCIsICJqb2IiKSA9PSAibXBpam9iIgoKICAgICAgICBpZiBpc19tcGk6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIGZyb20gbXBpNHB5IGltcG9ydCBNUEkKCiAgICAgICAgICAgICAgICByZXR1cm4gY29udGV4dCwgTVBJLkNPTU1fV09STEQKICAgICAgICAgICAgZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3IgYXMgbXBpNHB5X25vdF9mb3VuZDoKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICAgICAgICAgICJUbyBkaXN0cmlidXRlIHRoZSBmdW5jdGlvbiB1c2luZyBNTFJ1bidzICdtcGlqb2InIHlvdSBuZWVkIHRvIGhhdmUgYG1waTRweWAgcGFja2FnZSBpbiB5b3VyICIKICAgICAgICAgICAgICAgICAgICAiaW50ZXJwcmV0ZXIuIFBsZWFzZSBydW4gYHBpcCBpbnN0YWxsIG1waTRweWAgYW5kIG1ha2Ugc3VyZSB5b3UgaGF2ZSBvcGVuLW1waS4iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBtcGk0cHlfbm90X2ZvdW5kCiAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvciBhcyBtb2R1bGVfbm90X2ZvdW5kOgogICAgICAgIGlmIGlzX21waToKICAgICAgICAgICAgcmFpc2UgbW9kdWxlX25vdF9mb3VuZAogICAgcmV0dXJuIE5vbmUsIE5vbmUKCgpkZWYgb3Blbl9tcGlfaGFuZGxlcigKICAgIHdvcmtlcl9pbnB1dHM6IGxpc3Rbc3RyXSwgcm9vdF93b3JrZXJfaW5wdXRzOiBkaWN0W3N0ciwgQW55XSA9IE5vbmUKKToKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBDaGVjayBmb3IgTUxSdW4gYW5kIE9wZW5NUEkgYXZhaWxhYmlsaXR5OgogICAgY29udGV4dCwgY29tbSA9IF9jaGVja19tbHJ1bl9hbmRfb3Blbl9tcGkoKQoKICAgIGRlZiBkZWNvcmF0b3IoaGFuZGxlcik6CiAgICAgICAgaWYgY29tbSBpcyBOb25lIG9yIGNvbW0uR2V0X3NpemUoKSA9PSAxOgogICAgICAgICAgICByZXR1cm4gaGFuZGxlcgoKICAgICAgICBAd3JhcHMoaGFuZGxlcikKICAgICAgICBkZWYgd3JhcHBlcigqKmt3YXJncyk6CiAgICAgICAgICAgICMgR2V0IHRoZSBvcGVuIG1waSBlbnZpcm9ubWVudCBwcm9wZXJ0aWVzOgogICAgICAgICAgICBzaXplID0gY29tbS5HZXRfc2l6ZSgpCiAgICAgICAgICAgIHJhbmsgPSBjb21tLkdldF9yYW5rKCkKCiAgICAgICAgICAgICMgR2l2ZSB0aGUgY29ycmVjdCBjaHVuayBvZiB0aGUgd29ya2VycyBpbnB1dHM6CiAgICAgICAgICAgIGZvciB3b3JrZXJfaW5wdXQgaW4gd29ya2VyX2lucHV0czoKICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0ga3dhcmdzW3dvcmtlcl9pbnB1dF0KICAgICAgICAgICAgICAgIGlmIGlucHV0X2FyZ3VtZW50IGlzIE5vbmU6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIHN0cik6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBfZ2V0X3RleHRfZmlsZXMoCiAgICAgICAgICAgICAgICAgICAgICAgIGRhdGFfcGF0aD1wYXRobGliLlBhdGgoaW5wdXRfYXJndW1lbnQpLmFic29sdXRlKCkKICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBpZiBsZW4oaW5wdXRfYXJndW1lbnQpIDwgc2l6ZToKICAgICAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgICAgICAgICBmIkNhbm5vdCBzcGxpdCB0aGUgaW5wdXQgJ3t3b3JrZXJfaW5wdXR9JyBvZiBsZW5ndGgge2xlbihpbnB1dF9hcmd1bWVudCl9IHRvIHtzaXplfSB3b3JrZXJzLiAiCiAgICAgICAgICAgICAgICAgICAgICAgIGYiUGxlYXNlIHJlZHVjZSB0aGUgYW1vdW50IG9mIHdvcmtlcnMgZm9yIHRoaXMgaW5wdXQuIgogICAgICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGV2ZW5fY2h1bmtfc2l6ZSA9IGxlbihpbnB1dF9hcmd1bWVudCkgLy8gc2l6ZQogICAgICAgICAgICAgICAgY2h1bmtfc3RhcnQgPSByYW5rICogZXZlbl9jaHVua19zaXplCiAgICAgICAgICAgICAgICBjaHVua19lbmQgPSAoCiAgICAgICAgICAgICAgICAgICAgKHJhbmsgKyAxKSAqIGV2ZW5fY2h1bmtfc2l6ZQogICAgICAgICAgICAgICAgICAgIGlmIHJhbmsgKyAxIDwgc2l6ZQogICAgICAgICAgICAgICAgICAgIGVsc2UgbGVuKGlucHV0X2FyZ3VtZW50KQogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICBmIlJhbmsgI3tyYW5rfTogUHJvY2Vzc2luZyBpbnB1dCBjaHVuayBvZiAne3dvcmtlcl9pbnB1dH0nICIKICAgICAgICAgICAgICAgICAgICBmImZyb20gaW5kZXgge2NodW5rX3N0YXJ0fSB0byB7Y2h1bmtfZW5kfS4iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBpZiBpc2luc3RhbmNlKGlucHV0X2FyZ3VtZW50LCBsaXN0KToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IGlucHV0X2FyZ3VtZW50W2NodW5rX3N0YXJ0OmNodW5rX2VuZF0KICAgICAgICAgICAgICAgIGVsaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgcGQuRGF0YUZyYW1lKToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IGlucHV0X2FyZ3VtZW50Lmlsb2NbY2h1bmtfc3RhcnQ6Y2h1bmtfZW5kOiwgOl0KICAgICAgICAgICAgICAgIGt3YXJnc1t3b3JrZXJfaW5wdXRdID0gaW5wdXRfYXJndW1lbnQKCiAgICAgICAgICAgICMgU2V0IHRoZSByb290IHdvcmtlciBvbmx5IGFyZ3VtZW50czoKICAgICAgICAgICAgaWYgcmFuayA9PSAwIGFuZCByb290X3dvcmtlcl9pbnB1dHM6CiAgICAgICAgICAgICAgICBrd2FyZ3MudXBkYXRlKHJvb3Rfd29ya2VyX2lucHV0cykKCiAgICAgICAgICAgICMgUnVuIHRoZSB3b3JrZXI6CiAgICAgICAgICAgIG91dHB1dCA9IGhhbmRsZXIoKiprd2FyZ3MpCgogICAgICAgICAgICAjIFNlbmQgdGhlIG91dHB1dCB0byB0aGUgcm9vdCByYW5rIChyYW5rICMwKToKICAgICAgICAgICAgb3V0cHV0ID0gY29tbS5nYXRoZXIob3V0cHV0LCByb290PTApCiAgICAgICAgICAgIGlmIHJhbmsgPT0gMDoKICAgICAgICAgICAgICAgICMgSm9pbiB0aGUgb3V0cHV0czoKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkNvbGxlY3RpbmcgZGF0YSBmcm9tIHdvcmtlcnMgdG8gcm9vdCB3b3JrZXIuIikKICAgICAgICAgICAgICAgIGRhdGFmcmFtZSA9IHBkLmNvbmNhdChvYmpzPVtkZiBmb3IgZGYsIF8gaW4gb3V0cHV0XSwgYXhpcz0wKQogICAgICAgICAgICAgICAgZXJyb3JzX2RpY3Rpb25hcnkgPSByZWR1Y2Uob3BlcmF0b3IuaW9yLCBbZXJyIGZvciBfLCBlcnIgaW4gb3V0cHV0XSwge30pCiAgICAgICAgICAgICAgICByZXR1cm4gZGF0YWZyYW1lLCBlcnJvcnNfZGljdGlvbmFyeQogICAgICAgICAgICByZXR1cm4gTm9uZQoKICAgICAgICByZXR1cm4gd3JhcHBlcgoKICAgIHJldHVybiBkZWNvcmF0b3IKCgpAb3Blbl9tcGlfaGFuZGxlcih3b3JrZXJfaW5wdXRzPVsiZGF0YV9wYXRoIl0sIHJvb3Rfd29ya2VyX2lucHV0cz17InZlcmJvc2UiOiBUcnVlfSkKZGVmIGFuc3dlcl9xdWVzdGlvbnMoCiAgICBkYXRhX3BhdGg6IHN0ciB8IGxpc3Rbc3RyXSwKICAgIG1vZGVsX25hbWU6IHN0ciwKICAgIHF1ZXN0aW9uczogbGlzdFtzdHJdIHwgbGlzdFtsaXN0W3N0cl1dLAogICAgZGV2aWNlX21hcDogc3RyIHwgZGljdCA9IE5vbmUsCiAgICBtb2RlbF9rd2FyZ3M6IGRpY3QgPSBOb25lLAogICAgYXV0b19ncHRxX2V4bGxhbWFfbWF4X2lucHV0X2xlbmd0aDogaW50ID0gTm9uZSwKICAgIHRva2VuaXplcl9uYW1lOiBzdHIgPSBOb25lLAogICAgdG9rZW5pemVyX2t3YXJnczogZGljdCA9IE5vbmUsCiAgICB0ZXh0X3dyYXBwZXI6IHN0ciB8IGxpc3Rbc3RyXSA9ICIiLAogICAgcXVlc3Rpb25zX3dyYXBwZXI6IHN0ciB8IGxpc3Rbc3RyXSA9ICIiLAogICAgZ2VuZXJhdGlvbl9jb25maWc6IGRpY3QgfCBsaXN0W2RpY3RdID0gTm9uZSwKICAgIHF1ZXN0aW9uc19jb25maWc6IGRpY3QgfCBsaXN0W2RpY3RdID0gTm9uZSwKICAgIGJhdGNoX3NpemU6IGludCA9IDEsCiAgICBxdWVzdGlvbnNfY29sdW1uczogbGlzdFtzdHJdID0gTm9uZSwKICAgIHZlcmJvc2U6IGJvb2wgPSBGYWxzZSwKKSAtPiB0dXBsZVtwZC5EYXRhRnJhbWUsIGRpY3RdOgogICAgIiIiCiAgICBBbnN3ZXIgcXVlc3Rpb25zIHdpdGggYSBjb250ZXh0IHRvIHRoZSBnaXZlbiB0ZXh0IGZpbGVzIGNvbnRlbnRzIGJ5IGEgcHJldHJhaW5lZCBMTE0gbW9kZWwuIEVhY2ggdGV4dCBmaWxlIHdpbGwgaGF2ZQogICAgdGhlIGZvbGxvd2luZyBwcm9tcHQgYnVpbHQ6CgogICAgc3RhcnQgb2YgYHRleHRfd3JhcHBlcmAKICAgIDx0ZXh0IGZpbGUgY29udGVudD4KICAgIGVuZCBvZiBgdGV4dF93cmFwcGVyYAoKICAgIHN0YXJ0IG9mIGBxdWVzdGlvbnNfd3JhcHBlcmAKICAgIDEuIDxxdWVzdGlvbnNbMF0+CiAgICAyLiA8cXVlc3Rpb25zWzFdPgogICAgLi4uCiAgICBuLiA8cXVlc3Rpb25zW24tMV0+CiAgICBlbmQgb2YgYHF1ZXN0aW9uc193cmFwcGVyYAoKICAgIDpwYXJhbSBkYXRhX3BhdGg6ICAgICAgICAgICAgICAgICAgICAgICAgICBBIHBhdGggdG8gYSBkaXJlY3Rvcnkgb2YgdGV4dCBmaWxlcyBvciBhIHBhdGggdG8gYSB0ZXh0IGZpbGUgdG8gYXNrCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcXVlc3Rpb25zIGFib3V0LgogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSBwcmUtdHJhaW5lZCBtb2RlbCBuYW1lIGZyb20gdGhlIGh1Z2dpbmdmYWNlIGh1YiB0byB1c2UgZm9yIGFza2luZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHF1ZXN0aW9ucy4KICAgIDpwYXJhbSBxdWVzdGlvbnM6ICAgICAgICAgICAgICAgICAgICAgICAgICBUaGUgcXVlc3Rpb25zIHRvIGFzay4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBBIGxpc3Qgb2YgbGlzdHMgb2YgcXVlc3Rpb25zIHRvIGFzayBwZXIgdGV4dCBmaWxlLCBhbmQgZGV2aWRlZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJ5IHF1ZXN0aW9uIGdyb3VwcywgdGhlIGdyb3VwcyBjYW4gYmUgZHRlcm1haW5lZCBieSBzaXplIChpbiBvcmRlciB0bwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGF2b2lkIGxhcmdlIGlucHV0cyB0byB0aGUgbGxtKSBvciBieSBxdWVzdGlvbmluZyBtZXRob2QKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAocmVndWxhciBvciBwb2xsIGxpa2UgcXVlc3Rpb25pbmcpLgogICAgOnBhcmFtIGRldmljZV9tYXA6ICAgICAgICAgICAgICAgICAgICAgICAgIEEgbWFwIHRvIHVzZSBmb3IgbG9hZGluZyB0aGUgbW9kZWwgb24gbXVsdGlwbGUgZGV2aWNlcy4KICAgIDpwYXJhbSBtb2RlbF9rd2FyZ3M6ICAgICAgICAgICAgICAgICAgICAgICBLZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIGZvciBsb2FkaW5nIHRoZSBtb2RlbCB1c2luZyBIdWdnaW5nRmFjZSdzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYHRyYW5zZm9ybWVycy5BdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWRgIGZ1bmN0aW9uLgogICAgOnBhcmFtIGF1dG9fZ3B0cV9leGxsYW1hX21heF9pbnB1dF9sZW5ndGg6IEZvciBBdXRvR1BUUSBtb2RlbHMgdG8gc2V0IGFuZCBleHRlbmQgdGhlIG1vZGVsJ3MgaW5wdXQgYnVmZmVyIHNpemUuCiAgICA6cGFyYW0gdG9rZW5pemVyX25hbWU6ICAgICAgICAgICAgICAgICAgICAgVGhlIHRva2VuaXplciBuYW1lIGZyb20gdGhlIGh1Z2dpbmdmYWNlIGh1YiB0byB1c2UuIElmIG5vdCBnaXZlbiwgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9kZWwgbmFtZSB3aWxsIGJlIHVzZWQuCiAgICA6cGFyYW0gdG9rZW5pemVyX2t3YXJnczogICAgICAgICAgICAgICAgICAgS2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyBmb3IgbG9hZGluZyB0aGUgdG9rZW5pemVyIHVzaW5nIEh1Z2dpbmdGYWNlJ3MKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBgdHJhbnNmb3JtZXJzLkF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkYCBmdW5jdGlvbi4KICAgIDpwYXJhbSB0ZXh0X3dyYXBwZXI6ICAgICAgICAgICAgICAgICAgICAgICBBIHdyYXBwZXIgZm9yIHRoZSBmaWxlJ3MgdGV4dC4gV2lsbCBiZSBhZGRlZCBhdCB0aGUgc3RhcnQgb2YgdGhlIHByb21wdC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBNdXN0IGhhdmUgYSBwbGFjZWhvbGRlciAoJ3t9JykgZm9yIHRoZSB0ZXh0IG9mIHRoZSBmaWxlLgogICAgOnBhcmFtIHF1ZXN0aW9uc193cmFwcGVyOiAgICAgICAgICAgICAgICAgIEEgd3JhcHBlciBmb3IgdGhlIHF1ZXN0aW9ucyByZWNlaXZlZC4gV2lsbCBiZSBhZGRlZCBhZnRlciB0aGUgdGV4dAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHdyYXBwZXIgaW4gdGhlIHByb21wdCB0ZW1wbGF0ZS4gTXVzdCBoYXZlIGEgcGxhY2Vob2xkZXIgKCd7fScpIGZvciB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBxdWVzdGlvbnMuCiAgICA6cGFyYW0gZ2VuZXJhdGlvbl9jb25maWc6ICAgICAgICAgICAgICAgICAgSHVnZ2luZ0ZhY2UncyBgR2VuZXJhdGlvbkNvbmZpZ2Aga2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBgZ2VuZXJhdGVgIG1ldGhvZC4KICAgIDpwYXJhbSBxdWVzdGlvbnNfY29uZmlnOiAgICAgICAgICAgICAgICAgICBBIGRpY3Rpb25hcnkgb3IgbGlzdCBvZiBkaWN0aW9uYXJpZXMgY29udGFpbmluZyBzcGVjaWZpYyB3YXlzIHRvIGFuc3dlcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHF1ZXN0aW9ucyAodXNpbmcgYSBwb2xsIGZvciBleGFtcGxlKSwgZWFjaCBkaWN0aW9uYXJ5IGluIHRoZSBsaXN0IGlzIGZvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGNvcnJlc3BvbmRpbmcgcXVlc3Rpb24gZ3JvdXAgYW5kIGRldGVybWluZXMgdGhlIHF1ZXN0aW9uIGFza2luZyBtZXRob2QKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb3Igc2FpZCBncm91cC4KICAgIDpwYXJhbSBiYXRjaF9zaXplOiAgICAgICAgICAgICAgICAgICAgICAgICBCYXRjaCBzaXplIGZvciBpbmZlcmVuY2UuCiAgICA6cGFyYW0gcXVlc3Rpb25zX2NvbHVtbnM6ICAgICAgICAgICAgICAgICAgQ29sdW1ucyB0byB1c2UgZm9yIHRoZSBkYXRhZnJhbWUgcmV0dXJuZWQuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICAgICAgICAgICAgICAgICAgV2hldGhlciB0byBwcmVzZW50IGxvZ3Mgb2YgYSBwcm9ncmVzcyBiYXIgYW5kIGVycm9ycy4gRGVmYXVsdDogVHJ1ZS4KCgogICAgOnJldHVybnM6IEEgdHVwbGUgb2Y6CgogICAgICAgICAgICAgICogQSBkYXRhZnJhbWUgZGF0YXNldCBvZiB0aGUgcXVlc3Rpb25zIGFuc3dlcnMuCiAgICAgICAgICAgICAgKiBBIGRpY3Rpb25hcnkgb2YgZXJyb3JlZCBmaWxlcyB0aGF0IHdlcmUgbm90IGluZmVycmVkIG9yIHdlcmUgbm90IGFuc3dlcmVkIHByb3Blcmx5LgogICAgIiIiCiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgU2V0IGNvbmZpZ3MgdG8gZW1wdHkgZGljdCBpZiBub3QgZ2l2ZW46CiAgICBpZiBnZW5lcmF0aW9uX2NvbmZpZyBpcyBOb25lOgogICAgICAgIGdlbmVyYXRpb25fY29uZmlnID0ge30KICAgIGlmIHF1ZXN0aW9uc19jb25maWcgaXMgTm9uZToKICAgICAgICBxdWVzdGlvbnNfY29uZmlnID0ge30KCiAgICAjIEdldCB0aGUgaW5wdXQgdGV4dCBmaWxlcyB0byBxdWVzdGlvbjoKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJDb2xsZWN0aW5nIHRleHQgZmlsZXMuIikKICAgIGlmIGlzaW5zdGFuY2UoZGF0YV9wYXRoLCBzdHIpOgogICAgICAgIGRhdGFfcGF0aCA9IHBhdGhsaWIuUGF0aChkYXRhX3BhdGgpLmFic29sdXRlKCkKICAgICAgICB0ZXh0X2ZpbGVzID0gX2dldF90ZXh0X2ZpbGVzKGRhdGFfcGF0aD1kYXRhX3BhdGgpCiAgICBlbHNlOgogICAgICAgIHRleHRfZmlsZXMgPSBkYXRhX3BhdGgKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiQ29sbGVjdGVkIHtsZW4odGV4dF9maWxlcyl9IHRleHQgZmlsZXMuIikKCiAgICAjIEdldCB0aGUgcHJvbXB0IHRlbXBsYXRlOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNyZWF0aW5nIHByb21wdCB0ZW1wbGF0ZS4iKQoKICAgICMgT3JnYW5pemUgcXVlc3Rpb25zIGFzIGEgbGlzdCBvZiBsaXN0LCBhbmQgY291bnQgbnVtYmVyIG9mIHN1Yi1saXN0cyBmb3IgZnV0dXJlIHVzZQogICAgbnVtYmVyX29mX3F1ZXN0aW9uX2dyb3VwcyA9IDEgaWYgaXNpbnN0YW5jZShxdWVzdGlvbnNbMF0sIHN0cikgZWxzZSBsZW4ocXVlc3Rpb25zKQogICAgcXVlc3Rpb25zID0gX3RvX2dyb3VwX2xpc3QoCiAgICAgICAgYXJndW1lbnRfdmFsdWU9cXVlc3Rpb25zLAogICAgICAgIGFyZ3VtZW50X25hbWU9InF1ZXN0aW9ucyIsCiAgICAgICAgbGVuZ3RoPW51bWJlcl9vZl9xdWVzdGlvbl9ncm91cHMsCiAgICApCgogICAgIyBPcmdhbml6ZSBwcm9tcHQgcGFydHMgYXQgcHJvcGVyIGxlbmd0aAogICAgdGV4dF93cmFwcGVyID0gX3RvX2dyb3VwX2xpc3QoCiAgICAgICAgYXJndW1lbnRfdmFsdWU9dGV4dF93cmFwcGVyLAogICAgICAgIGFyZ3VtZW50X25hbWU9InRleHRfd3JhcHBlciIsCiAgICAgICAgbGVuZ3RoPW51bWJlcl9vZl9xdWVzdGlvbl9ncm91cHMsCiAgICApCiAgICBxdWVzdGlvbnNfd3JhcHBlciA9IF90b19ncm91cF9saXN0KAogICAgICAgIGFyZ3VtZW50X3ZhbHVlPXF1ZXN0aW9uc193cmFwcGVyLAogICAgICAgIGFyZ3VtZW50X25hbWU9InF1ZXN0aW9uc193cmFwcGVyIiwKICAgICAgICBsZW5ndGg9bnVtYmVyX29mX3F1ZXN0aW9uX2dyb3VwcywKICAgICkKCiAgICAjIENyZWF0ZSBhIGxpc3Qgb2YgcHJvbXB0IGFjY29yZGluZyB0byBnaXZlbiBwYXJ0cyBhbmQgcXVlc3Rpb25zCiAgICBwcm9tcHRfdGVtcGxhdGUgPSBbXQogICAgcXVlc3Rpb25zID0gcXVlc3Rpb25zIGlmIGlzaW5zdGFuY2UocXVlc3Rpb25zWzBdLCBsaXN0KSBlbHNlIFtxdWVzdGlvbnNdCgogICAgIyBCdWlsZCBhbGwgcHJvbXB0cwogICAgZm9yIGkgaW4gcmFuZ2UobnVtYmVyX29mX3F1ZXN0aW9uX2dyb3Vwcyk6CiAgICAgICAgcHJvbXB0X3RlbXBsYXRlLmFwcGVuZCgKICAgICAgICAgICAgX2dldF9wcm9tcHRfdGVtcGxhdGUoCiAgICAgICAgICAgICAgICB0ZXh0X3dyYXBwZXI9dGV4dF93cmFwcGVyW2ldLAogICAgICAgICAgICAgICAgcXVlc3Rpb25zX3dyYXBwZXI9cXVlc3Rpb25zX3dyYXBwZXJbaV0sCiAgICAgICAgICAgICAgICBxdWVzdGlvbnM9cXVlc3Rpb25zW2ldLAogICAgICAgICAgICApCiAgICAgICAgKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJQcm9tcHQgdGVtcGxhdGUgY3JlYXRlZDpcblxue3Byb21wdF90ZW1wbGF0ZX1cbiIpCgogICAgIyBHZXQgdGhlIHRvdGFsIGFtb3VudCBvZiBxdWVzdGlvbnM6CiAgICBxdWVzdGlvbnNfYW1vdW50ID0gc3VtKFtsZW4oc3VibGlzdCkgZm9yIHN1Ymxpc3QgaW4gcXVlc3Rpb25zXSkKCiAgICAjIEdldCB0aGUgcXVlc3Rpb25zIGNvbHVtbnM6CiAgICBxdWVzdGlvbnNfY29sdW1ucyA9IHF1ZXN0aW9uc19jb2x1bW5zIG9yIFsKICAgICAgICBmInF7aX0iIGZvciBpIGluIHJhbmdlKDEsIHF1ZXN0aW9uc19hbW91bnQgKyAxKQogICAgXQoKICAgICMgQ2hlY2sgaWYgd2UgaGF2ZSB0aGUgY29ycmVjdCBhbW91bnQgb2YgcXVlc3Rpb25zIGNvbHVtbnM6CiAgICBpZiBsZW4ocXVlc3Rpb25zX2NvbHVtbnMpICE9IHF1ZXN0aW9uc19hbW91bnQ6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJUaGUgcHJvdmlkZWQgcXVlc3Rpb25zIGNvbHVtbnMgbGVuZ3RoICh7bGVuKHF1ZXN0aW9uc19jb2x1bW5zKX0pICIKICAgICAgICAgICAgZiJkb2VzIG5vdCBtYXRjaCB0aGUgcXVlc3Rpb25zIGFtb3VudCAoe3F1ZXN0aW9uc19hbW91bnR9KSIKICAgICAgICApCgogICAgIyBMb2FkIHRoZSBnZW5lcmF0aW9uIGNvbmZpZzoKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJMb2FkaW5nIGdlbmVyYXRpb24gY29uZmlndXJhdGlvbi4iKQogICAgZ2VuZXJhdGlvbl9jb25maWcgPSBbCiAgICAgICAgdHJhbnNmb3JtZXJzLkdlbmVyYXRpb25Db25maWcoKiooY2ZnIG9yIHt9KSkKICAgICAgICBmb3IgY2ZnIGluIF90b19ncm91cF9saXN0KAogICAgICAgICAgICBhcmd1bWVudF92YWx1ZT1nZW5lcmF0aW9uX2NvbmZpZywKICAgICAgICAgICAgYXJndW1lbnRfbmFtZT0iZ2VuZXJhdGlvbl9jb25maWciLAogICAgICAgICAgICBsZW5ndGg9bnVtYmVyX29mX3F1ZXN0aW9uX2dyb3VwcywKICAgICAgICApCiAgICBdCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkdlbmVyYXRpb24gY29uZmlndXJhdGlvbiBsb2FkZWQ6IHtnZW5lcmF0aW9uX2NvbmZpZ30iKQoKICAgICMgTG9hZCB0aGUgbW9kZWwgYW5kIHRva2VuaXplciBpbnRvIGEgcGlwZWxpbmUgb2JqZWN0OgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJMb2FkaW5nIG1vZGVsICd7bW9kZWxfbmFtZX0nLiIpCiAgICBnZW5lcmF0aW9uX3BpcGVsaW5lID0gX2dldF9nZW5lcmF0aW9uX3BpcGVsaW5lKAogICAgICAgIG1vZGVsX25hbWU9bW9kZWxfbmFtZSwKICAgICAgICBkZXZpY2VfbWFwPWRldmljZV9tYXAsCiAgICAgICAgdG9rZW5pemVyX25hbWU9dG9rZW5pemVyX25hbWUgb3IgbW9kZWxfbmFtZSwKICAgICAgICBtb2RlbF9rd2FyZ3M9bW9kZWxfa3dhcmdzIG9yIHt9LAogICAgICAgIHRva2VuaXplcl9rd2FyZ3M9dG9rZW5pemVyX2t3YXJncyBvciB7fSwKICAgICAgICBhdXRvX2dwdHFfZXhsbGFtYV9tYXhfaW5wdXRfbGVuZ3RoPWF1dG9fZ3B0cV9leGxsYW1hX21heF9pbnB1dF9sZW5ndGgsCiAgICAgICAgYmF0Y2hfc2l6ZT1iYXRjaF9zaXplLAogICAgKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIk1vZGVsIGxvYWRlZC4iKQoKICAgICMgUHJlcGFyZSB0aGUgc3VjY2Vzc2VzIGRhdGFmcmFtZSBhbmQgZXJyb3JzIGRpY3Rpb25hcnkgdG8gYmUgcmV0dXJuZWQ6CiAgICBzdWNjZXNzZXMgPSBbXQogICAgZXJyb3JzID0ge30KCiAgICAjIFNwbGl0IHRoZSBmaWxlcyBpbnRvIGJhdGNoZXM6CiAgICBmaWxlX2JhdGNoZXMgPSBbCiAgICAgICAgdGV4dF9maWxlc1tpIDogaSArIGJhdGNoX3NpemVdCiAgICAgICAgaWYgaSArIGJhdGNoX3NpemUgPCBsZW4odGV4dF9maWxlcykKICAgICAgICBlbHNlIHRleHRfZmlsZXNbaTpdCiAgICAgICAgZm9yIGkgaW4gcmFuZ2UoMCwgbGVuKHRleHRfZmlsZXMpLCBiYXRjaF9zaXplKQogICAgXQogICAgcXVlc3Rpb25zX2NvbmZpZyA9IF90b19ncm91cF9saXN0KAogICAgICAgIGFyZ3VtZW50X3ZhbHVlPXF1ZXN0aW9uc19jb25maWcsCiAgICAgICAgYXJndW1lbnRfbmFtZT0icXVlc3Rpb25zX2NvbmZpZyIsCiAgICAgICAgbGVuZ3RoPW51bWJlcl9vZl9xdWVzdGlvbl9ncm91cHMsCiAgICApCgogICAgIyBDcmVhdGUgYSBsaXN0IG9mIHF1ZXN0aW9uIGhhbmRsZXJzIGFjY29yZGluZyB0byBnaXZlbiBjb25maWdzCiAgICBoYW5kbGVycyA9IFtdCiAgICBmb3IgY2ZnIGluIHF1ZXN0aW9uc19jb25maWc6CiAgICAgICAgcXVlc3Rpb25fdHlwZSA9IGNmZy5wb3AoInR5cGUiLCAiZGVmYXVsdCIpCiAgICAgICAgaGFuZGxlcnMuYXBwZW5kKFFVRVNUSU9OX01BUFBJTkcuZ2V0KHF1ZXN0aW9uX3R5cGUpKCoqY2ZnKSkKCiAgICAjIEdvIG92ZXIgdGhlIGJhdGNoZXMgb2YgdGV4dCBmaWxlcyBhbmQgcXVlc3Rpb24gdGhlbToKICAgIGZvciBmaWxlX2JhdGNoIGluIHRxZG0oCiAgICAgICAgZmlsZV9iYXRjaGVzLAogICAgICAgIGRlc2M9IkdlbmVyYXRpbmcgYW5zd2VycyIsCiAgICAgICAgdW5pdD1mImZpbGUgKGJhdGNoIG9mIHtiYXRjaF9zaXplfSkiLAogICAgICAgIGRpc2FibGU9bm90IHZlcmJvc2UsCiAgICApOgogICAgICAgIHRyeToKICAgICAgICAgICAgdG90YWxfYW5zd2VycyA9IFtbXSBmb3IgXyBpbiByYW5nZShiYXRjaF9zaXplKV0KCiAgICAgICAgICAgICMgR28gb3ZlciBhbGwgcXVlc3Rpb24gZ3JvdXAgcGVyIGJhdGNoIG9mIGRvY3VtZW50cwogICAgICAgICAgICBmb3IgcXVlc3Rpb25fZ3JvdXAgaW4gcmFuZ2UobnVtYmVyX29mX3F1ZXN0aW9uX2dyb3Vwcyk6CiAgICAgICAgICAgICAgICBjdXJyZW50X3F1ZXN0aW9uc19hbW91bnQgPSBsZW4ocXVlc3Rpb25zW3F1ZXN0aW9uX2dyb3VwXSkKCiAgICAgICAgICAgICAgICAjIFJlYWQgYmF0Y2ggKHJlYWQgdGhlIHRleHQgZnJvbSB0aGUgdGV4dCBmaWxlcyk6CiAgICAgICAgICAgICAgICBiYXRjaGVkX2lucHV0ID0gX3JlYWRfZmlsZV9iYXRjaCgKICAgICAgICAgICAgICAgICAgICBmaWxlX2JhdGNoPWZpbGVfYmF0Y2gsCiAgICAgICAgICAgICAgICAgICAgcHJvbXB0X3RlbXBsYXRlPXByb21wdF90ZW1wbGF0ZVtxdWVzdGlvbl9ncm91cF0sCiAgICAgICAgICAgICAgICApCgogICAgICAgICAgICAgICAgIyBBbnN3ZXIgdGhlIHF1ZXN0aW9ucyB3aXRoIGVhY2ggcXVlc3Rpb24gaGFuZGxlcjoKICAgICAgICAgICAgICAgIGJhdGNoZWRfYW5zd2VycyA9IGhhbmRsZXJzW3F1ZXN0aW9uX2dyb3VwXS5hbnN3ZXIoCiAgICAgICAgICAgICAgICAgICAgcXVlc3Rpb25zX2Ftb3VudD1jdXJyZW50X3F1ZXN0aW9uc19hbW91bnQsCiAgICAgICAgICAgICAgICAgICAgYmF0Y2hlZF9pbnB1dD1iYXRjaGVkX2lucHV0LAogICAgICAgICAgICAgICAgICAgIGdlbmVyYXRpb25fcGlwZWxpbmU9Z2VuZXJhdGlvbl9waXBlbGluZSwKICAgICAgICAgICAgICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZz1nZW5lcmF0aW9uX2NvbmZpZ1txdWVzdGlvbl9ncm91cF0sCiAgICAgICAgICAgICAgICApCgogICAgICAgICAgICAgICAgIyBQdXQgdGhlIGFuc3dlcnMgaW4gdGhlIGNvcnJlY3QgcGxhY2UgaW4gdGhlIHRvdGFsIGFuc3dlcnMgbGlzdCBhY2NvcmRpbmcgdG8gdGhlIHBsYWNlIGluIHRoZSBiYXRjaDoKICAgICAgICAgICAgICAgIGZvciBpIGluIHJhbmdlKGJhdGNoX3NpemUpOgogICAgICAgICAgICAgICAgICAgIHRvdGFsX2Fuc3dlcnNbaV0uZXh0ZW5kKGJhdGNoZWRfYW5zd2Vyc1tpXSkKCiAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgYW5zd2VycyBhbmQgYXR0YWNoIHRoZSBmaWxlIG5hbWU6CiAgICAgICAgICAgIHN1Y2Nlc3Nlcy5leHRlbmQoCiAgICAgICAgICAgICAgICBbCiAgICAgICAgICAgICAgICAgICAgW2ZpbGUubmFtZSwgKmFuc3dlcnNdCiAgICAgICAgICAgICAgICAgICAgZm9yIGZpbGUsIGFuc3dlcnMgaW4gemlwKGZpbGVfYmF0Y2gsIHRvdGFsX2Fuc3dlcnMpCiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgICkKICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGV4Y2VwdGlvbjoKICAgICAgICAgICAgIyBOb3RlIHRoZSBleGNlcHRpb24gYXMgZXJyb3IgaW4gdGhlIGRpY3Rpb25hcnk6CiAgICAgICAgICAgIGJhdGNoX2ZpbGVfbmFtZXMgPSAiLCAiLmpvaW4oW2ZpbGUubmFtZSBmb3IgZmlsZSBpbiBmaWxlX2JhdGNoXSkKICAgICAgICAgICAgaWYgdmVyYm9zZToKICAgICAgICAgICAgICAgIF9MT0dHRVIud2FybmluZygKICAgICAgICAgICAgICAgICAgICBmIkVycm9yIGluIGJhdGNoICd7YmF0Y2hfZmlsZV9uYW1lc30nOiB7c3RyKGV4Y2VwdGlvbil9IgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICBlcnJvcnNbYmF0Y2hfZmlsZV9uYW1lc10gPSBzdHIoZXhjZXB0aW9uKQogICAgICAgICAgICBjb250aW51ZQoKICAgICMgQ29uc3RydWN0IHRoZSBhbnN3ZXJzIGRhdGFmcmFtZToKICAgIGNvbHVtbnMgPSBbCiAgICAgICAgInRleHRfZmlsZSIsCiAgICAgICAgKnF1ZXN0aW9uc19jb2x1bW5zLAogICAgXQoKICAgICMgQ3JlYXRlIGEgZGF0YSBmcmFtZSBvZiBhbnN3ZXJzIGJ5IGZpbGVzCiAgICBzdWNjZXNzZXMgPSBwZC5EYXRhRnJhbWUoCiAgICAgICAgc3VjY2Vzc2VzLAogICAgICAgIGNvbHVtbnM9Y29sdW1ucywKICAgICkKCiAgICAjIFByaW50IHRoZSBoZWFkIG9mIHRoZSBwcm9kdWNlZCBkYXRhZnJhbWUgYW5kIHJldHVybjoKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKAogICAgICAgICAgICBmIkRvbmUgKHtzdWNjZXNzZXMuc2hhcGVbMF19L3tsZW4odGV4dF9maWxlcyl9KVxuIgogICAgICAgICAgICBmIkFuc3dlcnMgc3VtbWFyeTpcbiIKICAgICAgICAgICAgZiJ7c3VjY2Vzc2VzLmhlYWQoKX0iCiAgICAgICAgKQogICAgcmV0dXJuIHN1Y2Nlc3NlcywgZXJyb3JzCgoKZGVmIF9nZXRfdGV4dF9maWxlcygKICAgIGRhdGFfcGF0aDogcGF0aGxpYi5QYXRoLAopIC0+IGxpc3RbcGF0aGxpYi5QYXRoXToKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICB0ZXh0X2ZpbGVzID0gbGlzdChkYXRhX3BhdGguZ2xvYigiKi4qIikpCiAgICBlbGlmIGRhdGFfcGF0aC5pc19maWxlKCk6CiAgICAgICAgdGV4dF9maWxlcyA9IFtkYXRhX3BhdGhdCiAgICBlbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgIGYiVW5yZWNvZ25pemVkIGRhdGEgcGF0aC4gVGhlIHBhcmFtZXRlciBgZGF0YV9wYXRoYCBtdXN0IGJlIGVpdGhlciBhIGRpcmVjdG9yeSBwYXRoIG9yIGEgZmlsZSBwYXRoLiAiCiAgICAgICAgICAgIGYiR2l2ZW46IHtzdHIoZGF0YV9wYXRoKX0gIgogICAgICAgICkKCiAgICByZXR1cm4gdGV4dF9maWxlcwoKCmRlZiBfZ2V0X3Byb21wdF90ZW1wbGF0ZSgKICAgIHRleHRfd3JhcHBlcjogc3RyLAogICAgcXVlc3Rpb25zX3dyYXBwZXI6IHN0ciwKICAgIHF1ZXN0aW9uczogbGlzdFtzdHJdLAopIC0+IHN0cjoKICAgICMgVmFsaWRhdGUgYW5kIGJ1aWxkIHRoZSB0ZXh0IHdyYXBwZXI6CiAgICB0ZXh0X3dyYXBwZXIgPSB0ZXh0X3dyYXBwZXIgb3IgKCJHaXZlbiB0aGUgZm9sbG93aW5nIHRleHQ6XG4tLS0tLVxue31cbi0tLS0tIikKICAgIGlmIHRleHRfd3JhcHBlci5jb3VudCgie30iKSAhPSAxOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICJUaGUgYHRleHRfd3JhcHBlcmAgbXVzdCBpbmNsdWRlIG9uZSBwbGFjZWhvbGRlciAne30nIGZvciB0aGUgdGV4dCBvZiB0aGUgZmlsZSB0byBiZSBhc2tlZCBhYm91dC4iCiAgICAgICAgKQoKICAgICMgVmFsaWRhdGUgYW5kIGJ1aWxkIHRoZSBxdWVzdGlvbiB3cmFwcGVyOgogICAgcXVlc3Rpb25zX3dyYXBwZXIgPSBxdWVzdGlvbnNfd3JhcHBlciBvciAiQW5zd2VyIHRoZSBxdWVzdGlvbnM6XG57fSIKICAgIGlmIHF1ZXN0aW9uc193cmFwcGVyLmNvdW50KCJ7fSIpICE9IDE6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgIlRoZSBgcXVlc3Rpb25zX3dyYXBwZXJgIG11c3QgaW5jbHVkZSBvbmUgcGxhY2Vob2xkZXIgJ3t9JyBmb3IgdGhlIGxpc3Qgb2YgcXVlc3Rpb25zLiIKICAgICAgICApCgogICAgIyBWYWxpZGF0ZSBhbmQgcGFyc2UgdGhlIHF1ZXN0aW9uczoKICAgIGlmIGxlbihxdWVzdGlvbnMpID09IDA6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigiUGxlYXNlIGluY2x1ZGUgYXQgbGVhc3Qgb25lIHF1ZXN0aW9uLiIpCiAgICBxdWVzdGlvbnMgPSAiXG4iLmpvaW4oCiAgICAgICAgW2Yie2l9LiB7cXVlc3Rpb259IiBmb3IgaSwgcXVlc3Rpb24gaW4gZW51bWVyYXRlKHF1ZXN0aW9ucywgMSldCiAgICApCgogICAgIyBDb25zdHJ1Y3QgdGhlIHRlbXBsYXRlOgogICAgcmV0dXJuIGYie3RleHRfd3JhcHBlcn1cbntxdWVzdGlvbnNfd3JhcHBlci5mb3JtYXQocXVlc3Rpb25zKX1cbiIKCgpkZWYgX2dldF9nZW5lcmF0aW9uX3BpcGVsaW5lKAogICAgbW9kZWxfbmFtZTogc3RyLAogICAgZGV2aWNlX21hcDogc3RyIHwgZGljdCwKICAgIHRva2VuaXplcl9uYW1lOiBzdHIsCiAgICBtb2RlbF9rd2FyZ3M6IGRpY3QsCiAgICB0b2tlbml6ZXJfa3dhcmdzOiBkaWN0LAogICAgYXV0b19ncHRxX2V4bGxhbWFfbWF4X2lucHV0X2xlbmd0aDogaW50ID0gTm9uZSwKICAgIGJhdGNoX3NpemU6IGludCA9IDEsCik6CiAgICAjIExvYWQgdGhlIG1vZGVsOgogICAgbW9kZWwgPSB0cmFuc2Zvcm1lcnMuQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKAogICAgICAgIG1vZGVsX25hbWUsIGRldmljZV9tYXA9ZGV2aWNlX21hcCwgKiptb2RlbF9rd2FyZ3MKICAgICkKCiAgICAjIFNldCBleGxsYW1hIG1heCBpbnB1dCBsZW5ndGggaWYgcHJvdmlkZWQ6CiAgICAjIFRoaXMgY2hhbmdlcyB0aGUgbW9kZWwncyBjb250ZXh0IHNpemUuCiAgICBpZiBhdXRvX2dwdHFfZXhsbGFtYV9tYXhfaW5wdXRfbGVuZ3RoOgogICAgICAgIGZyb20gYXV0b19ncHRxIGltcG9ydCBleGxsYW1hX3NldF9tYXhfaW5wdXRfbGVuZ3RoCgogICAgICAgIG1vZGVsID0gZXhsbGFtYV9zZXRfbWF4X2lucHV0X2xlbmd0aCgKICAgICAgICAgICAgbW9kZWw9bW9kZWwsIG1heF9pbnB1dF9sZW5ndGg9YXV0b19ncHRxX2V4bGxhbWFfbWF4X2lucHV0X2xlbmd0aAogICAgICAgICkKCiAgICAjIExvYWQgdGhlIHRva2VuaXplcjoKICAgIHRva2VuaXplciA9IHRyYW5zZm9ybWVycy5BdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCgKICAgICAgICB0b2tlbml6ZXJfbmFtZSwgKip0b2tlbml6ZXJfa3dhcmdzCiAgICApCgogICAgIyBJbml0aWFsaXplIGEgZ2VuZXJhdGlvbiBwaXBsaW5lIGFuZCByZXR1cm46CiAgICBwaXBlID0gdHJhbnNmb3JtZXJzLnBpcGVsaW5lKAogICAgICAgIHRhc2s9InRleHQtZ2VuZXJhdGlvbiIsCiAgICAgICAgbW9kZWw9bW9kZWwsCiAgICAgICAgdG9rZW5pemVyPXRva2VuaXplciwKICAgICAgICBiYXRjaF9zaXplPWJhdGNoX3NpemUsCiAgICApCiAgICBwaXBlLnRva2VuaXplci5wYWRfdG9rZW5faWQgPSBtb2RlbC5jb25maWcuZW9zX3Rva2VuX2lkCiAgICByZXR1cm4gcGlwZQoKCmRlZiBfcmVhZF9maWxlX2JhdGNoKAogICAgZmlsZV9iYXRjaDogbGlzdFtwYXRobGliLlBhdGhdLAogICAgcHJvbXB0X3RlbXBsYXRlOiBzdHIsCikgLT4gbGlzdFtzdHJdOgogICAgYmF0Y2ggPSBbXQoKICAgICMgR28gb3ZlciBhbGwgZmlsZXMgYW5kIHJlYWQgaW4gdXNhYmxlIGZvcm1hdAogICAgZm9yIGZpbGUgaW4gZmlsZV9iYXRjaDoKICAgICAgICB3aXRoIG9wZW4oZmlsZSwgZW5jb2Rpbmc9InV0Zi04IikgYXMgZnA6CiAgICAgICAgICAgIGJhdGNoLmFwcGVuZChwcm9tcHRfdGVtcGxhdGUuZm9ybWF0KGZwLnJlYWQoKSkpCiAgICByZXR1cm4gYmF0Y2gKCgpkZWYgX3RvX2dyb3VwX2xpc3QoYXJndW1lbnRfdmFsdWU6IGxpc3QsIGFyZ3VtZW50X25hbWU6IHN0ciwgbGVuZ3RoOiBpbnQpOgogICAgIyBDaGVjayBpZiBpcyBsaXN0LCB0dXJuIHRvIGxpc3QgaWYgbm90CiAgICBhcmd1bWVudF92YWx1ZSA9ICgKICAgICAgICBhcmd1bWVudF92YWx1ZSBpZiBpc2luc3RhbmNlKGFyZ3VtZW50X3ZhbHVlLCBsaXN0KSBlbHNlIFthcmd1bWVudF92YWx1ZV0KICAgICkKICAgIGxpc3RfbGVuID0gbGVuKGFyZ3VtZW50X3ZhbHVlKQoKICAgICMgSWYgbm90IGEgbGlzdCwgb3IgaXMgYSBsaXN0IG9mIGxlbiAxIHdlIGR1cGxpY2F0ZSBmb3IgY29ycmVjdCBsZW5ndGgKICAgICMgSWYgbGlzdCBpbiB3cm9uZyBsZW5ndGggdGhyb3cgYW4gZXJyb3IKICAgIGlmIGxpc3RfbGVuICE9IGxlbmd0aDoKICAgICAgICBpZiBsaXN0X2xlbiA9PSAxOgogICAgICAgICAgICByZXR1cm4gYXJndW1lbnRfdmFsdWUgKiBsZW5ndGgKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlRoZSBhcmd1bWVudCB2YWx1ZSBvZiAne2FyZ3VtZW50X25hbWV9JyBpcyBub3QgZXF1YWwgdG8gdGhlIGxlbmd0aCBvZiB0aGUgZ2l2ZW4gcXVlc3Rpb25zIC0ge2xlbmd0aH0iCiAgICAgICAgKQogICAgcmV0dXJuIGFyZ3VtZW50X3ZhbHVlCgoKY2xhc3MgUXVlc3Rpb25IYW5kbGVyOgogICAgIiIiCiAgICBBIGNsYXNzIGZvciBoYW5kbGluZyBxdWVzdGlvbnMgYW5zd2VyaW5nIGZvciBhIGdpdmVuIHF1ZXN0aW9uIHR5cGUuCiAgICBUaGlzIGNsYXNzIGlzIHVzZWQgYXMgYSBiYXNlIGNsYXNzIGZvciBhbGwgcXVlc3Rpb24gdHlwZXMsIGFuZCBmb3IgZGVmYXVsdCBxdWVzdGlvbiB0eXBlIChyZWd1bGFyIHF1ZXN0aW9uCiAgICBhbnN3ZXJpbmcgd2l0aG91dCBhbnkgc3BlY2lhbCBoYW5kbGluZykuCiAgICAiIiIKCiAgICBjbGFzcyBDb25maWdLZXlzOgogICAgICAgIHBhc3MKCiAgICBkZWYgX19pbml0X18oc2VsZik6CiAgICAgICAgcGFzcwoKICAgIEBzdGF0aWNtZXRob2QKICAgIGRlZiBfZ2V0X2Fuc3dlcnMoZ2VuZXJhdGVkX3RleHQ6IHN0ciwgcXVlc3Rpb25zX2Ftb3VudDogaW50KSAtPiBsaXN0W3N0cl06CiAgICAgICAgIyBDbGVhciBhbnN3ZXIgc3RhcnQgKHBhcnQgYmVmb3JlIG51bWJlcnMpOgogICAgICAgICMgVE9ETyBmaW5kIGJldHRlciB3YXkgdG8gdmVyaWZ5LCBmb3IgbGlzdCBvZiBxdWVzdGlvbnMgdGhpcyBpcyByZWR1bmRhbnQgZm9yIGV4YW1wbGUKICAgICAgICBpZiAiMS4iIG5vdCBpbiBnZW5lcmF0ZWRfdGV4dDoKICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgIGYiQW5zd2VyIDEuIGlzIG1pc3NpbmcgZnJvbSB0aGUgZ2VuZXJhdGVkIHRleHQ6ICd7Z2VuZXJhdGVkX3RleHR9JyIKICAgICAgICAgICAgKQogICAgICAgIHRleHQgPSBnZW5lcmF0ZWRfdGV4dC5zcGxpdCgiMS4iLCAxKVsxXQoKICAgICAgICAjIFN0YXJ0IGV4dHJhY3RpbmcgdGhlIGFuc3dlcnM6CiAgICAgICAgYW5zd2VycyA9IFtdCiAgICAgICAgZm9yIGkgaW4gcmFuZ2UoMSwgcXVlc3Rpb25zX2Ftb3VudCArIDEpOgogICAgICAgICAgICAjIElmIGl0J3MgdGhlIGxhc3QgYW5zd2VyIHRvIGxvb2sgZm9yLCB0YWtlIHRoZSByZXN0IG9mIHRoZSB0ZXh0OgogICAgICAgICAgICBpZiBpID09IHF1ZXN0aW9uc19hbW91bnQ6CiAgICAgICAgICAgICAgICBhbnN3ZXJfaSA9IHRleHQKICAgICAgICAgICAgIyBWZXJpZnkgdGhlcmUgaXMgYSBxdWVzdGlvbiBudW1iZXIgaW4gdGhlIHRleHQ6CiAgICAgICAgICAgIGVsaWYgZiJ7aSArIDF9LiIgbm90IGluIHRleHQ6CiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgICAgIGYiQW5zd2VyIHtpICsgMX0uIGlzIG1pc3NpbmcgZnJvbSB0aGUgZ2VuZXJhdGVkIHRleHQ6ICd7Z2VuZXJhdGVkX3RleHR9JyIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgIyBUYWtlIGkncyBhbnN3ZXI6CiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICBhbnN3ZXJfaSwgdGV4dCA9IHRleHQuc3BsaXQoZiJ7aSArIDF9LiIsIDEpCiAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgYW5zd2VyIHJlbW92aW5nIHJlZHVuZGFudCBzcGFjZXM6CiAgICAgICAgICAgIGFuc3dlcnMuYXBwZW5kKGFuc3dlcl9pLnN0cmlwKCkpCgogICAgICAgIHJldHVybiBhbnN3ZXJzCgogICAgZGVmIF9pbmZlcl9xdWVzdGlvbnMoCiAgICAgICAgc2VsZiwKICAgICAgICBxdWVzdGlvbnNfYW1vdW50OiBpbnQsCiAgICAgICAgYmF0Y2hlZF9pbnB1dDogbGlzdFtzdHJdLAogICAgICAgIGdlbmVyYXRpb25fcGlwZWxpbmU6IHRyYW5zZm9ybWVycy5QaXBlbGluZSwKICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZzogdHJhbnNmb3JtZXJzLkdlbmVyYXRpb25Db25maWcsCiAgICApIC0+IGxpc3RbbGlzdFtzdHJdXToKICAgICAgICAjIEluZmVyIHRocm91Z2ggdGhlIGxsbToKICAgICAgICBiYXRjaGVkX291dHB1dCA9IGdlbmVyYXRpb25fcGlwZWxpbmUoCiAgICAgICAgICAgIGJhdGNoZWRfaW5wdXQsCiAgICAgICAgICAgIGdlbmVyYXRpb25fY29uZmlnPWdlbmVyYXRpb25fY29uZmlnLAogICAgICAgICAgICBlb3NfdG9rZW5faWQ9Z2VuZXJhdGlvbl9waXBlbGluZS50b2tlbml6ZXIuZW9zX3Rva2VuX2lkLAogICAgICAgICAgICByZXR1cm5fZnVsbF90ZXh0PUZhbHNlLAogICAgICAgICAgICBudW1fcmV0dXJuX3NlcXVlbmNlcz0xLAogICAgICAgICkKCiAgICAgICAgIyBQcm9jZXNzIHRoZSBvdXRwdXRzIHRvIGdldCB0aGUgYW5zd2VyczoKICAgICAgICBiYXRjaGVkX2Fuc3dlcnMgPSBbXQogICAgICAgIGZvciBvdXRwdXQgaW4gYmF0Y2hlZF9vdXRwdXQ6CiAgICAgICAgICAgICMgR2V0IHRoZSBnZW5lcmF0ZWQgYW5zd2VyczoKICAgICAgICAgICAgYW5zd2VycyA9IHNlbGYuX2dldF9hbnN3ZXJzKAogICAgICAgICAgICAgICAgZ2VuZXJhdGVkX3RleHQ9b3V0cHV0WzBdWyJnZW5lcmF0ZWRfdGV4dCJdLAogICAgICAgICAgICAgICAgcXVlc3Rpb25zX2Ftb3VudD1xdWVzdGlvbnNfYW1vdW50LAogICAgICAgICAgICApCiAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgcHJvY2Vzc2VkIGFuc3dlcnM6CiAgICAgICAgICAgIGJhdGNoZWRfYW5zd2Vycy5hcHBlbmQoYW5zd2VycykKICAgICAgICByZXR1cm4gYmF0Y2hlZF9hbnN3ZXJzCgogICAgZGVmIGFuc3dlcigKICAgICAgICBzZWxmLAogICAgICAgIHF1ZXN0aW9uc19hbW91bnQ6IGludCwKICAgICAgICBiYXRjaGVkX2lucHV0OiBsaXN0W3N0cl0sCiAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZTogdHJhbnNmb3JtZXJzLlBpcGVsaW5lLAogICAgICAgIGdlbmVyYXRpb25fY29uZmlnOiB0cmFuc2Zvcm1lcnMuR2VuZXJhdGlvbkNvbmZpZywKICAgICkgLT4gbGlzdFtsaXN0W3N0cl1dOgogICAgICAgICIiIgogICAgICAgIEFuc3dlciBxdWVzdGlvbnMgd2l0aCBhIGNvbnRleHQgdG8gdGhlIGdpdmVuIHRleHQgZmlsZXMgY29udGVudHMgYnkgYSBwcmV0cmFpbmVkIExMTSBtb2RlbCBpbiBnaXZlbiBwaXBlbGluZS4KICAgICAgICAiIiIKICAgICAgICByZXR1cm4gc2VsZi5faW5mZXJfcXVlc3Rpb25zKAogICAgICAgICAgICBxdWVzdGlvbnNfYW1vdW50PXF1ZXN0aW9uc19hbW91bnQsCiAgICAgICAgICAgIGJhdGNoZWRfaW5wdXQ9YmF0Y2hlZF9pbnB1dCwKICAgICAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZT1nZW5lcmF0aW9uX3BpcGVsaW5lLAogICAgICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZz1nZW5lcmF0aW9uX2NvbmZpZywKICAgICAgICApCgoKY2xhc3MgUG9sbFF1ZXN0aW9uSGFuZGxlcihRdWVzdGlvbkhhbmRsZXIpOgogICAgIiIiCiAgICBTdGF0aWMgY2xhc3MgdG8gaG9sZCBhbGwgdGhlIHBvc3NpYmxlIHBvbGwgcXVlc3Rpb24gY29uZmlndXJhdGlvbnMgb3B0aW9ucyBrZXlzCiAgICAiIiIKCiAgICBjbGFzcyBDb25maWdLZXlzOgogICAgICAgICIiIgogICAgICAgIEEgY2xhc3MgZm9yIGhhbmRsaW5nIHF1ZXN0aW9ucyBhbnN3ZXJpbmcgZm9yIHBvbGwgdHlwZSBxdWVzdGlvbnMuCiAgICAgICAgVGhlc2UgdHlwZSBvZiBxdWVzdGlvbiBhcmUgYW5zd2VyZWQgYnkgYXNraW5nIHRoZSBzYW1lIHF1ZXN0aW9uIG11bHRpcGxlIHRpbWVzCiAgICAgICAgYW5kIGNob29zaW5nIHRoZSBtb3N0IGNvbW1vbiBhbnN3ZXIgb3IgdGhlIGF2ZXJhZ2UgYW5zd2VyLgogICAgICAgICIiIgoKICAgICAgICAjOiBUaGUgbnVtYmVyIG9mIHRpbWVzIHRvIGFzayB0aGUgc2FtZSBxdWVzdGlvbi4KICAgICAgICBQT0xMX0NPVU5UID0gInBvbGxfY291bnQiCgogICAgICAgICM6IFRoZSBzdHJhdGVneSB0byB1c2UgZm9yIGNob29zaW5nIHRoZSBhbnN3ZXIgZnJvbSB0aGUgcG9sbC4KICAgICAgICBQT0xMX1NUUkFURUdZID0gInBvbGxfc3RyYXRlZ3kiCgogICAgY2xhc3MgU3RyYXRlZ3koZW51bS5FbnVtKToKICAgICAgICAjOiBUaGUgbW9zdCBjb21tb24gYW5zd2VyIHN0cmF0ZWd5LgogICAgICAgIE1PU1RfQ09NTU9OID0gIm1vc3RfY29tbW9uIgoKICAgICAgICAjOiBUaGUgYXZlcmFnZSBhbnN3ZXIgc3RyYXRlZ3kuCiAgICAgICAgQVZFUkFHRSA9ICJhdmVyYWdlIgoKICAgICAgICBAc3RhdGljbWV0aG9kCiAgICAgICAgZGVmIG1vc3RfY29tbW9uKGFuc3dlcnMpOgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgQ2FsY3VsYXRlIHRoZSBtb3N0IGNvbW1vbiBhbnN3ZXIgZm9yIGEgZ2l2ZW4gbGlzdCBvZiBhbnN3ZXJzLgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgY291bnQgPSBDb3VudGVyKGFuc3dlcnMpCiAgICAgICAgICAgIG1vc3RfY29tbW9uID0gY291bnQubW9zdF9jb21tb24oMSkKICAgICAgICAgICAgcmV0dXJuIG1vc3RfY29tbW9uWzBdWzBdCgogICAgICAgIEBzdGF0aWNtZXRob2QKICAgICAgICBkZWYgYXZlcmFnZShhbnN3ZXJzKToKICAgICAgICAgICAgIiIiCiAgICAgICAgICAgIENhbGN1bGF0ZSB0aGUgYXZlcmFnZSBhbnN3ZXIgZm9yIGEgZ2l2ZW4gbGlzdCBvZiBhbnN3ZXJzLgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShhbnN3ZXJzWzBdLCBzdHIpOgogICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAiQ2Fubm90IHBlcmZvcm0gcG9sbCB3aXRoIGF2ZXJhZ2UgYW5zd2VyIHN0cmF0ZWd5IG9mIG5vbiBudW1lcmljIHZhbHVlcywiCiAgICAgICAgICAgICAgICAgICAgIiBwbGVhc2UgY2hhbmdlIHRoZSBxdWVzdGlvbiB0byBnaXZlIG51bWVyaWMgZGF0YSwgb3IgY2hvb3NlICdtb3N0X2NvbW1vbicgYXMgc3RyYXRlZ3kuIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgbnVtZXJpY192YWx1ZXMgPSBhbnN3ZXJzCiAgICAgICAgICAgIGF2ZyA9IHN1bShudW1lcmljX3ZhbHVlcykgLyBsZW4obnVtZXJpY192YWx1ZXMpCgogICAgICAgICAgICAjIFJvdW5kIHRvIHRoZSBjbG9zZXN0IGludGVnZXIgYW5kIHJldHVybiBjb3JyZXNwb25kaW5nIHZhbHVlCiAgICAgICAgICAgIHJldHVybiByb3VuZChhdmcpCgogICAgICAgIGRlZiBkbyhzZWxmLCBhbnN3ZXJzKToKICAgICAgICAgICAgIiIiCiAgICAgICAgICAgIFBlcmZvcm0gdGhlIHN0cmF0ZWd5LgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgcmV0dXJuIGdldGF0dHIoc2VsZiwgc2VsZi52YWx1ZSkoYW5zd2VycykKCiAgICBkZWYgX19pbml0X18oc2VsZiwgcG9sbF9jb3VudDogaW50ID0gNSwgcG9sbF9zdHJhdGVneTogc3RyID0gIm1vc3RfY29tbW9uIik6CiAgICAgICAgc3VwZXIoKS5fX2luaXRfXygpCiAgICAgICAgc2VsZi5wb2xsX2NvdW50ID0gcG9sbF9jb3VudAogICAgICAgIHNlbGYucG9sbF9zdHJhdGVneSA9IHNlbGYuU3RyYXRlZ3kocG9sbF9zdHJhdGVneSkKCiAgICBkZWYgYW5zd2VyKAogICAgICAgIHNlbGYsCiAgICAgICAgcXVlc3Rpb25zX2Ftb3VudDogaW50LAogICAgICAgIGJhdGNoZWRfaW5wdXQ6IGxpc3Rbc3RyXSwKICAgICAgICBnZW5lcmF0aW9uX3BpcGVsaW5lOiB0cmFuc2Zvcm1lcnMuUGlwZWxpbmUsCiAgICAgICAgZ2VuZXJhdGlvbl9jb25maWc6IHRyYW5zZm9ybWVycy5HZW5lcmF0aW9uQ29uZmlnLAogICAgKSAtPiBsaXN0W2xpc3Rbc3RyXV06CiAgICAgICAgIiIiCiAgICAgICAgQW5zd2VyIHF1ZXN0aW9ucyB3aXRoIGEgY29udGV4dCB0byB0aGUgZ2l2ZW4gdGV4dCBmaWxlcyBjb250ZW50cyBieSBhIHByZXRyYWluZWQgTExNIG1vZGVsIGluIGdpdmVuIHBpcGVsaW5lLgogICAgICAgICIiIgogICAgICAgIHJldHVybiBzZWxmLl9hbnN3ZXJfcG9sbF9xdWVzdGlvbnMoCiAgICAgICAgICAgIHF1ZXN0aW9uc19hbW91bnQ9cXVlc3Rpb25zX2Ftb3VudCwKICAgICAgICAgICAgYmF0Y2hlZF9pbnB1dD1iYXRjaGVkX2lucHV0LAogICAgICAgICAgICBnZW5lcmF0aW9uX3BpcGVsaW5lPWdlbmVyYXRpb25fcGlwZWxpbmUsCiAgICAgICAgICAgIGdlbmVyYXRpb25fY29uZmlnPWdlbmVyYXRpb25fY29uZmlnLAogICAgICAgICkKCiAgICBkZWYgX2Fuc3dlcl9wb2xsX3F1ZXN0aW9ucygKICAgICAgICBzZWxmLAogICAgICAgIHF1ZXN0aW9uc19hbW91bnQ6IGludCwKICAgICAgICBiYXRjaGVkX2lucHV0OiBsaXN0W3N0cl0sCiAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZTogdHJhbnNmb3JtZXJzLlBpcGVsaW5lLAogICAgICAgIGdlbmVyYXRpb25fY29uZmlnOiB0cmFuc2Zvcm1lcnMuR2VuZXJhdGlvbkNvbmZpZywKICAgICkgLT4gbGlzdFtsaXN0W3N0cl1dOgogICAgICAgIHZvdGVzID0gW10KCiAgICAgICAgIyBSdW4gdGhlIHBvbGwgZm9yIGVhY2ggcXVlc3Rpb24KICAgICAgICBmb3IgXyBpbiByYW5nZShzZWxmLnBvbGxfY291bnQpOgogICAgICAgICAgICBiYXRjaGVkX2Fuc3dlcnMgPSBzZWxmLl9pbmZlcl9xdWVzdGlvbnMoCiAgICAgICAgICAgICAgICBxdWVzdGlvbnNfYW1vdW50PXF1ZXN0aW9uc19hbW91bnQsCiAgICAgICAgICAgICAgICBiYXRjaGVkX2lucHV0PWJhdGNoZWRfaW5wdXQsCiAgICAgICAgICAgICAgICBnZW5lcmF0aW9uX3BpcGVsaW5lPWdlbmVyYXRpb25fcGlwZWxpbmUsCiAgICAgICAgICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZz1nZW5lcmF0aW9uX2NvbmZpZywKICAgICAgICAgICAgKQogICAgICAgICAgICB2b3Rlcy5hcHBlbmQoYmF0Y2hlZF9hbnN3ZXJzKQogICAgICAgIGFuc3dlcnMgPSBbXQoKICAgICAgICAjIENvbGxlY3QgdGhlIGFuc3dlcnMgYWNjb3JkaW5nIHRvIHRoZSBwb2xsIHN0cmF0ZWd5CiAgICAgICAgIyBBdmVyYWdlIHN0cmF0ZWd5IHdvcmtzIGZvciBudW1lcmljIHZhbHVlcyBvbmx5CiAgICAgICAgZm9yIGJhdGNoIGluIHJhbmdlKGxlbih2b3Rlc1swXSkpOgogICAgICAgICAgICBiYXRjaGVkX2Fuc3dlcnMgPSBbXQogICAgICAgICAgICBmb3IgcXVlc3Rpb24gaW4gcmFuZ2UocXVlc3Rpb25zX2Ftb3VudCk6CiAgICAgICAgICAgICAgICAjIENyZWF0ZSBhIGxpc3Qgb2YgYWxsIGFuc3dlcnMgdG8gcmVsZXZhbnQgcXVlc3Rpb24KICAgICAgICAgICAgICAgIGFuc3dlciA9IFsKICAgICAgICAgICAgICAgICAgICB2b3Rlc1t2b3Rlcl1bYmF0Y2hdW3F1ZXN0aW9uXSBmb3Igdm90ZXIgaW4gcmFuZ2Uoc2VsZi5wb2xsX2NvdW50KQogICAgICAgICAgICAgICAgXQogICAgICAgICAgICAgICAgYW5zd2VyID0gc2VsZi5wb2xsX3N0cmF0ZWd5LmRvKGFuc3dlcikKICAgICAgICAgICAgICAgIGJhdGNoZWRfYW5zd2Vycy5hcHBlbmQoYW5zd2VyKQogICAgICAgICAgICBhbnN3ZXJzLmFwcGVuZChiYXRjaGVkX2Fuc3dlcnMpCiAgICAgICAgcmV0dXJuIGFuc3dlcnMKCgojIEhvbGRzIG5hbWVzIG9mIFF1ZXN0aW9uSGFuZGxlcwpjbGFzcyBRdWVzdGlvblR5cGVzOgogICAgREVGQVVMVCA9ICJkZWZhdWx0IgogICAgUE9MTCA9ICJwb2xsIgoKCiMgTWFwcyBxdWVzdGlvbiB0eXBlcyB0byB0aGVpciBoYW5kbGVycwpRVUVTVElPTl9NQVBQSU5HID0gewogICAgUXVlc3Rpb25UeXBlcy5ERUZBVUxUOiBRdWVzdGlvbkhhbmRsZXIsCiAgICBRdWVzdGlvblR5cGVzLlBPTEw6IFBvbGxRdWVzdGlvbkhhbmRsZXIsCn0K requirements: - transformers - torch - tqdm code_origin: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgZW51bQppbXBvcnQgbG9nZ2luZwppbXBvcnQgb3BlcmF0b3IKaW1wb3J0IHBhdGhsaWIKZnJvbSBjb2xsZWN0aW9ucyBpbXBvcnQgQ291bnRlcgpmcm9tIGZ1bmN0b29scyBpbXBvcnQgcmVkdWNlLCB3cmFwcwpmcm9tIHR5cGluZyBpbXBvcnQgQW55LCBEaWN0LCBMaXN0LCBUdXBsZSwgVW5pb24KCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IHRyYW5zZm9ybWVycwpmcm9tIHRxZG0gaW1wb3J0IHRxZG0KCiMgR2V0IHRoZSBnbG9iYWwgbG9nZ2VyOgpfTE9HR0VSID0gbG9nZ2luZy5nZXRMb2dnZXIoKQoKCmRlZiBfY2hlY2tfbWxydW5fYW5kX29wZW5fbXBpKCkgLT4gVHVwbGVbIm1scnVuLk1MQ2xpZW50Q3R4IiwgIm1waTRweS5NUEkuSW50cmFjb21tIl06CiAgICBnbG9iYWwgX0xPR0dFUgoKICAgIGlzX21waSA9IEZhbHNlCiAgICB0cnk6CiAgICAgICAgaW1wb3J0IG1scnVuCgogICAgICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJtbHJ1biIpCiAgICAgICAgX0xPR0dFUiA9IGNvbnRleHQubG9nZ2VyCiAgICAgICAgaXNfbXBpID0gY29udGV4dC5sYWJlbHMuZ2V0KCJraW5kIiwgImpvYiIpID09ICJtcGlqb2IiCgogICAgICAgIGlmIGlzX21waToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnJvbSBtcGk0cHkgaW1wb3J0IE1QSQoKICAgICAgICAgICAgICAgIHJldHVybiBjb250ZXh0LCBNUEkuQ09NTV9XT1JMRAogICAgICAgICAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvciBhcyBtcGk0cHlfbm90X2ZvdW5kOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgIlRvIGRpc3RyaWJ1dGUgdGhlIGZ1bmN0aW9uIHVzaW5nIE1MUnVuJ3MgJ21waWpvYicgeW91IG5lZWQgdG8gaGF2ZSBgbXBpNHB5YCBwYWNrYWdlIGluIHlvdXIgIgogICAgICAgICAgICAgICAgICAgICJpbnRlcnByZXRlci4gUGxlYXNlIHJ1biBgcGlwIGluc3RhbGwgbXBpNHB5YCBhbmQgbWFrZSBzdXJlIHlvdSBoYXZlIG9wZW4tbXBpLiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIG1waTRweV9ub3RfZm91bmQKICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yIGFzIG1vZHVsZV9ub3RfZm91bmQ6CiAgICAgICAgaWYgaXNfbXBpOgogICAgICAgICAgICByYWlzZSBtb2R1bGVfbm90X2ZvdW5kCiAgICByZXR1cm4gTm9uZSwgTm9uZQoKCmRlZiBvcGVuX21waV9oYW5kbGVyKAogICAgd29ya2VyX2lucHV0czogTGlzdFtzdHJdLCByb290X3dvcmtlcl9pbnB1dHM6IERpY3Rbc3RyLCBBbnldID0gTm9uZQopOgogICAgZ2xvYmFsIF9MT0dHRVIKCiAgICAjIENoZWNrIGZvciBNTFJ1biBhbmQgT3Blbk1QSSBhdmFpbGFiaWxpdHk6CiAgICBjb250ZXh0LCBjb21tID0gX2NoZWNrX21scnVuX2FuZF9vcGVuX21waSgpCgogICAgZGVmIGRlY29yYXRvcihoYW5kbGVyKToKICAgICAgICBpZiBjb21tIGlzIE5vbmUgb3IgY29tbS5HZXRfc2l6ZSgpID09IDE6CiAgICAgICAgICAgIHJldHVybiBoYW5kbGVyCgogICAgICAgIEB3cmFwcyhoYW5kbGVyKQogICAgICAgIGRlZiB3cmFwcGVyKCoqa3dhcmdzKToKICAgICAgICAgICAgIyBHZXQgdGhlIG9wZW4gbXBpIGVudmlyb25tZW50IHByb3BlcnRpZXM6CiAgICAgICAgICAgIHNpemUgPSBjb21tLkdldF9zaXplKCkKICAgICAgICAgICAgcmFuayA9IGNvbW0uR2V0X3JhbmsoKQoKICAgICAgICAgICAgIyBHaXZlIHRoZSBjb3JyZWN0IGNodW5rIG9mIHRoZSB3b3JrZXJzIGlucHV0czoKICAgICAgICAgICAgZm9yIHdvcmtlcl9pbnB1dCBpbiB3b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBrd2FyZ3Nbd29ya2VyX2lucHV0XQogICAgICAgICAgICAgICAgaWYgaW5wdXRfYXJndW1lbnQgaXMgTm9uZToKICAgICAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgc3RyKToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IF9nZXRfdGV4dF9maWxlcygKICAgICAgICAgICAgICAgICAgICAgICAgZGF0YV9wYXRoPXBhdGhsaWIuUGF0aChpbnB1dF9hcmd1bWVudCkuYWJzb2x1dGUoKQogICAgICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGlmIGxlbihpbnB1dF9hcmd1bWVudCkgPCBzaXplOgogICAgICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICAgICAgICAgICAgIGYiQ2Fubm90IHNwbGl0IHRoZSBpbnB1dCAne3dvcmtlcl9pbnB1dH0nIG9mIGxlbmd0aCB7bGVuKGlucHV0X2FyZ3VtZW50KX0gdG8ge3NpemV9IHdvcmtlcnMuICIKICAgICAgICAgICAgICAgICAgICAgICAgZiJQbGVhc2UgcmVkdWNlIHRoZSBhbW91bnQgb2Ygd29ya2VycyBmb3IgdGhpcyBpbnB1dC4iCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZXZlbl9jaHVua19zaXplID0gbGVuKGlucHV0X2FyZ3VtZW50KSAvLyBzaXplCiAgICAgICAgICAgICAgICBjaHVua19zdGFydCA9IHJhbmsgKiBldmVuX2NodW5rX3NpemUKICAgICAgICAgICAgICAgIGNodW5rX2VuZCA9ICgKICAgICAgICAgICAgICAgICAgICAocmFuayArIDEpICogZXZlbl9jaHVua19zaXplCiAgICAgICAgICAgICAgICAgICAgaWYgcmFuayArIDEgPCBzaXplCiAgICAgICAgICAgICAgICAgICAgZWxzZSBsZW4oaW5wdXRfYXJndW1lbnQpCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgICAgIGYiUmFuayAje3Jhbmt9OiBQcm9jZXNzaW5nIGlucHV0IGNodW5rIG9mICd7d29ya2VyX2lucHV0fScgIgogICAgICAgICAgICAgICAgICAgIGYiZnJvbSBpbmRleCB7Y2h1bmtfc3RhcnR9IHRvIHtjaHVua19lbmR9LiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIGxpc3QpOgogICAgICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0gaW5wdXRfYXJndW1lbnRbY2h1bmtfc3RhcnQ6Y2h1bmtfZW5kXQogICAgICAgICAgICAgICAgZWxpZiBpc2luc3RhbmNlKGlucHV0X2FyZ3VtZW50LCBwZC5EYXRhRnJhbWUpOgogICAgICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0gaW5wdXRfYXJndW1lbnQuaWxvY1tjaHVua19zdGFydDpjaHVua19lbmQ6LCA6XQogICAgICAgICAgICAgICAga3dhcmdzW3dvcmtlcl9pbnB1dF0gPSBpbnB1dF9hcmd1bWVudAoKICAgICAgICAgICAgIyBTZXQgdGhlIHJvb3Qgd29ya2VyIG9ubHkgYXJndW1lbnRzOgogICAgICAgICAgICBpZiByYW5rID09IDAgYW5kIHJvb3Rfd29ya2VyX2lucHV0czoKICAgICAgICAgICAgICAgIGt3YXJncy51cGRhdGUocm9vdF93b3JrZXJfaW5wdXRzKQoKICAgICAgICAgICAgIyBSdW4gdGhlIHdvcmtlcjoKICAgICAgICAgICAgb3V0cHV0ID0gaGFuZGxlcigqKmt3YXJncykKCiAgICAgICAgICAgICMgU2VuZCB0aGUgb3V0cHV0IHRvIHRoZSByb290IHJhbmsgKHJhbmsgIzApOgogICAgICAgICAgICBvdXRwdXQgPSBjb21tLmdhdGhlcihvdXRwdXQsIHJvb3Q9MCkKICAgICAgICAgICAgaWYgcmFuayA9PSAwOgogICAgICAgICAgICAgICAgIyBKb2luIHRoZSBvdXRwdXRzOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygiQ29sbGVjdGluZyBkYXRhIGZyb20gd29ya2VycyB0byByb290IHdvcmtlci4iKQogICAgICAgICAgICAgICAgZGF0YWZyYW1lID0gcGQuY29uY2F0KG9ianM9W2RmIGZvciBkZiwgXyBpbiBvdXRwdXRdLCBheGlzPTApCiAgICAgICAgICAgICAgICBlcnJvcnNfZGljdGlvbmFyeSA9IHJlZHVjZShvcGVyYXRvci5pb3IsIFtlcnIgZm9yIF8sIGVyciBpbiBvdXRwdXRdLCB7fSkKICAgICAgICAgICAgICAgIHJldHVybiBkYXRhZnJhbWUsIGVycm9yc19kaWN0aW9uYXJ5CiAgICAgICAgICAgIHJldHVybiBOb25lCgogICAgICAgIHJldHVybiB3cmFwcGVyCgogICAgcmV0dXJuIGRlY29yYXRvcgoKCkBvcGVuX21waV9oYW5kbGVyKHdvcmtlcl9pbnB1dHM9WyJkYXRhX3BhdGgiXSwgcm9vdF93b3JrZXJfaW5wdXRzPXsidmVyYm9zZSI6IFRydWV9KQpkZWYgYW5zd2VyX3F1ZXN0aW9ucygKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBMaXN0W3N0cl1dLAogICAgbW9kZWxfbmFtZTogc3RyLAogICAgcXVlc3Rpb25zOiBVbmlvbltMaXN0W3N0cl0sIExpc3RbTGlzdFtzdHJdXV0sCiAgICBkZXZpY2VfbWFwOiBVbmlvbltzdHIsIGRpY3RdID0gTm9uZSwKICAgIG1vZGVsX2t3YXJnczogZGljdCA9IE5vbmUsCiAgICBhdXRvX2dwdHFfZXhsbGFtYV9tYXhfaW5wdXRfbGVuZ3RoOiBpbnQgPSBOb25lLAogICAgdG9rZW5pemVyX25hbWU6IHN0ciA9IE5vbmUsCiAgICB0b2tlbml6ZXJfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIHRleHRfd3JhcHBlcjogVW5pb25bc3RyLCBMaXN0W3N0cl1dID0gIiIsCiAgICBxdWVzdGlvbnNfd3JhcHBlcjogVW5pb25bc3RyLCBMaXN0W3N0cl1dID0gIiIsCiAgICBnZW5lcmF0aW9uX2NvbmZpZzogVW5pb25bRGljdCwgTGlzdFtEaWN0XV0gPSBOb25lLAogICAgcXVlc3Rpb25zX2NvbmZpZzogVW5pb25bRGljdCwgTGlzdFtEaWN0XV0gPSBOb25lLAogICAgYmF0Y2hfc2l6ZTogaW50ID0gMSwKICAgIHF1ZXN0aW9uc19jb2x1bW5zOiBMaXN0W3N0cl0gPSBOb25lLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopIC0+IFR1cGxlW3BkLkRhdGFGcmFtZSwgZGljdF06CiAgICAiIiIKICAgIEFuc3dlciBxdWVzdGlvbnMgd2l0aCBhIGNvbnRleHQgdG8gdGhlIGdpdmVuIHRleHQgZmlsZXMgY29udGVudHMgYnkgYSBwcmV0cmFpbmVkIExMTSBtb2RlbC4gRWFjaCB0ZXh0IGZpbGUgd2lsbCBoYXZlCiAgICB0aGUgZm9sbG93aW5nIHByb21wdCBidWlsdDoKCiAgICBzdGFydCBvZiBgdGV4dF93cmFwcGVyYAogICAgPHRleHQgZmlsZSBjb250ZW50PgogICAgZW5kIG9mIGB0ZXh0X3dyYXBwZXJgCgogICAgc3RhcnQgb2YgYHF1ZXN0aW9uc193cmFwcGVyYAogICAgMS4gPHF1ZXN0aW9uc1swXT4KICAgIDIuIDxxdWVzdGlvbnNbMV0+CiAgICAuLi4KICAgIG4uIDxxdWVzdGlvbnNbbi0xXT4KICAgIGVuZCBvZiBgcXVlc3Rpb25zX3dyYXBwZXJgCgogICAgOnBhcmFtIGRhdGFfcGF0aDogICAgICAgICAgICAgICAgICAgICAgICAgIEEgcGF0aCB0byBhIGRpcmVjdG9yeSBvZiB0ZXh0IGZpbGVzIG9yIGEgcGF0aCB0byBhIHRleHQgZmlsZSB0byBhc2sKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBxdWVzdGlvbnMgYWJvdXQuCiAgICA6cGFyYW0gbW9kZWxfbmFtZTogICAgICAgICAgICAgICAgICAgICAgICAgVGhlIHByZS10cmFpbmVkIG1vZGVsIG5hbWUgZnJvbSB0aGUgaHVnZ2luZ2ZhY2UgaHViIHRvIHVzZSBmb3IgYXNraW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcXVlc3Rpb25zLgogICAgOnBhcmFtIHF1ZXN0aW9uczogICAgICAgICAgICAgICAgICAgICAgICAgIFRoZSBxdWVzdGlvbnMgdG8gYXNrLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIEEgbGlzdCBvZiBsaXN0cyBvZiBxdWVzdGlvbnMgdG8gYXNrIHBlciB0ZXh0IGZpbGUsIGFuZCBkZXZpZGVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYnkgcXVlc3Rpb24gZ3JvdXBzLCB0aGUgZ3JvdXBzIGNhbiBiZSBkdGVybWFpbmVkIGJ5IHNpemUgKGluIG9yZGVyIHRvCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYXZvaWQgbGFyZ2UgaW5wdXRzIHRvIHRoZSBsbG0pIG9yIGJ5IHF1ZXN0aW9uaW5nIG1ldGhvZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChyZWd1bGFyIG9yIHBvbGwgbGlrZSBxdWVzdGlvbmluZykuCiAgICA6cGFyYW0gZGV2aWNlX21hcDogICAgICAgICAgICAgICAgICAgICAgICAgQSBtYXAgdG8gdXNlIGZvciBsb2FkaW5nIHRoZSBtb2RlbCBvbiBtdWx0aXBsZSBkZXZpY2VzLgogICAgOnBhcmFtIG1vZGVsX2t3YXJnczogICAgICAgICAgICAgICAgICAgICAgIEtleXdvcmQgYXJndW1lbnRzIHRvIHBhc3MgZm9yIGxvYWRpbmcgdGhlIG1vZGVsIHVzaW5nIEh1Z2dpbmdGYWNlJ3MKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBgdHJhbnNmb3JtZXJzLkF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZGAgZnVuY3Rpb24uCiAgICA6cGFyYW0gYXV0b19ncHRxX2V4bGxhbWFfbWF4X2lucHV0X2xlbmd0aDogRm9yIEF1dG9HUFRRIG1vZGVscyB0byBzZXQgYW5kIGV4dGVuZCB0aGUgbW9kZWwncyBpbnB1dCBidWZmZXIgc2l6ZS4KICAgIDpwYXJhbSB0b2tlbml6ZXJfbmFtZTogICAgICAgICAgICAgICAgICAgICBUaGUgdG9rZW5pemVyIG5hbWUgZnJvbSB0aGUgaHVnZ2luZ2ZhY2UgaHViIHRvIHVzZS4gSWYgbm90IGdpdmVuLCB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBtb2RlbCBuYW1lIHdpbGwgYmUgdXNlZC4KICAgIDpwYXJhbSB0b2tlbml6ZXJfa3dhcmdzOiAgICAgICAgICAgICAgICAgICBLZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIGZvciBsb2FkaW5nIHRoZSB0b2tlbml6ZXIgdXNpbmcgSHVnZ2luZ0ZhY2UncwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGB0cmFuc2Zvcm1lcnMuQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWRgIGZ1bmN0aW9uLgogICAgOnBhcmFtIHRleHRfd3JhcHBlcjogICAgICAgICAgICAgICAgICAgICAgIEEgd3JhcHBlciBmb3IgdGhlIGZpbGUncyB0ZXh0LiBXaWxsIGJlIGFkZGVkIGF0IHRoZSBzdGFydCBvZiB0aGUgcHJvbXB0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE11c3QgaGF2ZSBhIHBsYWNlaG9sZGVyICgne30nKSBmb3IgdGhlIHRleHQgb2YgdGhlIGZpbGUuCiAgICA6cGFyYW0gcXVlc3Rpb25zX3dyYXBwZXI6ICAgICAgICAgICAgICAgICAgQSB3cmFwcGVyIGZvciB0aGUgcXVlc3Rpb25zIHJlY2VpdmVkLiBXaWxsIGJlIGFkZGVkIGFmdGVyIHRoZSB0ZXh0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd3JhcHBlciBpbiB0aGUgcHJvbXB0IHRlbXBsYXRlLiBNdXN0IGhhdmUgYSBwbGFjZWhvbGRlciAoJ3t9JykgZm9yIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHF1ZXN0aW9ucy4KICAgIDpwYXJhbSBnZW5lcmF0aW9uX2NvbmZpZzogICAgICAgICAgICAgICAgICBIdWdnaW5nRmFjZSdzIGBHZW5lcmF0aW9uQ29uZmlnYCBrZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIHRvIHRoZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBnZW5lcmF0ZWAgbWV0aG9kLgogICAgOnBhcmFtIHF1ZXN0aW9uc19jb25maWc6ICAgICAgICAgICAgICAgICAgIEEgZGljdGlvbmFyeSBvciBsaXN0IG9mIGRpY3Rpb25hcmllcyBjb250YWluaW5nIHNwZWNpZmljIHdheXMgdG8gYW5zd2VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcXVlc3Rpb25zICh1c2luZyBhIHBvbGwgZm9yIGV4YW1wbGUpLCBlYWNoIGRpY3Rpb25hcnkgaW4gdGhlIGxpc3QgaXMgZm9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgY29ycmVzcG9uZGluZyBxdWVzdGlvbiBncm91cCBhbmQgZGV0ZXJtaW5lcyB0aGUgcXVlc3Rpb24gYXNraW5nIG1ldGhvZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZvciBzYWlkIGdyb3VwLgogICAgOnBhcmFtIGJhdGNoX3NpemU6ICAgICAgICAgICAgICAgICAgICAgICAgIEJhdGNoIHNpemUgZm9yIGluZmVyZW5jZS4KICAgIDpwYXJhbSBxdWVzdGlvbnNfY29sdW1uczogICAgICAgICAgICAgICAgICBDb2x1bW5zIHRvIHVzZSBmb3IgdGhlIGRhdGFmcmFtZSByZXR1cm5lZC4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHByZXNlbnQgbG9ncyBvZiBhIHByb2dyZXNzIGJhciBhbmQgZXJyb3JzLiBEZWZhdWx0OiBUcnVlLgoKCiAgICA6cmV0dXJuczogQSB0dXBsZSBvZjoKCiAgICAgICAgICAgICAgKiBBIGRhdGFmcmFtZSBkYXRhc2V0IG9mIHRoZSBxdWVzdGlvbnMgYW5zd2Vycy4KICAgICAgICAgICAgICAqIEEgZGljdGlvbmFyeSBvZiBlcnJvcmVkIGZpbGVzIHRoYXQgd2VyZSBub3QgaW5mZXJyZWQgb3Igd2VyZSBub3QgYW5zd2VyZWQgcHJvcGVybHkuCiAgICAiIiIKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBTZXQgY29uZmlncyB0byBlbXB0eSBkaWN0IGlmIG5vdCBnaXZlbjoKICAgIGlmIGdlbmVyYXRpb25fY29uZmlnIGlzIE5vbmU6CiAgICAgICAgZ2VuZXJhdGlvbl9jb25maWcgPSB7fQogICAgaWYgcXVlc3Rpb25zX2NvbmZpZyBpcyBOb25lOgogICAgICAgIHF1ZXN0aW9uc19jb25maWcgPSB7fQoKICAgICMgR2V0IHRoZSBpbnB1dCB0ZXh0IGZpbGVzIHRvIHF1ZXN0aW9uOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgdGV4dCBmaWxlcy4iKQogICAgaWYgaXNpbnN0YW5jZShkYXRhX3BhdGgsIHN0cik6CiAgICAgICAgZGF0YV9wYXRoID0gcGF0aGxpYi5QYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQogICAgICAgIHRleHRfZmlsZXMgPSBfZ2V0X3RleHRfZmlsZXMoZGF0YV9wYXRoPWRhdGFfcGF0aCkKICAgIGVsc2U6CiAgICAgICAgdGV4dF9maWxlcyA9IGRhdGFfcGF0aAogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbih0ZXh0X2ZpbGVzKX0gdGV4dCBmaWxlcy4iKQoKICAgICMgR2V0IHRoZSBwcm9tcHQgdGVtcGxhdGU6CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiQ3JlYXRpbmcgcHJvbXB0IHRlbXBsYXRlLiIpCgogICAgIyBPcmdhbml6ZSBxdWVzdGlvbnMgYXMgYSBsaXN0IG9mIGxpc3QsIGFuZCBjb3VudCBudW1iZXIgb2Ygc3ViLWxpc3RzIGZvciBmdXR1cmUgdXNlCiAgICBudW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzID0gMSBpZiBpc2luc3RhbmNlKHF1ZXN0aW9uc1swXSwgc3RyKSBlbHNlIGxlbihxdWVzdGlvbnMpCiAgICBxdWVzdGlvbnMgPSBfdG9fZ3JvdXBfbGlzdCgKICAgICAgICBhcmd1bWVudF92YWx1ZT1xdWVzdGlvbnMsCiAgICAgICAgYXJndW1lbnRfbmFtZT0icXVlc3Rpb25zIiwKICAgICAgICBsZW5ndGg9bnVtYmVyX29mX3F1ZXN0aW9uX2dyb3VwcywKICAgICkKCiAgICAjIE9yZ2FuaXplIHByb21wdCBwYXJ0cyBhdCBwcm9wZXIgbGVuZ3RoCiAgICB0ZXh0X3dyYXBwZXIgPSBfdG9fZ3JvdXBfbGlzdCgKICAgICAgICBhcmd1bWVudF92YWx1ZT10ZXh0X3dyYXBwZXIsCiAgICAgICAgYXJndW1lbnRfbmFtZT0idGV4dF93cmFwcGVyIiwKICAgICAgICBsZW5ndGg9bnVtYmVyX29mX3F1ZXN0aW9uX2dyb3VwcywKICAgICkKICAgIHF1ZXN0aW9uc193cmFwcGVyID0gX3RvX2dyb3VwX2xpc3QoCiAgICAgICAgYXJndW1lbnRfdmFsdWU9cXVlc3Rpb25zX3dyYXBwZXIsCiAgICAgICAgYXJndW1lbnRfbmFtZT0icXVlc3Rpb25zX3dyYXBwZXIiLAogICAgICAgIGxlbmd0aD1udW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzLAogICAgKQoKICAgICMgQ3JlYXRlIGEgbGlzdCBvZiBwcm9tcHQgYWNjb3JkaW5nIHRvIGdpdmVuIHBhcnRzIGFuZCBxdWVzdGlvbnMKICAgIHByb21wdF90ZW1wbGF0ZSA9IFtdCiAgICBxdWVzdGlvbnMgPSBxdWVzdGlvbnMgaWYgaXNpbnN0YW5jZShxdWVzdGlvbnNbMF0sIGxpc3QpIGVsc2UgW3F1ZXN0aW9uc10KCiAgICAjIEJ1aWxkIGFsbCBwcm9tcHRzCiAgICBmb3IgaSBpbiByYW5nZShudW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzKToKICAgICAgICBwcm9tcHRfdGVtcGxhdGUuYXBwZW5kKAogICAgICAgICAgICBfZ2V0X3Byb21wdF90ZW1wbGF0ZSgKICAgICAgICAgICAgICAgIHRleHRfd3JhcHBlcj10ZXh0X3dyYXBwZXJbaV0sCiAgICAgICAgICAgICAgICBxdWVzdGlvbnNfd3JhcHBlcj1xdWVzdGlvbnNfd3JhcHBlcltpXSwKICAgICAgICAgICAgICAgIHF1ZXN0aW9ucz1xdWVzdGlvbnNbaV0sCiAgICAgICAgICAgICkKICAgICAgICApCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIlByb21wdCB0ZW1wbGF0ZSBjcmVhdGVkOlxuXG57cHJvbXB0X3RlbXBsYXRlfVxuIikKCiAgICAjIEdldCB0aGUgdG90YWwgYW1vdW50IG9mIHF1ZXN0aW9uczoKICAgIHF1ZXN0aW9uc19hbW91bnQgPSBzdW0oW2xlbihzdWJsaXN0KSBmb3Igc3VibGlzdCBpbiBxdWVzdGlvbnNdKQoKICAgICMgR2V0IHRoZSBxdWVzdGlvbnMgY29sdW1uczoKICAgIHF1ZXN0aW9uc19jb2x1bW5zID0gcXVlc3Rpb25zX2NvbHVtbnMgb3IgWwogICAgICAgIGYicXtpfSIgZm9yIGkgaW4gcmFuZ2UoMSwgcXVlc3Rpb25zX2Ftb3VudCArIDEpCiAgICBdCgogICAgIyBDaGVjayBpZiB3ZSBoYXZlIHRoZSBjb3JyZWN0IGFtb3VudCBvZiBxdWVzdGlvbnMgY29sdW1uczoKICAgIGlmIGxlbihxdWVzdGlvbnNfY29sdW1ucykgIT0gcXVlc3Rpb25zX2Ftb3VudDoKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlRoZSBwcm92aWRlZCBxdWVzdGlvbnMgY29sdW1ucyBsZW5ndGggKHtsZW4ocXVlc3Rpb25zX2NvbHVtbnMpfSkgIgogICAgICAgICAgICBmImRvZXMgbm90IG1hdGNoIHRoZSBxdWVzdGlvbnMgYW1vdW50ICh7cXVlc3Rpb25zX2Ftb3VudH0pIgogICAgICAgICkKCiAgICAjIExvYWQgdGhlIGdlbmVyYXRpb24gY29uZmlnOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkxvYWRpbmcgZ2VuZXJhdGlvbiBjb25maWd1cmF0aW9uLiIpCiAgICBnZW5lcmF0aW9uX2NvbmZpZyA9IFsKICAgICAgICB0cmFuc2Zvcm1lcnMuR2VuZXJhdGlvbkNvbmZpZygqKihjZmcgb3Ige30pKQogICAgICAgIGZvciBjZmcgaW4gX3RvX2dyb3VwX2xpc3QoCiAgICAgICAgICAgIGFyZ3VtZW50X3ZhbHVlPWdlbmVyYXRpb25fY29uZmlnLAogICAgICAgICAgICBhcmd1bWVudF9uYW1lPSJnZW5lcmF0aW9uX2NvbmZpZyIsCiAgICAgICAgICAgIGxlbmd0aD1udW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzLAogICAgICAgICkKICAgIF0KICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiR2VuZXJhdGlvbiBjb25maWd1cmF0aW9uIGxvYWRlZDoge2dlbmVyYXRpb25fY29uZmlnfSIpCgogICAgIyBMb2FkIHRoZSBtb2RlbCBhbmQgdG9rZW5pemVyIGludG8gYSBwaXBlbGluZSBvYmplY3Q6CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkxvYWRpbmcgbW9kZWwgJ3ttb2RlbF9uYW1lfScuIikKICAgIGdlbmVyYXRpb25fcGlwZWxpbmUgPSBfZ2V0X2dlbmVyYXRpb25fcGlwZWxpbmUoCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGRldmljZV9tYXA9ZGV2aWNlX21hcCwKICAgICAgICB0b2tlbml6ZXJfbmFtZT10b2tlbml6ZXJfbmFtZSBvciBtb2RlbF9uYW1lLAogICAgICAgIG1vZGVsX2t3YXJncz1tb2RlbF9rd2FyZ3Mgb3Ige30sCiAgICAgICAgdG9rZW5pemVyX2t3YXJncz10b2tlbml6ZXJfa3dhcmdzIG9yIHt9LAogICAgICAgIGF1dG9fZ3B0cV9leGxsYW1hX21heF9pbnB1dF9sZW5ndGg9YXV0b19ncHRxX2V4bGxhbWFfbWF4X2lucHV0X2xlbmd0aCwKICAgICAgICBiYXRjaF9zaXplPWJhdGNoX3NpemUsCiAgICApCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiTW9kZWwgbG9hZGVkLiIpCgogICAgIyBQcmVwYXJlIHRoZSBzdWNjZXNzZXMgZGF0YWZyYW1lIGFuZCBlcnJvcnMgZGljdGlvbmFyeSB0byBiZSByZXR1cm5lZDoKICAgIHN1Y2Nlc3NlcyA9IFtdCiAgICBlcnJvcnMgPSB7fQoKICAgICMgU3BsaXQgdGhlIGZpbGVzIGludG8gYmF0Y2hlczoKICAgIGZpbGVfYmF0Y2hlcyA9IFsKICAgICAgICB0ZXh0X2ZpbGVzW2kgOiBpICsgYmF0Y2hfc2l6ZV0KICAgICAgICBpZiBpICsgYmF0Y2hfc2l6ZSA8IGxlbih0ZXh0X2ZpbGVzKQogICAgICAgIGVsc2UgdGV4dF9maWxlc1tpOl0KICAgICAgICBmb3IgaSBpbiByYW5nZSgwLCBsZW4odGV4dF9maWxlcyksIGJhdGNoX3NpemUpCiAgICBdCiAgICBxdWVzdGlvbnNfY29uZmlnID0gX3RvX2dyb3VwX2xpc3QoCiAgICAgICAgYXJndW1lbnRfdmFsdWU9cXVlc3Rpb25zX2NvbmZpZywKICAgICAgICBhcmd1bWVudF9uYW1lPSJxdWVzdGlvbnNfY29uZmlnIiwKICAgICAgICBsZW5ndGg9bnVtYmVyX29mX3F1ZXN0aW9uX2dyb3VwcywKICAgICkKCiAgICAjIENyZWF0ZSBhIGxpc3Qgb2YgcXVlc3Rpb24gaGFuZGxlcnMgYWNjb3JkaW5nIHRvIGdpdmVuIGNvbmZpZ3MKICAgIGhhbmRsZXJzID0gW10KICAgIGZvciBjZmcgaW4gcXVlc3Rpb25zX2NvbmZpZzoKICAgICAgICBxdWVzdGlvbl90eXBlID0gY2ZnLnBvcCgidHlwZSIsICJkZWZhdWx0IikKICAgICAgICBoYW5kbGVycy5hcHBlbmQoUVVFU1RJT05fTUFQUElORy5nZXQocXVlc3Rpb25fdHlwZSkoKipjZmcpKQoKICAgICMgR28gb3ZlciB0aGUgYmF0Y2hlcyBvZiB0ZXh0IGZpbGVzIGFuZCBxdWVzdGlvbiB0aGVtOgogICAgZm9yIGZpbGVfYmF0Y2ggaW4gdHFkbSgKICAgICAgICBmaWxlX2JhdGNoZXMsCiAgICAgICAgZGVzYz0iR2VuZXJhdGluZyBhbnN3ZXJzIiwKICAgICAgICB1bml0PWYiZmlsZSAoYmF0Y2ggb2Yge2JhdGNoX3NpemV9KSIsCiAgICAgICAgZGlzYWJsZT1ub3QgdmVyYm9zZSwKICAgICk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICB0b3RhbF9hbnN3ZXJzID0gW1tdIGZvciBfIGluIHJhbmdlKGJhdGNoX3NpemUpXQoKICAgICAgICAgICAgIyBHbyBvdmVyIGFsbCBxdWVzdGlvbiBncm91cCBwZXIgYmF0Y2ggb2YgZG9jdW1lbnRzCiAgICAgICAgICAgIGZvciBxdWVzdGlvbl9ncm91cCBpbiByYW5nZShudW1iZXJfb2ZfcXVlc3Rpb25fZ3JvdXBzKToKICAgICAgICAgICAgICAgIGN1cnJlbnRfcXVlc3Rpb25zX2Ftb3VudCA9IGxlbihxdWVzdGlvbnNbcXVlc3Rpb25fZ3JvdXBdKQoKICAgICAgICAgICAgICAgICMgUmVhZCBiYXRjaCAocmVhZCB0aGUgdGV4dCBmcm9tIHRoZSB0ZXh0IGZpbGVzKToKICAgICAgICAgICAgICAgIGJhdGNoZWRfaW5wdXQgPSBfcmVhZF9maWxlX2JhdGNoKAogICAgICAgICAgICAgICAgICAgIGZpbGVfYmF0Y2g9ZmlsZV9iYXRjaCwKICAgICAgICAgICAgICAgICAgICBwcm9tcHRfdGVtcGxhdGU9cHJvbXB0X3RlbXBsYXRlW3F1ZXN0aW9uX2dyb3VwXSwKICAgICAgICAgICAgICAgICkKCiAgICAgICAgICAgICAgICAjIEFuc3dlciB0aGUgcXVlc3Rpb25zIHdpdGggZWFjaCBxdWVzdGlvbiBoYW5kbGVyOgogICAgICAgICAgICAgICAgYmF0Y2hlZF9hbnN3ZXJzID0gaGFuZGxlcnNbcXVlc3Rpb25fZ3JvdXBdLmFuc3dlcigKICAgICAgICAgICAgICAgICAgICBxdWVzdGlvbnNfYW1vdW50PWN1cnJlbnRfcXVlc3Rpb25zX2Ftb3VudCwKICAgICAgICAgICAgICAgICAgICBiYXRjaGVkX2lucHV0PWJhdGNoZWRfaW5wdXQsCiAgICAgICAgICAgICAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZT1nZW5lcmF0aW9uX3BpcGVsaW5lLAogICAgICAgICAgICAgICAgICAgIGdlbmVyYXRpb25fY29uZmlnPWdlbmVyYXRpb25fY29uZmlnW3F1ZXN0aW9uX2dyb3VwXSwKICAgICAgICAgICAgICAgICkKCiAgICAgICAgICAgICAgICAjIFB1dCB0aGUgYW5zd2VycyBpbiB0aGUgY29ycmVjdCBwbGFjZSBpbiB0aGUgdG90YWwgYW5zd2VycyBsaXN0IGFjY29yZGluZyB0byB0aGUgcGxhY2UgaW4gdGhlIGJhdGNoOgogICAgICAgICAgICAgICAgZm9yIGkgaW4gcmFuZ2UoYmF0Y2hfc2l6ZSk6CiAgICAgICAgICAgICAgICAgICAgdG90YWxfYW5zd2Vyc1tpXS5leHRlbmQoYmF0Y2hlZF9hbnN3ZXJzW2ldKQoKICAgICAgICAgICAgIyBDb2xsZWN0IHRoZSBhbnN3ZXJzIGFuZCBhdHRhY2ggdGhlIGZpbGUgbmFtZToKICAgICAgICAgICAgc3VjY2Vzc2VzLmV4dGVuZCgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICBbZmlsZS5uYW1lLCAqYW5zd2Vyc10KICAgICAgICAgICAgICAgICAgICBmb3IgZmlsZSwgYW5zd2VycyBpbiB6aXAoZmlsZV9iYXRjaCwgdG90YWxfYW5zd2VycykKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIE5vdGUgdGhlIGV4Y2VwdGlvbiBhcyBlcnJvciBpbiB0aGUgZGljdGlvbmFyeToKICAgICAgICAgICAgYmF0Y2hfZmlsZV9uYW1lcyA9ICIsICIuam9pbihbZmlsZS5uYW1lIGZvciBmaWxlIGluIGZpbGVfYmF0Y2hdKQogICAgICAgICAgICBpZiB2ZXJib3NlOgogICAgICAgICAgICAgICAgX0xPR0dFUi53YXJuaW5nKAogICAgICAgICAgICAgICAgICAgIGYiRXJyb3IgaW4gYmF0Y2ggJ3tiYXRjaF9maWxlX25hbWVzfSc6IHtzdHIoZXhjZXB0aW9uKX0iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgIGVycm9yc1tiYXRjaF9maWxlX25hbWVzXSA9IHN0cihleGNlcHRpb24pCiAgICAgICAgICAgIGNvbnRpbnVlCgogICAgIyBDb25zdHJ1Y3QgdGhlIGFuc3dlcnMgZGF0YWZyYW1lOgogICAgY29sdW1ucyA9IFsKICAgICAgICAidGV4dF9maWxlIiwKICAgICAgICAqcXVlc3Rpb25zX2NvbHVtbnMsCiAgICBdCgogICAgIyBDcmVhdGUgYSBkYXRhIGZyYW1lIG9mIGFuc3dlcnMgYnkgZmlsZXMKICAgIHN1Y2Nlc3NlcyA9IHBkLkRhdGFGcmFtZSgKICAgICAgICBzdWNjZXNzZXMsCiAgICAgICAgY29sdW1ucz1jb2x1bW5zLAogICAgKQoKICAgICMgUHJpbnQgdGhlIGhlYWQgb2YgdGhlIHByb2R1Y2VkIGRhdGFmcmFtZSBhbmQgcmV0dXJuOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oCiAgICAgICAgICAgIGYiRG9uZSAoe3N1Y2Nlc3Nlcy5zaGFwZVswXX0ve2xlbih0ZXh0X2ZpbGVzKX0pXG4iCiAgICAgICAgICAgIGYiQW5zd2VycyBzdW1tYXJ5OlxuIgogICAgICAgICAgICBmIntzdWNjZXNzZXMuaGVhZCgpfSIKICAgICAgICApCiAgICByZXR1cm4gc3VjY2Vzc2VzLCBlcnJvcnMKCgpkZWYgX2dldF90ZXh0X2ZpbGVzKAogICAgZGF0YV9wYXRoOiBwYXRobGliLlBhdGgsCikgLT4gTGlzdFtwYXRobGliLlBhdGhdOgoKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgoKICAgICAgICAjIEdldCBhbGwgZmlsZXMgaW5zaWRlIHRoZSBkaXJlY3Rvcnk6CiAgICAgICAgdGV4dF9maWxlcyA9IGxpc3QoZGF0YV9wYXRoLmdsb2IoIiouKiIpKQogICAgZWxpZiBkYXRhX3BhdGguaXNfZmlsZSgpOgogICAgICAgIHRleHRfZmlsZXMgPSBbZGF0YV9wYXRoXQogICAgZWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlVucmVjb2duaXplZCBkYXRhIHBhdGguIFRoZSBwYXJhbWV0ZXIgYGRhdGFfcGF0aGAgbXVzdCBiZSBlaXRoZXIgYSBkaXJlY3RvcnkgcGF0aCBvciBhIGZpbGUgcGF0aC4gIgogICAgICAgICAgICBmIkdpdmVuOiB7c3RyKGRhdGFfcGF0aCl9ICIKICAgICAgICApCgogICAgcmV0dXJuIHRleHRfZmlsZXMKCgpkZWYgX2dldF9wcm9tcHRfdGVtcGxhdGUoCiAgICB0ZXh0X3dyYXBwZXI6IHN0ciwKICAgIHF1ZXN0aW9uc193cmFwcGVyOiBzdHIsCiAgICBxdWVzdGlvbnM6IExpc3Rbc3RyXSwKKSAtPiBzdHI6CgogICAgIyBWYWxpZGF0ZSBhbmQgYnVpbGQgdGhlIHRleHQgd3JhcHBlcjoKICAgIHRleHRfd3JhcHBlciA9IHRleHRfd3JhcHBlciBvciAoCiAgICAgICAgIkdpdmVuIHRoZSBmb2xsb3dpbmcgdGV4dDpcbiIgIi0tLS0tXG4iICJ7fVxuIiAiLS0tLS0iCiAgICApCiAgICBpZiB0ZXh0X3dyYXBwZXIuY291bnQoInt9IikgIT0gMToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAiVGhlIGB0ZXh0X3dyYXBwZXJgIG11c3QgaW5jbHVkZSBvbmUgcGxhY2Vob2xkZXIgJ3t9JyBmb3IgdGhlIHRleHQgb2YgdGhlIGZpbGUgdG8gYmUgYXNrZWQgYWJvdXQuIgogICAgICAgICkKCiAgICAjIFZhbGlkYXRlIGFuZCBidWlsZCB0aGUgcXVlc3Rpb24gd3JhcHBlcjoKICAgIHF1ZXN0aW9uc193cmFwcGVyID0gcXVlc3Rpb25zX3dyYXBwZXIgb3IgIkFuc3dlciB0aGUgcXVlc3Rpb25zOlxuIiAie30iCiAgICBpZiBxdWVzdGlvbnNfd3JhcHBlci5jb3VudCgie30iKSAhPSAxOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICJUaGUgYHF1ZXN0aW9uc193cmFwcGVyYCBtdXN0IGluY2x1ZGUgb25lIHBsYWNlaG9sZGVyICd7fScgZm9yIHRoZSBsaXN0IG9mIHF1ZXN0aW9ucy4iCiAgICAgICAgKQoKICAgICMgVmFsaWRhdGUgYW5kIHBhcnNlIHRoZSBxdWVzdGlvbnM6CiAgICBpZiBsZW4ocXVlc3Rpb25zKSA9PSAwOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoIlBsZWFzZSBpbmNsdWRlIGF0IGxlYXN0IG9uZSBxdWVzdGlvbi4iKQogICAgcXVlc3Rpb25zID0gIlxuIi5qb2luKAogICAgICAgIFtmIntpfS4ge3F1ZXN0aW9ufSIgZm9yIGksIHF1ZXN0aW9uIGluIGVudW1lcmF0ZShxdWVzdGlvbnMsIDEpXQogICAgKQoKICAgICMgQ29uc3RydWN0IHRoZSB0ZW1wbGF0ZToKICAgIHJldHVybiBmInt0ZXh0X3dyYXBwZXJ9XG57cXVlc3Rpb25zX3dyYXBwZXIuZm9ybWF0KHF1ZXN0aW9ucyl9XG4iCgoKZGVmIF9nZXRfZ2VuZXJhdGlvbl9waXBlbGluZSgKICAgIG1vZGVsX25hbWU6IHN0ciwKICAgIGRldmljZV9tYXA6IFVuaW9uW3N0ciwgZGljdF0sCiAgICB0b2tlbml6ZXJfbmFtZTogc3RyLAogICAgbW9kZWxfa3dhcmdzOiBkaWN0LAogICAgdG9rZW5pemVyX2t3YXJnczogZGljdCwKICAgIGF1dG9fZ3B0cV9leGxsYW1hX21heF9pbnB1dF9sZW5ndGg6IGludCA9IE5vbmUsCiAgICBiYXRjaF9zaXplOiBpbnQgPSAxLAopOgogICAgIyBMb2FkIHRoZSBtb2RlbDoKICAgIG1vZGVsID0gdHJhbnNmb3JtZXJzLkF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCgKICAgICAgICBtb2RlbF9uYW1lLCBkZXZpY2VfbWFwPWRldmljZV9tYXAsICoqbW9kZWxfa3dhcmdzCiAgICApCgogICAgIyBTZXQgZXhsbGFtYSBtYXggaW5wdXQgbGVuZ3RoIGlmIHByb3ZpZGVkOgogICAgIyBUaGlzIGNoYW5nZXMgdGhlIG1vZGVsJ3MgY29udGV4dCBzaXplLgogICAgaWYgYXV0b19ncHRxX2V4bGxhbWFfbWF4X2lucHV0X2xlbmd0aDoKICAgICAgICBmcm9tIGF1dG9fZ3B0cSBpbXBvcnQgZXhsbGFtYV9zZXRfbWF4X2lucHV0X2xlbmd0aAoKICAgICAgICBtb2RlbCA9IGV4bGxhbWFfc2V0X21heF9pbnB1dF9sZW5ndGgoCiAgICAgICAgICAgIG1vZGVsPW1vZGVsLCBtYXhfaW5wdXRfbGVuZ3RoPWF1dG9fZ3B0cV9leGxsYW1hX21heF9pbnB1dF9sZW5ndGgKICAgICAgICApCgogICAgIyBMb2FkIHRoZSB0b2tlbml6ZXI6CiAgICB0b2tlbml6ZXIgPSB0cmFuc2Zvcm1lcnMuQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoCiAgICAgICAgdG9rZW5pemVyX25hbWUsICoqdG9rZW5pemVyX2t3YXJncwogICAgKQoKICAgICMgSW5pdGlhbGl6ZSBhIGdlbmVyYXRpb24gcGlwbGluZSBhbmQgcmV0dXJuOgogICAgcGlwZSA9IHRyYW5zZm9ybWVycy5waXBlbGluZSgKICAgICAgICB0YXNrPSJ0ZXh0LWdlbmVyYXRpb24iLAogICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgIHRva2VuaXplcj10b2tlbml6ZXIsCiAgICAgICAgYmF0Y2hfc2l6ZT1iYXRjaF9zaXplLAogICAgKQogICAgcGlwZS50b2tlbml6ZXIucGFkX3Rva2VuX2lkID0gbW9kZWwuY29uZmlnLmVvc190b2tlbl9pZAogICAgcmV0dXJuIHBpcGUKCgpkZWYgX3JlYWRfZmlsZV9iYXRjaCgKICAgIGZpbGVfYmF0Y2g6IExpc3RbcGF0aGxpYi5QYXRoXSwKICAgIHByb21wdF90ZW1wbGF0ZTogc3RyLAopIC0+IExpc3Rbc3RyXToKICAgIGJhdGNoID0gW10KCiAgICAjIEdvIG92ZXIgYWxsIGZpbGVzIGFuZCByZWFkIGluIHVzYWJsZSBmb3JtYXQKICAgIGZvciBmaWxlIGluIGZpbGVfYmF0Y2g6CiAgICAgICAgd2l0aCBvcGVuKGZpbGUsICJyIiwgZW5jb2Rpbmc9InV0Zi04IikgYXMgZnA6CiAgICAgICAgICAgIGJhdGNoLmFwcGVuZChwcm9tcHRfdGVtcGxhdGUuZm9ybWF0KGZwLnJlYWQoKSkpCiAgICByZXR1cm4gYmF0Y2gKCgpkZWYgX3RvX2dyb3VwX2xpc3QoYXJndW1lbnRfdmFsdWU6IGxpc3QsIGFyZ3VtZW50X25hbWU6IHN0ciwgbGVuZ3RoOiBpbnQpOgoKICAgICMgQ2hlY2sgaWYgaXMgbGlzdCwgdHVybiB0byBsaXN0IGlmIG5vdAogICAgYXJndW1lbnRfdmFsdWUgPSAoCiAgICAgICAgYXJndW1lbnRfdmFsdWUgaWYgaXNpbnN0YW5jZShhcmd1bWVudF92YWx1ZSwgbGlzdCkgZWxzZSBbYXJndW1lbnRfdmFsdWVdCiAgICApCiAgICBsaXN0X2xlbiA9IGxlbihhcmd1bWVudF92YWx1ZSkKCiAgICAjIElmIG5vdCBhIGxpc3QsIG9yIGlzIGEgbGlzdCBvZiBsZW4gMSB3ZSBkdXBsaWNhdGUgZm9yIGNvcnJlY3QgbGVuZ3RoCiAgICAjIElmIGxpc3QgaW4gd3JvbmcgbGVuZ3RoIHRocm93IGFuIGVycm9yCiAgICBpZiBsaXN0X2xlbiAhPSBsZW5ndGg6CiAgICAgICAgaWYgbGlzdF9sZW4gPT0gMToKICAgICAgICAgICAgcmV0dXJuIGFyZ3VtZW50X3ZhbHVlICogbGVuZ3RoCiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJUaGUgYXJndW1lbnQgdmFsdWUgb2YgJ3thcmd1bWVudF9uYW1lfScgaXMgbm90IGVxdWFsIHRvIHRoZSBsZW5ndGggb2YgdGhlIGdpdmVuIHF1ZXN0aW9ucyAtIHtsZW5ndGh9IgogICAgICAgICkKICAgIHJldHVybiBhcmd1bWVudF92YWx1ZQoKCmNsYXNzIFF1ZXN0aW9uSGFuZGxlcjoKICAgICIiIgogICAgQSBjbGFzcyBmb3IgaGFuZGxpbmcgcXVlc3Rpb25zIGFuc3dlcmluZyBmb3IgYSBnaXZlbiBxdWVzdGlvbiB0eXBlLgogICAgVGhpcyBjbGFzcyBpcyB1c2VkIGFzIGEgYmFzZSBjbGFzcyBmb3IgYWxsIHF1ZXN0aW9uIHR5cGVzLCBhbmQgZm9yIGRlZmF1bHQgcXVlc3Rpb24gdHlwZSAocmVndWxhciBxdWVzdGlvbgogICAgYW5zd2VyaW5nIHdpdGhvdXQgYW55IHNwZWNpYWwgaGFuZGxpbmcpLgogICAgIiIiCgogICAgY2xhc3MgQ29uZmlnS2V5czoKICAgICAgICBwYXNzCgogICAgZGVmIF9faW5pdF9fKHNlbGYpOgogICAgICAgIHBhc3MKCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgX2dldF9hbnN3ZXJzKGdlbmVyYXRlZF90ZXh0OiBzdHIsIHF1ZXN0aW9uc19hbW91bnQ6IGludCkgLT4gTGlzdFtzdHJdOgoKICAgICAgICAjIENsZWFyIGFuc3dlciBzdGFydCAocGFydCBiZWZvcmUgbnVtYmVycyk6CiAgICAgICAgIyBUT0RPIGZpbmQgYmV0dGVyIHdheSB0byB2ZXJpZnksIGZvciBsaXN0IG9mIHF1ZXN0aW9ucyB0aGlzIGlzIHJlZHVuZGFudCBmb3IgZXhhbXBsZQogICAgICAgIGlmICIxLiIgbm90IGluIGdlbmVyYXRlZF90ZXh0OgogICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgZiJBbnN3ZXIgMS4gaXMgbWlzc2luZyBmcm9tIHRoZSBnZW5lcmF0ZWQgdGV4dDogJ3tnZW5lcmF0ZWRfdGV4dH0nIgogICAgICAgICAgICApCiAgICAgICAgdGV4dCA9IGdlbmVyYXRlZF90ZXh0LnNwbGl0KCIxLiIsIDEpWzFdCgogICAgICAgICMgU3RhcnQgZXh0cmFjdGluZyB0aGUgYW5zd2VyczoKICAgICAgICBhbnN3ZXJzID0gW10KICAgICAgICBmb3IgaSBpbiByYW5nZSgxLCBxdWVzdGlvbnNfYW1vdW50ICsgMSk6CiAgICAgICAgICAgICMgSWYgaXQncyB0aGUgbGFzdCBhbnN3ZXIgdG8gbG9vayBmb3IsIHRha2UgdGhlIHJlc3Qgb2YgdGhlIHRleHQ6CiAgICAgICAgICAgIGlmIGkgPT0gcXVlc3Rpb25zX2Ftb3VudDoKICAgICAgICAgICAgICAgIGFuc3dlcl9pID0gdGV4dAogICAgICAgICAgICAjIFZlcmlmeSB0aGVyZSBpcyBhIHF1ZXN0aW9uIG51bWJlciBpbiB0aGUgdGV4dDoKICAgICAgICAgICAgZWxpZiBmIntpICsgMX0uIiBub3QgaW4gdGV4dDoKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICAgICAgICAgZiJBbnN3ZXIge2kgKyAxfS4gaXMgbWlzc2luZyBmcm9tIHRoZSBnZW5lcmF0ZWQgdGV4dDogJ3tnZW5lcmF0ZWRfdGV4dH0nIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAjIFRha2UgaSdzIGFuc3dlcjoKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIGFuc3dlcl9pLCB0ZXh0ID0gdGV4dC5zcGxpdChmIntpICsgMX0uIiwgMSkKICAgICAgICAgICAgIyBDb2xsZWN0IHRoZSBhbnN3ZXIgcmVtb3ZpbmcgcmVkdW5kYW50IHNwYWNlczoKICAgICAgICAgICAgYW5zd2Vycy5hcHBlbmQoYW5zd2VyX2kuc3RyaXAoKSkKCiAgICAgICAgcmV0dXJuIGFuc3dlcnMKCiAgICBkZWYgX2luZmVyX3F1ZXN0aW9ucygKICAgICAgICBzZWxmLAogICAgICAgIHF1ZXN0aW9uc19hbW91bnQ6IGludCwKICAgICAgICBiYXRjaGVkX2lucHV0OiBMaXN0W3N0cl0sCiAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZTogdHJhbnNmb3JtZXJzLlBpcGVsaW5lLAogICAgICAgIGdlbmVyYXRpb25fY29uZmlnOiB0cmFuc2Zvcm1lcnMuR2VuZXJhdGlvbkNvbmZpZywKICAgICkgLT4gTGlzdFtMaXN0W3N0cl1dOgoKICAgICAgICAjIEluZmVyIHRocm91Z2ggdGhlIGxsbToKICAgICAgICBiYXRjaGVkX291dHB1dCA9IGdlbmVyYXRpb25fcGlwZWxpbmUoCiAgICAgICAgICAgIGJhdGNoZWRfaW5wdXQsCiAgICAgICAgICAgIGdlbmVyYXRpb25fY29uZmlnPWdlbmVyYXRpb25fY29uZmlnLAogICAgICAgICAgICBlb3NfdG9rZW5faWQ9Z2VuZXJhdGlvbl9waXBlbGluZS50b2tlbml6ZXIuZW9zX3Rva2VuX2lkLAogICAgICAgICAgICByZXR1cm5fZnVsbF90ZXh0PUZhbHNlLAogICAgICAgICAgICBudW1fcmV0dXJuX3NlcXVlbmNlcz0xLAogICAgICAgICkKCiAgICAgICAgIyBQcm9jZXNzIHRoZSBvdXRwdXRzIHRvIGdldCB0aGUgYW5zd2VyczoKICAgICAgICBiYXRjaGVkX2Fuc3dlcnMgPSBbXQogICAgICAgIGZvciBvdXRwdXQgaW4gYmF0Y2hlZF9vdXRwdXQ6CiAgICAgICAgICAgICMgR2V0IHRoZSBnZW5lcmF0ZWQgYW5zd2VyczoKICAgICAgICAgICAgYW5zd2VycyA9IHNlbGYuX2dldF9hbnN3ZXJzKAogICAgICAgICAgICAgICAgZ2VuZXJhdGVkX3RleHQ9b3V0cHV0WzBdWyJnZW5lcmF0ZWRfdGV4dCJdLAogICAgICAgICAgICAgICAgcXVlc3Rpb25zX2Ftb3VudD1xdWVzdGlvbnNfYW1vdW50LAogICAgICAgICAgICApCiAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgcHJvY2Vzc2VkIGFuc3dlcnM6CiAgICAgICAgICAgIGJhdGNoZWRfYW5zd2Vycy5hcHBlbmQoYW5zd2VycykKICAgICAgICByZXR1cm4gYmF0Y2hlZF9hbnN3ZXJzCgogICAgZGVmIGFuc3dlcigKICAgICAgICBzZWxmLAogICAgICAgIHF1ZXN0aW9uc19hbW91bnQ6IGludCwKICAgICAgICBiYXRjaGVkX2lucHV0OiBMaXN0W3N0cl0sCiAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZTogdHJhbnNmb3JtZXJzLlBpcGVsaW5lLAogICAgICAgIGdlbmVyYXRpb25fY29uZmlnOiB0cmFuc2Zvcm1lcnMuR2VuZXJhdGlvbkNvbmZpZywKICAgICkgLT4gTGlzdFtMaXN0W3N0cl1dOgogICAgICAgICIiIgogICAgICAgIEFuc3dlciBxdWVzdGlvbnMgd2l0aCBhIGNvbnRleHQgdG8gdGhlIGdpdmVuIHRleHQgZmlsZXMgY29udGVudHMgYnkgYSBwcmV0cmFpbmVkIExMTSBtb2RlbCBpbiBnaXZlbiBwaXBlbGluZS4KICAgICAgICAiIiIKICAgICAgICByZXR1cm4gc2VsZi5faW5mZXJfcXVlc3Rpb25zKAogICAgICAgICAgICBxdWVzdGlvbnNfYW1vdW50PXF1ZXN0aW9uc19hbW91bnQsCiAgICAgICAgICAgIGJhdGNoZWRfaW5wdXQ9YmF0Y2hlZF9pbnB1dCwKICAgICAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZT1nZW5lcmF0aW9uX3BpcGVsaW5lLAogICAgICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZz1nZW5lcmF0aW9uX2NvbmZpZywKICAgICAgICApCgoKY2xhc3MgUG9sbFF1ZXN0aW9uSGFuZGxlcihRdWVzdGlvbkhhbmRsZXIpOgogICAgIiIiCiAgICBTdGF0aWMgY2xhc3MgdG8gaG9sZCBhbGwgdGhlIHBvc3NpYmxlIHBvbGwgcXVlc3Rpb24gY29uZmlndXJhdGlvbnMgb3B0aW9ucyBrZXlzCiAgICAiIiIKCiAgICBjbGFzcyBDb25maWdLZXlzOgogICAgICAgICIiIgogICAgICAgIEEgY2xhc3MgZm9yIGhhbmRsaW5nIHF1ZXN0aW9ucyBhbnN3ZXJpbmcgZm9yIHBvbGwgdHlwZSBxdWVzdGlvbnMuCiAgICAgICAgVGhlc2UgdHlwZSBvZiBxdWVzdGlvbiBhcmUgYW5zd2VyZWQgYnkgYXNraW5nIHRoZSBzYW1lIHF1ZXN0aW9uIG11bHRpcGxlIHRpbWVzCiAgICAgICAgYW5kIGNob29zaW5nIHRoZSBtb3N0IGNvbW1vbiBhbnN3ZXIgb3IgdGhlIGF2ZXJhZ2UgYW5zd2VyLgogICAgICAgICIiIgoKICAgICAgICAjOiBUaGUgbnVtYmVyIG9mIHRpbWVzIHRvIGFzayB0aGUgc2FtZSBxdWVzdGlvbi4KICAgICAgICBQT0xMX0NPVU5UID0gInBvbGxfY291bnQiCgogICAgICAgICM6IFRoZSBzdHJhdGVneSB0byB1c2UgZm9yIGNob29zaW5nIHRoZSBhbnN3ZXIgZnJvbSB0aGUgcG9sbC4KICAgICAgICBQT0xMX1NUUkFURUdZID0gInBvbGxfc3RyYXRlZ3kiCgogICAgY2xhc3MgU3RyYXRlZ3koZW51bS5FbnVtKToKICAgICAgICAjOiBUaGUgbW9zdCBjb21tb24gYW5zd2VyIHN0cmF0ZWd5LgogICAgICAgIE1PU1RfQ09NTU9OID0gIm1vc3RfY29tbW9uIgoKICAgICAgICAjOiBUaGUgYXZlcmFnZSBhbnN3ZXIgc3RyYXRlZ3kuCiAgICAgICAgQVZFUkFHRSA9ICJhdmVyYWdlIgoKICAgICAgICBAc3RhdGljbWV0aG9kCiAgICAgICAgZGVmIG1vc3RfY29tbW9uKGFuc3dlcnMpOgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgQ2FsY3VsYXRlIHRoZSBtb3N0IGNvbW1vbiBhbnN3ZXIgZm9yIGEgZ2l2ZW4gbGlzdCBvZiBhbnN3ZXJzLgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgY291bnQgPSBDb3VudGVyKGFuc3dlcnMpCiAgICAgICAgICAgIG1vc3RfY29tbW9uID0gY291bnQubW9zdF9jb21tb24oMSkKICAgICAgICAgICAgcmV0dXJuIG1vc3RfY29tbW9uWzBdWzBdCgogICAgICAgIEBzdGF0aWNtZXRob2QKICAgICAgICBkZWYgYXZlcmFnZShhbnN3ZXJzKToKICAgICAgICAgICAgIiIiCiAgICAgICAgICAgIENhbGN1bGF0ZSB0aGUgYXZlcmFnZSBhbnN3ZXIgZm9yIGEgZ2l2ZW4gbGlzdCBvZiBhbnN3ZXJzLgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShhbnN3ZXJzWzBdLCBzdHIpOgogICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAiQ2Fubm90IHBlcmZvcm0gcG9sbCB3aXRoIGF2ZXJhZ2UgYW5zd2VyIHN0cmF0ZWd5IG9mIG5vbiBudW1lcmljIHZhbHVlcywiCiAgICAgICAgICAgICAgICAgICAgIiBwbGVhc2UgY2hhbmdlIHRoZSBxdWVzdGlvbiB0byBnaXZlIG51bWVyaWMgZGF0YSwgb3IgY2hvb3NlICdtb3N0X2NvbW1vbicgYXMgc3RyYXRlZ3kuIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgbnVtZXJpY192YWx1ZXMgPSBhbnN3ZXJzCiAgICAgICAgICAgIGF2ZyA9IHN1bShudW1lcmljX3ZhbHVlcykgLyBsZW4obnVtZXJpY192YWx1ZXMpCgogICAgICAgICAgICAjIFJvdW5kIHRvIHRoZSBjbG9zZXN0IGludGVnZXIgYW5kIHJldHVybiBjb3JyZXNwb25kaW5nIHZhbHVlCiAgICAgICAgICAgIHJldHVybiByb3VuZChhdmcpCgogICAgICAgIGRlZiBkbyhzZWxmLCBhbnN3ZXJzKToKICAgICAgICAgICAgIiIiCiAgICAgICAgICAgIFBlcmZvcm0gdGhlIHN0cmF0ZWd5LgogICAgICAgICAgICAiIiIKICAgICAgICAgICAgcmV0dXJuIGdldGF0dHIoc2VsZiwgc2VsZi52YWx1ZSkoYW5zd2VycykKCiAgICBkZWYgX19pbml0X18oCiAgICAgICAgc2VsZiwgcG9sbF9jb3VudDogaW50ID0gNSwgcG9sbF9zdHJhdGVneTogc3RyID0gIm1vc3RfY29tbW9uIik6CiAgICAgICAgc3VwZXIoKS5fX2luaXRfXygpCiAgICAgICAgc2VsZi5wb2xsX2NvdW50ID0gcG9sbF9jb3VudAogICAgICAgIHNlbGYucG9sbF9zdHJhdGVneSA9IHNlbGYuU3RyYXRlZ3kocG9sbF9zdHJhdGVneSkKCiAgICBkZWYgYW5zd2VyKAogICAgICAgIHNlbGYsCiAgICAgICAgcXVlc3Rpb25zX2Ftb3VudDogaW50LAogICAgICAgIGJhdGNoZWRfaW5wdXQ6IExpc3Rbc3RyXSwKICAgICAgICBnZW5lcmF0aW9uX3BpcGVsaW5lOiB0cmFuc2Zvcm1lcnMuUGlwZWxpbmUsCiAgICAgICAgZ2VuZXJhdGlvbl9jb25maWc6IHRyYW5zZm9ybWVycy5HZW5lcmF0aW9uQ29uZmlnLAogICAgKSAtPiBMaXN0W0xpc3Rbc3RyXV06CiAgICAgICAgIiIiCiAgICAgICAgQW5zd2VyIHF1ZXN0aW9ucyB3aXRoIGEgY29udGV4dCB0byB0aGUgZ2l2ZW4gdGV4dCBmaWxlcyBjb250ZW50cyBieSBhIHByZXRyYWluZWQgTExNIG1vZGVsIGluIGdpdmVuIHBpcGVsaW5lLgogICAgICAgICIiIgogICAgICAgIHJldHVybiBzZWxmLl9hbnN3ZXJfcG9sbF9xdWVzdGlvbnMoCiAgICAgICAgICAgIHF1ZXN0aW9uc19hbW91bnQ9cXVlc3Rpb25zX2Ftb3VudCwKICAgICAgICAgICAgYmF0Y2hlZF9pbnB1dD1iYXRjaGVkX2lucHV0LAogICAgICAgICAgICBnZW5lcmF0aW9uX3BpcGVsaW5lPWdlbmVyYXRpb25fcGlwZWxpbmUsCiAgICAgICAgICAgIGdlbmVyYXRpb25fY29uZmlnPWdlbmVyYXRpb25fY29uZmlnLAogICAgICAgICkKCiAgICBkZWYgX2Fuc3dlcl9wb2xsX3F1ZXN0aW9ucygKICAgICAgICBzZWxmLAogICAgICAgIHF1ZXN0aW9uc19hbW91bnQ6IGludCwKICAgICAgICBiYXRjaGVkX2lucHV0OiBMaXN0W3N0cl0sCiAgICAgICAgZ2VuZXJhdGlvbl9waXBlbGluZTogdHJhbnNmb3JtZXJzLlBpcGVsaW5lLAogICAgICAgIGdlbmVyYXRpb25fY29uZmlnOiB0cmFuc2Zvcm1lcnMuR2VuZXJhdGlvbkNvbmZpZywKICAgICkgLT4gTGlzdFtMaXN0W3N0cl1dOgogICAgICAgIHZvdGVzID0gW10KCiAgICAgICAgIyBSdW4gdGhlIHBvbGwgZm9yIGVhY2ggcXVlc3Rpb24KICAgICAgICBmb3IgXyBpbiByYW5nZShzZWxmLnBvbGxfY291bnQpOgogICAgICAgICAgICBiYXRjaGVkX2Fuc3dlcnMgPSBzZWxmLl9pbmZlcl9xdWVzdGlvbnMoCiAgICAgICAgICAgICAgICBxdWVzdGlvbnNfYW1vdW50PXF1ZXN0aW9uc19hbW91bnQsCiAgICAgICAgICAgICAgICBiYXRjaGVkX2lucHV0PWJhdGNoZWRfaW5wdXQsCiAgICAgICAgICAgICAgICBnZW5lcmF0aW9uX3BpcGVsaW5lPWdlbmVyYXRpb25fcGlwZWxpbmUsCiAgICAgICAgICAgICAgICBnZW5lcmF0aW9uX2NvbmZpZz1nZW5lcmF0aW9uX2NvbmZpZywKICAgICAgICAgICAgKQogICAgICAgICAgICB2b3Rlcy5hcHBlbmQoYmF0Y2hlZF9hbnN3ZXJzKQogICAgICAgIGFuc3dlcnMgPSBbXQoKICAgICAgICAjIENvbGxlY3QgdGhlIGFuc3dlcnMgYWNjb3JkaW5nIHRvIHRoZSBwb2xsIHN0cmF0ZWd5CiAgICAgICAgIyBBdmVyYWdlIHN0cmF0ZWd5IHdvcmtzIGZvciBudW1lcmljIHZhbHVlcyBvbmx5CiAgICAgICAgZm9yIGJhdGNoIGluIHJhbmdlKGxlbih2b3Rlc1swXSkpOgogICAgICAgICAgICBiYXRjaGVkX2Fuc3dlcnMgPSBbXQogICAgICAgICAgICBmb3IgcXVlc3Rpb24gaW4gcmFuZ2UocXVlc3Rpb25zX2Ftb3VudCk6CiAgICAgICAgICAgICAgICAjIENyZWF0ZSBhIGxpc3Qgb2YgYWxsIGFuc3dlcnMgdG8gcmVsZXZhbnQgcXVlc3Rpb24KICAgICAgICAgICAgICAgIGFuc3dlciA9IFsKICAgICAgICAgICAgICAgICAgICB2b3Rlc1t2b3Rlcl1bYmF0Y2hdW3F1ZXN0aW9uXSBmb3Igdm90ZXIgaW4gcmFuZ2Uoc2VsZi5wb2xsX2NvdW50KQogICAgICAgICAgICAgICAgXQogICAgICAgICAgICAgICAgYW5zd2VyID0gc2VsZi5wb2xsX3N0cmF0ZWd5LmRvKGFuc3dlcikKICAgICAgICAgICAgICAgIGJhdGNoZWRfYW5zd2Vycy5hcHBlbmQoYW5zd2VyKQogICAgICAgICAgICBhbnN3ZXJzLmFwcGVuZChiYXRjaGVkX2Fuc3dlcnMpCiAgICAgICAgcmV0dXJuIGFuc3dlcnMKCgojIEhvbGRzIG5hbWVzIG9mIFF1ZXN0aW9uSGFuZGxlcwpjbGFzcyBRdWVzdGlvblR5cGVzOgogICAgREVGQVVMVCA9ICJkZWZhdWx0IgogICAgUE9MTCA9ICJwb2xsIgoKCiMgTWFwcyBxdWVzdGlvbiB0eXBlcyB0byB0aGVpciBoYW5kbGVycwpRVUVTVElPTl9NQVBQSU5HID0gewogICAgUXVlc3Rpb25UeXBlcy5ERUZBVUxUOiBRdWVzdGlvbkhhbmRsZXIsCiAgICBRdWVzdGlvblR5cGVzLlBPTEw6IFBvbGxRdWVzdGlvbkhhbmRsZXIsCn0K + base_image: mlrun/mlrun + filename: question_answering.py entry_points: open_mpi_handler: - name: open_mpi_handler - has_varargs: false - doc: '' - lineno: 58 parameters: - name: worker_inputs - type: List[str] + type: list[str] - name: root_worker_inputs - type: Dict[str, Any] + type: dict[str, Any] default: null + name: open_mpi_handler + doc: '' has_kwargs: false - decorator: - name: decorator has_varargs: false - doc: '' - lineno: 66 + lineno: 58 + decorator: parameters: - name: handler + name: decorator + doc: '' has_kwargs: false + has_varargs: false + lineno: 66 wrapper: name: wrapper - has_varargs: false doc: '' - lineno: 71 has_kwargs: true + has_varargs: false + lineno: 71 answer_questions: outputs: - doc: 'A tuple of:' - type: Tuple[pd.DataFrame, dict] - name: answer_questions - has_varargs: false - doc: 'Answer questions with a context to the given text files contents by a - pretrained LLM model. Each text file will have - - the following prompt built: - - - start of `text_wrapper` - - - - end of `text_wrapper` - - - start of `questions_wrapper` - - 1. - - 2. - - ... - - n. - - end of `questions_wrapper`' - lineno: 130 + type: tuple[pd.DataFrame, dict] parameters: - name: data_path - type: Union[str, List[str]] doc: A path to a directory of text files or a path to a text file to ask questions about. - name: model_name @@ -85,13 +58,11 @@ spec: doc: The pre-trained model name from the huggingface hub to use for asking questions. - name: questions - type: Union[List[str], List[List[str]]] doc: The questions to ask. A list of lists of questions to ask per text file, and devided by question groups, the groups can be dtermained by size (in order to avoid large inputs to the llm) or by questioning method (regular or poll like questioning). - name: device_map - type: Union[str, dict] doc: A map to use for loading the model on multiple devices. default: null - name: model_kwargs @@ -114,22 +85,18 @@ spec: `transformers.AutoTokenizer.from_pretrained` function. default: null - name: text_wrapper - type: Union[str, List[str]] doc: A wrapper for the file's text. Will be added at the start of the prompt. Must have a placeholder ('{}') for the text of the file. default: '' - name: questions_wrapper - type: Union[str, List[str]] doc: A wrapper for the questions received. Will be added after the text wrapper in the prompt template. Must have a placeholder ('{}') for the questions. default: '' - name: generation_config - type: Union[Dict, List[Dict]] doc: HuggingFace's `GenerationConfig` keyword arguments to pass to the `generate` method. default: null - name: questions_config - type: Union[Dict, List[Dict]] doc: A dictionary or list of dictionaries containing specific ways to answer questions (using a poll for example), each dictionary in the list is for corresponding question group and determines the question asking method for @@ -140,58 +107,85 @@ spec: doc: Batch size for inference. default: 1 - name: questions_columns - type: List[str] + type: list[str] doc: Columns to use for the dataframe returned. default: null - name: verbose type: bool doc: 'Whether to present logs of a progress bar and errors. Default: True.' default: false + name: answer_questions + doc: 'Answer questions with a context to the given text files contents by a + pretrained LLM model. Each text file will have + + the following prompt built: + + + start of `text_wrapper` + + + + end of `text_wrapper` + + + start of `questions_wrapper` + + 1. + + 2. + + ... + + n. + + end of `questions_wrapper`' has_kwargs: false + has_varargs: false + lineno: 130 answer: outputs: - - type: List[List[str]] - name: answer - has_varargs: false - doc: Answer questions with a context to the given text files contents by a pretrained - LLM model in given pipeline. - lineno: 674 + - type: list[list[str]] parameters: - name: self - name: questions_amount type: int - name: batched_input - type: List[str] + type: list[str] - name: generation_pipeline type: Pipeline - name: generation_config type: GenerationConfig + name: answer + doc: Answer questions with a context to the given text files contents by a pretrained + LLM model in given pipeline. has_kwargs: false - most_common: - name: most_common has_varargs: false - doc: Calculate the most common answer for a given list of answers. - lineno: 637 + lineno: 665 + most_common: parameters: - name: answers + name: most_common + doc: Calculate the most common answer for a given list of answers. has_kwargs: false - average: - name: average has_varargs: false - doc: Calculate the average answer for a given list of answers. - lineno: 646 + lineno: 629 + average: parameters: - name: answers + name: average + doc: Calculate the average answer for a given list of answers. has_kwargs: false - do: - name: do has_varargs: false - doc: Perform the strategy. - lineno: 662 + lineno: 638 + do: parameters: - name: self - name: answers + name: do + doc: Perform the strategy. has_kwargs: false - image: '' + has_varargs: false + lineno: 654 + command: '' description: GenAI approach of question answering on a given data - disable_auto_mount: false + default_handler: answer_questions diff --git a/functions/src/question_answering/question_answering.py b/functions/src/question_answering/question_answering.py index 2e4e96d03..0ad4bb015 100644 --- a/functions/src/question_answering/question_answering.py +++ b/functions/src/question_answering/question_answering.py @@ -17,7 +17,7 @@ import pathlib from collections import Counter from functools import reduce, wraps -from typing import Any, Dict, List, Tuple, Union +from typing import Any import pandas as pd import transformers @@ -27,7 +27,7 @@ _LOGGER = logging.getLogger() -def _check_mlrun_and_open_mpi() -> Tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intracomm"]: +def _check_mlrun_and_open_mpi() -> tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intracomm"]: global _LOGGER is_mpi = False @@ -56,7 +56,7 @@ def _check_mlrun_and_open_mpi() -> Tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intrac def open_mpi_handler( - worker_inputs: List[str], root_worker_inputs: Dict[str, Any] = None + worker_inputs: list[str], root_worker_inputs: dict[str, Any] = None ): global _LOGGER @@ -128,22 +128,22 @@ def wrapper(**kwargs): @open_mpi_handler(worker_inputs=["data_path"], root_worker_inputs={"verbose": True}) def answer_questions( - data_path: Union[str, List[str]], + data_path: str | list[str], model_name: str, - questions: Union[List[str], List[List[str]]], - device_map: Union[str, dict] = None, + questions: list[str] | list[list[str]], + device_map: str | dict = None, model_kwargs: dict = None, auto_gptq_exllama_max_input_length: int = None, tokenizer_name: str = None, tokenizer_kwargs: dict = None, - text_wrapper: Union[str, List[str]] = "", - questions_wrapper: Union[str, List[str]] = "", - generation_config: Union[Dict, List[Dict]] = None, - questions_config: Union[Dict, List[Dict]] = None, + text_wrapper: str | list[str] = "", + questions_wrapper: str | list[str] = "", + generation_config: dict | list[dict] = None, + questions_config: dict | list[dict] = None, batch_size: int = 1, - questions_columns: List[str] = None, + questions_columns: list[str] = None, verbose: bool = False, -) -> Tuple[pd.DataFrame, dict]: +) -> tuple[pd.DataFrame, dict]: """ Answer questions with a context to the given text files contents by a pretrained LLM model. Each text file will have the following prompt built: @@ -396,11 +396,9 @@ def answer_questions( def _get_text_files( data_path: pathlib.Path, -) -> List[pathlib.Path]: - +) -> list[pathlib.Path]: # Check if the path is of a directory or a file: if data_path.is_dir(): - # Get all files inside the directory: text_files = list(data_path.glob("*.*")) elif data_path.is_file(): @@ -417,20 +415,17 @@ def _get_text_files( def _get_prompt_template( text_wrapper: str, questions_wrapper: str, - questions: List[str], + questions: list[str], ) -> str: - # Validate and build the text wrapper: - text_wrapper = text_wrapper or ( - "Given the following text:\n" "-----\n" "{}\n" "-----" - ) + text_wrapper = text_wrapper or ("Given the following text:\n-----\n{}\n-----") if text_wrapper.count("{}") != 1: raise ValueError( "The `text_wrapper` must include one placeholder '{}' for the text of the file to be asked about." ) # Validate and build the question wrapper: - questions_wrapper = questions_wrapper or "Answer the questions:\n" "{}" + questions_wrapper = questions_wrapper or "Answer the questions:\n{}" if questions_wrapper.count("{}") != 1: raise ValueError( "The `questions_wrapper` must include one placeholder '{}' for the list of questions." @@ -449,7 +444,7 @@ def _get_prompt_template( def _get_generation_pipeline( model_name: str, - device_map: Union[str, dict], + device_map: str | dict, tokenizer_name: str, model_kwargs: dict, tokenizer_kwargs: dict, @@ -487,20 +482,19 @@ def _get_generation_pipeline( def _read_file_batch( - file_batch: List[pathlib.Path], + file_batch: list[pathlib.Path], prompt_template: str, -) -> List[str]: +) -> list[str]: batch = [] # Go over all files and read in usable format for file in file_batch: - with open(file, "r", encoding="utf-8") as fp: + with open(file, encoding="utf-8") as fp: batch.append(prompt_template.format(fp.read())) return batch def _to_group_list(argument_value: list, argument_name: str, length: int): - # Check if is list, turn to list if not argument_value = ( argument_value if isinstance(argument_value, list) else [argument_value] @@ -532,8 +526,7 @@ def __init__(self): pass @staticmethod - def _get_answers(generated_text: str, questions_amount: int) -> List[str]: - + def _get_answers(generated_text: str, questions_amount: int) -> list[str]: # Clear answer start (part before numbers): # TODO find better way to verify, for list of questions this is redundant for example if "1." not in generated_text: @@ -564,11 +557,10 @@ def _get_answers(generated_text: str, questions_amount: int) -> List[str]: def _infer_questions( self, questions_amount: int, - batched_input: List[str], + batched_input: list[str], generation_pipeline: transformers.Pipeline, generation_config: transformers.GenerationConfig, - ) -> List[List[str]]: - + ) -> list[list[str]]: # Infer through the llm: batched_output = generation_pipeline( batched_input, @@ -593,10 +585,10 @@ def _infer_questions( def answer( self, questions_amount: int, - batched_input: List[str], + batched_input: list[str], generation_pipeline: transformers.Pipeline, generation_config: transformers.GenerationConfig, - ) -> List[List[str]]: + ) -> list[list[str]]: """ Answer questions with a context to the given text files contents by a pretrained LLM model in given pipeline. """ @@ -665,8 +657,7 @@ def do(self, answers): """ return getattr(self, self.value)(answers) - def __init__( - self, poll_count: int = 5, poll_strategy: str = "most_common"): + def __init__(self, poll_count: int = 5, poll_strategy: str = "most_common"): super().__init__() self.poll_count = poll_count self.poll_strategy = self.Strategy(poll_strategy) @@ -674,10 +665,10 @@ def __init__( def answer( self, questions_amount: int, - batched_input: List[str], + batched_input: list[str], generation_pipeline: transformers.Pipeline, generation_config: transformers.GenerationConfig, - ) -> List[List[str]]: + ) -> list[list[str]]: """ Answer questions with a context to the given text files contents by a pretrained LLM model in given pipeline. """ @@ -691,10 +682,10 @@ def answer( def _answer_poll_questions( self, questions_amount: int, - batched_input: List[str], + batched_input: list[str], generation_pipeline: transformers.Pipeline, generation_config: transformers.GenerationConfig, - ) -> List[List[str]]: + ) -> list[list[str]]: votes = [] # Run the poll for each question diff --git a/functions/src/question_answering/test_question_answering.py b/functions/src/question_answering/test_question_answering.py index f35b4364e..41469ebe3 100644 --- a/functions/src/question_answering/test_question_answering.py +++ b/functions/src/question_answering/test_question_answering.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import tempfile + import mlrun import transformers -import tempfile APPLE_COLOR = "red" @@ -36,18 +37,15 @@ def test_question_answering(monkeypatch): input_path = "./data" artifact_path = tempfile.mkdtemp() project = mlrun.new_project("qa", context="./") - fn = project.set_function("question_answering.py", "answer_questions", kind="job", image="mlrun/mlrun") + fn = project.set_function( + "question_answering.py", "answer_questions", kind="job", image="mlrun/mlrun" + ) qa_run = fn.run( handler="answer_questions", params={ "model_name": "distilgpt2", "data_path": input_path, - "text_wrapper": ( - "Given the following sentence:\n" - "-----\n" - "{}\n" - "-----" - ), + "text_wrapper": ("Given the following sentence:\n-----\n{}\n-----"), "questions": [ "What is the color of the apple?", ], @@ -67,7 +65,7 @@ def test_question_answering(monkeypatch): "question_answering_errors: result", ], local=True, - artifact_path=artifact_path + artifact_path=artifact_path, ) qa_df = mlrun.get_dataitem( qa_run.status.artifacts[0]["spec"]["target_path"] diff --git a/functions/src/send_email/function.yaml b/functions/src/send_email/function.yaml index 1722fb586..00a0f2ad8 100644 --- a/functions/src/send_email/function.yaml +++ b/functions/src/send_email/function.yaml @@ -1,44 +1,35 @@ -kind: job metadata: - name: send-email tag: '' - hash: 5c4528084ea98992b77f65e29359bbcb4a0df8ab - project: '' - labels: - author: Iguazio + name: send-email categories: - utils +verbose: false +kind: job spec: - command: '' - args: [] image: mlrun/mlrun + disable_auto_mount: false build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKZnJvbSBtbHJ1bi5leGVjdXRpb24gaW1wb3J0IE1MQ2xpZW50Q3R4CmZyb20gdHlwaW5nIGltcG9ydCBMaXN0CgppbXBvcnQgc210cGxpYgpmcm9tIGVtYWlsLm1lc3NhZ2UgaW1wb3J0IEVtYWlsTWVzc2FnZQppbXBvcnQgb3MKCmltcG9ydCBtaW1ldHlwZXMKCgpkZWYgc2VuZF9lbWFpbCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgc2VuZGVyOiBzdHIsCiAgICB0bzogc3RyLAogICAgc3ViamVjdDogc3RyLAogICAgY29udGVudDogc3RyID0gIiIsCiAgICBzZXJ2ZXJfYWRkcjogc3RyID0gTm9uZSwKICAgIGF0dGFjaG1lbnRzOiBMaXN0W3N0cl0gPSBbXSwKKSAtPiBOb25lOgogICAgIiIiU2VuZCBhbiBlbWFpbC4KICAgIDpwYXJhbSBzZW5kZXI6IFNlbmRlciBlbWFpbCBhZGRyZXNzCiAgICA6cGFyYW0gY29udGV4dDogVGhlIGZ1bmN0aW9uIGNvbnRleHQKICAgIDpwYXJhbSB0bzogRW1haWwgYWRkcmVzcyBvZiBtYWlsIHJlY2lwaWVudAogICAgOnBhcmFtIHN1YmplY3Q6IEVtYWlsIHN1YmplY3QKICAgIDpwYXJhbSBjb250ZW50OiBPcHRpb25hbCBtYWlsIHRleHQKICAgIDpwYXJhbSBzZXJ2ZXJfYWRkcjogQWRkcmVzcyBvZiBTTVRQIHNlcnZlciB0byB1c2UuIFVzZSBmb3JtYXQgPGFkZHI+Ojxwb3J0PgogICAgOnBhcmFtIGF0dGFjaG1lbnRzOiBMaXN0IG9mIGF0dGFjaG1lbnRzIHRvIGFkZC4KICAgICIiIgoKICAgIGVtYWlsX3VzZXIgPSBjb250ZXh0LmdldF9zZWNyZXQoIlNNVFBfVVNFUiIpCiAgICBlbWFpbF9wYXNzID0gY29udGV4dC5nZXRfc2VjcmV0KCJTTVRQX1BBU1NXT1JEIikKICAgIGlmIGVtYWlsX3VzZXIgaXMgTm9uZSBvciBlbWFpbF9wYXNzIGlzIE5vbmU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoIk1pc3Npbmcgc2VuZGVyIGVtYWlsIG9yIHBhc3N3b3JkIC0gY2Fubm90IHNlbmQgZW1haWwuIikKICAgICAgICByZXR1cm4KCiAgICBpZiBzZXJ2ZXJfYWRkciBpcyBOb25lOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKCJTZXJ2ZXIgbm90IHNwZWNpZmllZCAtIGNhbm5vdCBzZW5kIGVtYWlsLiIpCiAgICAgICAgcmV0dXJuCgogICAgbXNnID0gRW1haWxNZXNzYWdlKCkKICAgIG1zZ1siRnJvbSJdID0gc2VuZGVyCiAgICBtc2dbIlN1YmplY3QiXSA9IHN1YmplY3QKICAgIG1zZ1siVG8iXSA9IHRvCiAgICBtc2cuc2V0X2NvbnRlbnQoY29udGVudCkKCiAgICBmb3IgZmlsZW5hbWUgaW4gYXR0YWNobWVudHM6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIkxvb2tpbmcgYXQgYXR0YWNobWVudDoge2ZpbGVuYW1lfSIpCiAgICAgICAgaWYgbm90IG9zLnBhdGguaXNmaWxlKGZpbGVuYW1lKToKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIud2FybmluZyhmIkZpbGVuYW1lIGRvZXMgbm90IGV4aXN0IHtmaWxlbmFtZX0iKQogICAgICAgICAgICBjb250aW51ZQogICAgICAgIGN0eXBlLCBlbmNvZGluZyA9IG1pbWV0eXBlcy5ndWVzc190eXBlKGZpbGVuYW1lKQogICAgICAgIGlmIGN0eXBlIGlzIE5vbmUgb3IgZW5jb2RpbmcgaXMgbm90IE5vbmU6CiAgICAgICAgICAgIGN0eXBlID0gImFwcGxpY2F0aW9uL29jdGV0LXN0cmVhbSIKICAgICAgICBtYWludHlwZSwgc3VidHlwZSA9IGN0eXBlLnNwbGl0KCIvIiwgMSkKICAgICAgICB3aXRoIG9wZW4oZmlsZW5hbWUsICJyYiIpIGFzIGZwOgogICAgICAgICAgICBtc2cuYWRkX2F0dGFjaG1lbnQoCiAgICAgICAgICAgICAgICBmcC5yZWFkKCksCiAgICAgICAgICAgICAgICBtYWludHlwZT1tYWludHlwZSwKICAgICAgICAgICAgICAgIHN1YnR5cGU9c3VidHlwZSwKICAgICAgICAgICAgICAgIGZpbGVuYW1lPW9zLnBhdGguYmFzZW5hbWUoZmlsZW5hbWUpLAogICAgICAgICAgICApCiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICBmIkFkZGVkIGF0dGFjaG1lbnQ6IEZpbGVuYW1lOiB7ZmlsZW5hbWV9LCBvZiBtaW1ldHlwZToge21haW50eXBlfSwge3N1YnR5cGV9IgogICAgICAgICAgICApCgogICAgdHJ5OgogICAgICAgIHMgPSBzbXRwbGliLlNNVFAoaG9zdD1zZXJ2ZXJfYWRkcikKICAgICAgICBzLnN0YXJ0dGxzKCkKICAgICAgICBzLmxvZ2luKGVtYWlsX3VzZXIsIGVtYWlsX3Bhc3MpCiAgICAgICAgcy5zZW5kX21lc3NhZ2UobXNnKQogICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkVtYWlsIHNlbnQgc3VjY2Vzc2Z1bGx5LiIpCiAgICBleGNlcHQgc210cGxpYi5TTVRQRXhjZXB0aW9uIGFzIGV4cDoKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcihmIlNNVFAgZXhjZXB0aW9uIGNhdWdodCBpbiBTTVRQIGNvZGU6IHtleHB9IikKICAgIGV4Y2VwdCBDb25uZWN0aW9uRXJyb3IgYXMgY2U6CiAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoZiJDb25uZWN0aW9uIGVycm9yIGNhdWdodCBpbiBTTVRQIGNvZGU6IHtjZX0iKQo= - commands: [] - code_origin: "" - origin_filename: "" - requirements: [] + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IG1pbWV0eXBlcwppbXBvcnQgb3MKaW1wb3J0IHNtdHBsaWIKZnJvbSBlbWFpbC5tZXNzYWdlIGltcG9ydCBFbWFpbE1lc3NhZ2UKCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eAoKCmRlZiBzZW5kX2VtYWlsKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBzZW5kZXI6IHN0ciwKICAgIHRvOiBzdHIsCiAgICBzdWJqZWN0OiBzdHIsCiAgICBjb250ZW50OiBzdHIgPSAiIiwKICAgIHNlcnZlcl9hZGRyOiBzdHIgPSBOb25lLAogICAgYXR0YWNobWVudHM6IGxpc3Rbc3RyXSA9IFtdLAopIC0+IE5vbmU6CiAgICAiIiJTZW5kIGFuIGVtYWlsLgogICAgOnBhcmFtIHNlbmRlcjogU2VuZGVyIGVtYWlsIGFkZHJlc3MKICAgIDpwYXJhbSBjb250ZXh0OiBUaGUgZnVuY3Rpb24gY29udGV4dAogICAgOnBhcmFtIHRvOiBFbWFpbCBhZGRyZXNzIG9mIG1haWwgcmVjaXBpZW50CiAgICA6cGFyYW0gc3ViamVjdDogRW1haWwgc3ViamVjdAogICAgOnBhcmFtIGNvbnRlbnQ6IE9wdGlvbmFsIG1haWwgdGV4dAogICAgOnBhcmFtIHNlcnZlcl9hZGRyOiBBZGRyZXNzIG9mIFNNVFAgc2VydmVyIHRvIHVzZS4gVXNlIGZvcm1hdCA8YWRkcj46PHBvcnQ+CiAgICA6cGFyYW0gYXR0YWNobWVudHM6IExpc3Qgb2YgYXR0YWNobWVudHMgdG8gYWRkLgogICAgIiIiCgogICAgZW1haWxfdXNlciA9IGNvbnRleHQuZ2V0X3NlY3JldCgiU01UUF9VU0VSIikKICAgIGVtYWlsX3Bhc3MgPSBjb250ZXh0LmdldF9zZWNyZXQoIlNNVFBfUEFTU1dPUkQiKQogICAgaWYgZW1haWxfdXNlciBpcyBOb25lIG9yIGVtYWlsX3Bhc3MgaXMgTm9uZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigiTWlzc2luZyBzZW5kZXIgZW1haWwgb3IgcGFzc3dvcmQgLSBjYW5ub3Qgc2VuZCBlbWFpbC4iKQogICAgICAgIHJldHVybgoKICAgIGlmIHNlcnZlcl9hZGRyIGlzIE5vbmU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoIlNlcnZlciBub3Qgc3BlY2lmaWVkIC0gY2Fubm90IHNlbmQgZW1haWwuIikKICAgICAgICByZXR1cm4KCiAgICBtc2cgPSBFbWFpbE1lc3NhZ2UoKQogICAgbXNnWyJGcm9tIl0gPSBzZW5kZXIKICAgIG1zZ1siU3ViamVjdCJdID0gc3ViamVjdAogICAgbXNnWyJUbyJdID0gdG8KICAgIG1zZy5zZXRfY29udGVudChjb250ZW50KQoKICAgIGZvciBmaWxlbmFtZSBpbiBhdHRhY2htZW50czoKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiTG9va2luZyBhdCBhdHRhY2htZW50OiB7ZmlsZW5hbWV9IikKICAgICAgICBpZiBub3Qgb3MucGF0aC5pc2ZpbGUoZmlsZW5hbWUpOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci53YXJuaW5nKGYiRmlsZW5hbWUgZG9lcyBub3QgZXhpc3Qge2ZpbGVuYW1lfSIpCiAgICAgICAgICAgIGNvbnRpbnVlCiAgICAgICAgY3R5cGUsIGVuY29kaW5nID0gbWltZXR5cGVzLmd1ZXNzX3R5cGUoZmlsZW5hbWUpCiAgICAgICAgaWYgY3R5cGUgaXMgTm9uZSBvciBlbmNvZGluZyBpcyBub3QgTm9uZToKICAgICAgICAgICAgY3R5cGUgPSAiYXBwbGljYXRpb24vb2N0ZXQtc3RyZWFtIgogICAgICAgIG1haW50eXBlLCBzdWJ0eXBlID0gY3R5cGUuc3BsaXQoIi8iLCAxKQogICAgICAgIHdpdGggb3BlbihmaWxlbmFtZSwgInJiIikgYXMgZnA6CiAgICAgICAgICAgIG1zZy5hZGRfYXR0YWNobWVudCgKICAgICAgICAgICAgICAgIGZwLnJlYWQoKSwKICAgICAgICAgICAgICAgIG1haW50eXBlPW1haW50eXBlLAogICAgICAgICAgICAgICAgc3VidHlwZT1zdWJ0eXBlLAogICAgICAgICAgICAgICAgZmlsZW5hbWU9b3MucGF0aC5iYXNlbmFtZShmaWxlbmFtZSksCiAgICAgICAgICAgICkKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgIGYiQWRkZWQgYXR0YWNobWVudDogRmlsZW5hbWU6IHtmaWxlbmFtZX0sIG9mIG1pbWV0eXBlOiB7bWFpbnR5cGV9LCB7c3VidHlwZX0iCiAgICAgICAgICAgICkKCiAgICB0cnk6CiAgICAgICAgcyA9IHNtdHBsaWIuU01UUChob3N0PXNlcnZlcl9hZGRyKQogICAgICAgIHMuc3RhcnR0bHMoKQogICAgICAgIHMubG9naW4oZW1haWxfdXNlciwgZW1haWxfcGFzcykKICAgICAgICBzLnNlbmRfbWVzc2FnZShtc2cpCiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygiRW1haWwgc2VudCBzdWNjZXNzZnVsbHkuIikKICAgIGV4Y2VwdCBzbXRwbGliLlNNVFBFeGNlcHRpb24gYXMgZXhwOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKGYiU01UUCBleGNlcHRpb24gY2F1Z2h0IGluIFNNVFAgY29kZToge2V4cH0iKQogICAgZXhjZXB0IENvbm5lY3Rpb25FcnJvciBhcyBjZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcihmIkNvbm5lY3Rpb24gZXJyb3IgY2F1Z2h0IGluIFNNVFAgY29kZToge2NlfSIpCg== + code_origin: '' + filename: send_email.py entry_points: send_email: - name: send_email - doc: Send an email. + outputs: + - type: None parameters: - name: context type: MLClientCtx doc: The function context - default: '' - name: sender type: str doc: Sender email address - default: '' - name: to type: str doc: Email address of mail recipient - default: '' - name: subject type: str doc: Email subject - default: '' - name: content type: str doc: Optional mail text @@ -48,20 +39,14 @@ spec: doc: Address of SMTP server to use. Use format : default: null - name: attachments - type: List[str] + type: list[str] doc: List of attachments to add. default: [] - outputs: - - default: '' - lineno: 27 + name: send_email + doc: Send an email. + has_kwargs: false + has_varargs: false + lineno: 25 + command: '' description: Send Email messages through SMTP server default_handler: send_email - disable_auto_mount: false - clone_target_dir: '' - env: [] - priority_class_name: '' - preemption_mode: prevent - affinity: null - tolerations: null - security_context: {} -verbose: false diff --git a/functions/src/send_email/send_email.py b/functions/src/send_email/send_email.py index 0dd9f7d0f..f6ab688ae 100644 --- a/functions/src/send_email/send_email.py +++ b/functions/src/send_email/send_email.py @@ -14,14 +14,12 @@ # # Generated by nuclio.export.NuclioExporter -from mlrun.execution import MLClientCtx -from typing import List - +import mimetypes +import os import smtplib from email.message import EmailMessage -import os -import mimetypes +from mlrun.execution import MLClientCtx def send_email( @@ -31,7 +29,7 @@ def send_email( subject: str, content: str = "", server_addr: str = None, - attachments: List[str] = [], + attachments: list[str] = [], ) -> None: """Send an email. :param sender: Sender email address diff --git a/functions/src/silero_vad/function.yaml b/functions/src/silero_vad/function.yaml index fd637f1c0..1d7b53d34 100644 --- a/functions/src/silero_vad/function.yaml +++ b/functions/src/silero_vad/function.yaml @@ -1,76 +1,74 @@ metadata: tag: '' + name: silero-vad categories: - deep-learning - audio - name: silero-vad verbose: false +kind: job spec: - description: Silero VAD (Voice Activity Detection) functions. + image: '' + disable_auto_mount: false build: - code_origin: '' - base_image: mlrun/mlrun + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAyNCBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgbG9nZ2luZwpmcm9tIG11bHRpcHJvY2Vzc2luZyBpbXBvcnQgUHJvY2VzcywgUXVldWUKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwZXMgaW1wb3J0IEZ1bmN0aW9uVHlwZQoKaW1wb3J0IHRvcmNoCmltcG9ydCB0b3JjaGF1ZGlvCmZyb20gdHFkbSBpbXBvcnQgdHFkbQoKCmNsYXNzIEJhc2VUYXNrOgogICAgIiIiCiAgICBBIGJhc2UgY2xhc3MgZm9yIGEgdGFzayB0byBjb21wbGV0ZSBhZnRlciBWQUQuCiAgICAiIiIKCiAgICBkZWYgX19pbml0X18oc2VsZiwgYXVkaW9fZmlsZTogUGF0aCk6CiAgICAgICAgIiIiCiAgICAgICAgSW5pdGlhbGl6ZSB0aGUgYmFzZSB0YXNrLgoKICAgICAgICA6cGFyYW0gYXVkaW9fZmlsZTogVGhlIGF1ZGlvIGZpbGUgYXNzaWduZWQgdG8gdGhlIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgIyBTdG9yZSB0aGUgYXVkaW8gZmlsZToKICAgICAgICBzZWxmLl9hdWRpb19maWxlID0gYXVkaW9fZmlsZQoKICAgICAgICAjIFByZXBhcmUgdGhlIHJlc3VsdDoKICAgICAgICBzZWxmLl9yZXN1bHQgPSBOb25lCgogICAgQHByb3BlcnR5CiAgICBkZWYgYXVkaW9fZmlsZShzZWxmKSAtPiBQYXRoOgogICAgICAgICIiIgogICAgICAgIEdldCB0aGUgYXVkaW8gZmlsZSBvZiB0aGUgdGFzay4KCiAgICAgICAgOnJldHVybnM6IFRoZSBhdWRpbyBmaWxlIG9mIHRoZSB0YXNrLgogICAgICAgICIiIgogICAgICAgIHJldHVybiBzZWxmLl9hdWRpb19maWxlCgogICAgZGVmIGRvX3Rhc2soCiAgICAgICAgc2VsZiwgc3BlZWNoX3RpbWVzdGFtcHM6IGxpc3RbZGljdFtzdHIsIGludF1dIHwgbGlzdFtsaXN0W2RpY3Rbc3RyLCBpbnRdXV0KICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgRG8gdGhlIHRhc2sgb24gdGhlIGdpdmVuIHNwZWVjaCB0aW1lc3RhbXBzLiBUaGUgYmFzZSB0YXNrIHdpbGwgc2ltcGx5IHNhdmUgdGhlIHNwZWVjaCB0aW1lc3RhbXBzIGFzIHRoZSByZXN1bHQuCgogICAgICAgIDpwYXJhbSBzcGVlY2hfdGltZXN0YW1wczogVGhlIHNwZWVjaCB0aW1lc3RhbXBzIHRvIGRvIHRoZSB0YXNrIG9uIGFzIG91dHB1dHRlZCBmcm9tIHRoZSBWQUQuCiAgICAgICAgIiIiCiAgICAgICAgc2VsZi5fcmVzdWx0ID0gc3BlZWNoX3RpbWVzdGFtcHMKCiAgICBkZWYgZ2V0X3Jlc3VsdChzZWxmKSAtPiB0dXBsZVtzdHIsIGxpc3RdOgogICAgICAgICIiIgogICAgICAgIEdldCB0aGUgcmVzdWx0IG9mIHRoZSB0YXNrLiBBIHR1cGxlIG9mIHRoZSBhdWRpbyBmaWxlIG5hbWUgYW5kIHRoZSByZXN1bHQuCgogICAgICAgIDpyZXR1cm5zOiBUaGUgcmVzdWx0IG9mIHRoZSB0YXNrLgogICAgICAgICIiIgogICAgICAgIHJldHVybiBzZWxmLl9hdWRpb19maWxlLm5hbWUsIHNlbGYuX3Jlc3VsdAoKICAgIGRlZiB0b190dXBsZShzZWxmKSAtPiB0dXBsZVtzdHIsIGRpY3RdOgogICAgICAgICIiIgogICAgICAgIENvbnZlcnQgdGhlIHRhc2sgdG8gYSB0dXBsZSB0byByZWNvbnN0cnVjdCBpdCBsYXRlciAodXNlZCBmb3IgbXVsdGlwcm9jZXNzaW5nIHRvIHBhc3MgaW4gcXVldWUpLgoKICAgICAgICA6cmV0dXJuczogVGhlIGNvbnZlcnRlZCB0YXNrLgogICAgICAgICIiIgogICAgICAgIHJldHVybiBzZWxmLl9fY2xhc3NfXy5fX25hbWVfXywgeyJhdWRpb19maWxlIjogc2VsZi5fYXVkaW9fZmlsZX0KCgpjbGFzcyBTcGVlY2hEaWFyaXphdGlvblRhc2soQmFzZVRhc2spOgogICAgIiIiCiAgICBBIHNwZWVjaCBkaWFyaXphdGlvbiB0YXNrLiBUaGUgdGFzayB3aWxsIGRpYXJpemUgdGhlIFZBRCBzcGVlY2ggdGltZXN0YW1wcyBpbnRvIHNwZWFrZXJzLgogICAgIiIiCgogICAgZGVmIF9faW5pdF9fKHNlbGYsIGF1ZGlvX2ZpbGU6IFBhdGgsIHNwZWFrZXJfbGFiZWxzOiBsaXN0W3N0cl0pOgogICAgICAgICIiIgogICAgICAgIEluaXRpYWxpemUgdGhlIHNwZWVjaCBkaWFyaXphdGlvbiB0YXNrLgoKICAgICAgICA6cGFyYW0gYXVkaW9fZmlsZTogICAgIFRoZSBhdWRpbyBmaWxlIGFzc2lnbmVkIHRvIHRoZSB0YXNrLgogICAgICAgIDpwYXJhbSBzcGVha2VyX2xhYmVsczogVGhlIHNwZWFrZXIgbGFiZWxzIHRvIHVzZSBmb3IgdGhlIGRpYXJpemF0aW9uLiBJZiBub3QgZ2l2ZW4sIHRoZSBzcGVha2VycyB3aWxsIGJlIG5hbWVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAic3BlYWtlcl8wIiwgInNwZWFrZXJfMSIsIGV0Yy4KICAgICAgICAiIiIKICAgICAgICBzdXBlcigpLl9faW5pdF9fKGF1ZGlvX2ZpbGU9YXVkaW9fZmlsZSkKICAgICAgICBzZWxmLl9zcGVha2VyX2xhYmVscyA9IHNwZWFrZXJfbGFiZWxzCgogICAgZGVmIGRvX3Rhc2soc2VsZiwgc3BlZWNoX3RpbWVzdGFtcHM6IGxpc3RbbGlzdFtkaWN0W3N0ciwgaW50XV1dKToKICAgICAgICAiIiIKICAgICAgICBEbyB0aGUgdGFzayBvbiB0aGUgZ2l2ZW4gc3BlZWNoIHRpbWVzdGFtcHMuIFRoZSB0YXNrIHdpbGwgZGlhcml6ZSB0aGUgVkFEIHNwZWVjaCB0aW1lc3RhbXBzIGludG8gc3BlYWtlcnMuCgogICAgICAgIDpwYXJhbSBzcGVlY2hfdGltZXN0YW1wczogVGhlIHNwZWVjaCB0aW1lc3RhbXBzIHBlciBjaGFubmVsIHRvIGRvIHRoZSB0YXNrIG9uIGFzIG91dHB1dHRlZCBmcm9tIHRoZSBWQUQuCiAgICAgICAgIiIiCiAgICAgICAgIyBHZXQgdGhlIHNwZWFrZXIgbGFiZWxzIChzZXQgZGVmYXVsdCBpZiBub3QgZ2l2ZW4pOgogICAgICAgIHNwZWFrZXJfbGFiZWxzID0gc2VsZi5fc3BlYWtlcl9sYWJlbHMgb3IgWwogICAgICAgICAgICBmInNwZWFrZXJfe2l9IiBmb3IgaSBpbiByYW5nZShsZW4oc3BlZWNoX3RpbWVzdGFtcHMpKQogICAgICAgIF0KCiAgICAgICAgIyBEaWFyaXplIC0gb3JnYW5pemUgdGhlIHNwZWVjaCB0aW1lc3RhbXBzIGludG8gYSBzaW5nbGUgbGlzdCBvZiBzcGVha2VycyBhbmQgc29ydCBpdCBieSBzdGFydCB0aW1lOgogICAgICAgIHNwZWVjaF9kaWFyaXphdGlvbiA9IFsKICAgICAgICAgICAgKHNwZWVjaF90aW1lc3RhbXBbInN0YXJ0Il0sIHNwZWVjaF90aW1lc3RhbXBbImVuZCJdLCBzcGVha2VyX2xhYmVsKQogICAgICAgICAgICBmb3Igc3BlYWtlcl9sYWJlbCwgY2hhbm5lbF9zcGVlY2hfdGltZXN0YW1wcyBpbiB6aXAoCiAgICAgICAgICAgICAgICBzcGVha2VyX2xhYmVscywgc3BlZWNoX3RpbWVzdGFtcHMKICAgICAgICAgICAgKQogICAgICAgICAgICBmb3Igc3BlZWNoX3RpbWVzdGFtcCBpbiBjaGFubmVsX3NwZWVjaF90aW1lc3RhbXBzCiAgICAgICAgXQogICAgICAgIHNwZWVjaF9kaWFyaXphdGlvbi5zb3J0KCkKICAgICAgICBzZWxmLl9yZXN1bHQgPSBzcGVlY2hfZGlhcml6YXRpb24KCiAgICBkZWYgdG9fdHVwbGUoc2VsZikgLT4gdHVwbGVbc3RyLCBkaWN0XToKICAgICAgICAiIiIKICAgICAgICBDb252ZXJ0IHRoZSB0YXNrIHRvIGEgdHVwbGUgdG8gcmVjb25zdHJ1Y3QgaXQgbGF0ZXIgKHVzZWQgZm9yIG11bHRpcHJvY2Vzc2luZyB0byBwYXNzIGluIHF1ZXVlKS4KCiAgICAgICAgOnJldHVybnM6IFRoZSBjb252ZXJ0ZWQgdGFzay4KICAgICAgICAiIiIKICAgICAgICB0YXNrX2NsYXNzLCB0YXNrX2t3YXJncyA9IHN1cGVyKCkudG9fdHVwbGUoKQogICAgICAgIHJldHVybiB0YXNrX2NsYXNzLCB7Kip0YXNrX2t3YXJncywgInNwZWFrZXJfbGFiZWxzIjogc2VsZi5fc3BlYWtlcl9sYWJlbHN9CgoKY2xhc3MgVGFza0NyZWF0b3I6CiAgICAiIiIKICAgIEEgdGFzayBjcmVhdG9yIHRvIGNyZWF0ZSBkaWZmZXJlbnQgdGFza3MgdG8gcnVuIGFmdGVyIHRoZSBWQUQuCiAgICAiIiIKCiAgICAjOiBBIG1hcCBmcm9tIHRhc2sgY2xhc3MgbmFtZSB0byB0YXNrIGNsYXNzIHRvIHVzZSBpbiBgZnJvbV90dXBsZWA6CiAgICBfTUFQID0gewogICAgICAgIEJhc2VUYXNrLl9fbmFtZV9fOiBCYXNlVGFzaywKICAgICAgICBTcGVlY2hEaWFyaXphdGlvblRhc2suX19uYW1lX186IFNwZWVjaERpYXJpemF0aW9uVGFzaywKICAgIH0KCiAgICBkZWYgX19pbml0X18oc2VsZiwgdGFza190eXBlOiB0eXBlW0Jhc2VUYXNrXSwgdGFza19rd2FyZ3M6IGRpY3QgPSBOb25lKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSB0YXNrIGNyZWF0b3IuCiAgICAgICAgOnBhcmFtIHRhc2tfdHlwZTogVGhlIHRhc2sgdHlwZSAtIGEgYEJhc2VUYXNrYCBzdWJjbGFzcy4KICAgICAgICA6cGFyYW0gdGFza19rd2FyZ3M6IEFkZGl0aW9uYWwga2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUgdG8gYmUgY3JlYXRlZCB0YXNrcy4KICAgICAgICAiIiIKICAgICAgICBzZWxmLl90YXNrX3R5cGUgPSB0YXNrX3R5cGUKICAgICAgICBzZWxmLl90YXNrX2t3YXJncyA9IHRhc2tfa3dhcmdzIG9yIHt9CgogICAgZGVmIGNyZWF0ZV90YXNrKHNlbGYsIGF1ZGlvX2ZpbGU6IFBhdGgpIC0+IEJhc2VUYXNrOgogICAgICAgICIiIgogICAgICAgIENyZWF0ZSBhIHRhc2sgd2l0aCB0aGUgZ2l2ZW4gYXVkaW8gZmlsZS4KCiAgICAgICAgOnBhcmFtIGF1ZGlvX2ZpbGU6IFRoZSBhdWRpbyBmaWxlIHRvIGFzc2lnbiB0byB0aGUgdGFzay4KCiAgICAgICAgOnJldHVybnM6IFRoZSBjcmVhdGVkIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX3Rhc2tfdHlwZShhdWRpb19maWxlPWF1ZGlvX2ZpbGUsICoqc2VsZi5fdGFza19rd2FyZ3MpCgogICAgQGNsYXNzbWV0aG9kCiAgICBkZWYgZnJvbV90dXBsZShjbHMsIHRhc2tfdHVwbGU6IHR1cGxlW3N0ciwgZGljdF0pIC0+IEJhc2VUYXNrOgogICAgICAgICIiIgogICAgICAgIENyZWF0ZSBhIHRhc2sgZnJvbSBhIHR1cGxlIG9mIHRoZSBhdWRpbyBmaWxlIG5hbWUgYW5kIHRoZSB0YXNrIGt3YXJncy4KCiAgICAgICAgOnBhcmFtIHRhc2tfdHVwbGU6IFRoZSB0YXNrIHR1cGxlIHRvIGNyZWF0ZSB0aGUgdGFzayBmcm9tLgoKICAgICAgICA6cmV0dXJuczogVGhlIGNyZWF0ZWQgdGFzay4KICAgICAgICAiIiIKICAgICAgICB0YXNrX2NsYXNzLCB0YXNrX2t3YXJncyA9IHRhc2tfdHVwbGUKICAgICAgICByZXR1cm4gY2xzLl9NQVBbdGFza19jbGFzc10oKip0YXNrX2t3YXJncykKCgpjbGFzcyBWb2ljZUFjdGl2aXR5RGV0ZWN0b3I6CiAgICAiIiIKICAgIEEgdm9pY2UgYWN0aXZpdHkgZGV0ZWN0aW9uIHdyYXBwZXIgZm9yIHRoZSBzaWxlcm8gVkFEIG1vZGVsIC0gaHR0cHM6Ly9naXRodWIuY29tL3NuYWtlcnM0L3NpbGVyby12YWQuCiAgICAiIiIKCiAgICBkZWYgX19pbml0X18oCiAgICAgICAgc2VsZiwKICAgICAgICAjIE1vZGVsIGxvYWRpbmcga3dhcmdzOgogICAgICAgIHVzZV9vbm54OiBib29sID0gVHJ1ZSwKICAgICAgICBmb3JjZV9vbm54X2NwdTogYm9vbCA9IFRydWUsCiAgICAgICAgIyBEZXRlY3Rpb24ga3dhcmdzOgogICAgICAgIHRocmVzaG9sZDogZmxvYXQgPSAwLjUsCiAgICAgICAgc2FtcGxpbmdfcmF0ZTogaW50ID0gMTZfMDAwLAogICAgICAgIG1pbl9zcGVlY2hfZHVyYXRpb25fbXM6IGludCA9IDI1MCwKICAgICAgICBtYXhfc3BlZWNoX2R1cmF0aW9uX3M6IGZsb2F0ID0gZmxvYXQoImluZiIpLAogICAgICAgIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zOiBpbnQgPSAxMDAsCiAgICAgICAgd2luZG93X3NpemVfc2FtcGxlczogaW50ID0gNTEyLAogICAgICAgIHNwZWVjaF9wYWRfbXM6IGludCA9IDMwLAogICAgICAgIHJldHVybl9zZWNvbmRzOiBib29sID0gRmFsc2UsCiAgICAgICAgcGVyX2NoYW5uZWw6IGJvb2wgPSBGYWxzZSwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgSW5pdGlhbGl6ZSB0aGUgdm9pY2UgYWN0aXZpdHkgZGV0ZWN0b3IuCgogICAgICAgIDpwYXJhbSB1c2Vfb25ueDogICAgICAgICAgICAgICAgV2hldGhlciB0byB1c2UgT05OWCBmb3IgaW5mZXJlbmNlLiBEZWZhdWx0IGlzIFRydWUuCiAgICAgICAgOnBhcmFtIGZvcmNlX29ubnhfY3B1OiAgICAgICAgICBXaGV0aGVyIHRvIGZvcmNlIE9OTlggdG8gdXNlIENQVSBmb3IgaW5mZXJlbmNlLiBEZWZhdWx0IGlzIFRydWUuCiAgICAgICAgOnBhcmFtIHRocmVzaG9sZDogICAgICAgICAgICAgICBTcGVlY2ggdGhyZXNob2xkLiBTaWxlcm8gVkFEIG91dHB1dHMgc3BlZWNoIHByb2JhYmlsaXRpZXMgZm9yIGVhY2ggYXVkaW8gY2h1bmssCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBwcm9iYWJpbGl0aWVzIEFCT1ZFIHRoaXMgdmFsdWUgYXJlIGNvbnNpZGVyZWQgYXMgU1BFRUNILiBJdCBpcyBiZXR0ZXIgdG8gdHVuZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhpcyBwYXJhbWV0ZXIgZm9yIGVhY2ggZGF0YXNldCBzZXBhcmF0ZWx5LCBidXQgImxhenkiIDAuNSBpcyBwcmV0dHkgZ29vZCBmb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vc3QgZGF0YXNldHMuCiAgICAgICAgOnBhcmFtIHNhbXBsaW5nX3JhdGU6ICAgICAgICAgICBDdXJyZW50bHksIHNpbGVybyBWQUQgbW9kZWxzIHN1cHBvcnQgODAwMCBhbmQgMTYwMDAgc2FtcGxlIHJhdGVzLgogICAgICAgIDpwYXJhbSBtaW5fc3BlZWNoX2R1cmF0aW9uX21zOiAgRmluYWwgc3BlZWNoIGNodW5rcyBzaG9ydGVyIG1pbl9zcGVlY2hfZHVyYXRpb25fbXMgYXJlIHRocm93biBvdXQuCiAgICAgICAgOnBhcmFtIG1heF9zcGVlY2hfZHVyYXRpb25fczogICBNYXhpbXVtIGR1cmF0aW9uIG9mIHNwZWVjaCBjaHVua3MgaW4gc2Vjb25kcy4gQ2h1bmtzIGxvbmdlciB0aGFuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBgbWF4X3NwZWVjaF9kdXJhdGlvbl9zYCB3aWxsIGJlIHNwbGl0IGF0IHRoZSB0aW1lc3RhbXAgb2YgdGhlIGxhc3Qgc2lsZW5jZSB0aGF0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBsYXN0cyBtb3JlIHRoYW4gMTAwbXMgKGlmIGFueSksIHRvIHByZXZlbnQgYWdncmVzc2l2ZSBjdXR0aW5nLiBPdGhlcndpc2UsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGV5IHdpbGwgYmUgc3BsaXQgYWdncmVzc2l2ZWx5IGp1c3QgYmVmb3JlIG1heF9zcGVlY2hfZHVyYXRpb25fcy4KICAgICAgICA6cGFyYW0gbWluX3NpbGVuY2VfZHVyYXRpb25fbXM6IEluIHRoZSBlbmQgb2YgZWFjaCBzcGVlY2ggY2h1bmsgd2FpdCBmb3IgbWluX3NpbGVuY2VfZHVyYXRpb25fbXMgYmVmb3JlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzZXBhcmF0aW5nIGl0LgogICAgICAgIDpwYXJhbSB3aW5kb3dfc2l6ZV9zYW1wbGVzOiAgICAgQXVkaW8gY2h1bmtzIG9mIHdpbmRvd19zaXplX3NhbXBsZXMgc2l6ZSBhcmUgZmVkIHRvIHRoZSBzaWxlcm8gVkFEIG1vZGVsLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV0FSTklORyEgU2lsZXJvIFZBRCBtb2RlbHMgd2VyZSB0cmFpbmVkIHVzaW5nIDUxMiwgMTAyNCwgMTUzNiBzYW1wbGVzIGZvciAxNjAwMAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc2FtcGxlIHJhdGUgYW5kIDI1NiwgNTEyLCA3Njggc2FtcGxlcyBmb3IgODAwMCBzYW1wbGUgcmF0ZS4gVmFsdWVzIG90aGVyIHRoYW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZXNlIG1heSBhZmZlY3QgbW9kZWwgcGVyZm9ybWFuY2UhCiAgICAgICAgOnBhcmFtIHNwZWVjaF9wYWRfbXM6ICAgICAgICAgICBGaW5hbCBzcGVlY2ggY2h1bmtzIGFyZSBwYWRkZWQgYnkgc3BlZWNoX3BhZF9tcyBlYWNoIHNpZGUuCiAgICAgICAgOnBhcmFtIHJldHVybl9zZWNvbmRzOiAgICAgICAgICBXaGV0aGVyIHJldHVybiB0aW1lc3RhbXBzIGluIHNlY29uZHMuIEZhbHNlIG1lYW5zIHRvIHJldHVybiB0aW1lc3RhbXBzIGluCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGVzIChkZWZhdWx0IC0gRmFsc2UpLgogICAgICAgIDpwYXJhbSBwZXJfY2hhbm5lbDogICAgICAgICAgICAgV2hldGhlciB0byByZXR1cm4gdGltZXN0YW1wcyBwZXIgY2hhbm5lbCAoZGVmYXVsdCAtIEZhbHNlKS4gVGhpcyB3aWxsIHJ1biBWQUQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9uIGVhY2ggY2hhbm5lbCBzZXBhcmF0ZWx5IGFuZCByZXR1cm4gYSBsaXN0IG9mIHRpbWVzdGFtcHMgcGVyIGNoYW5uZWwuCiAgICAgICAgIiIiCiAgICAgICAgIyBTdG9yZSBjb25maWd1cmF0aW9uczoKICAgICAgICBzZWxmLl91c2Vfb25ueCA9IHVzZV9vbm54CiAgICAgICAgc2VsZi5fZm9yY2Vfb25ueF9jcHUgPSBmb3JjZV9vbm54X2NwdQogICAgICAgIHNlbGYuX3RocmVzaG9sZCA9IHRocmVzaG9sZAogICAgICAgIHNlbGYuX3NhbXBsaW5nX3JhdGUgPSBzYW1wbGluZ19yYXRlCiAgICAgICAgc2VsZi5fbWluX3NwZWVjaF9kdXJhdGlvbl9tcyA9IG1pbl9zcGVlY2hfZHVyYXRpb25fbXMKICAgICAgICBzZWxmLl9tYXhfc3BlZWNoX2R1cmF0aW9uX3MgPSBtYXhfc3BlZWNoX2R1cmF0aW9uX3MKICAgICAgICBzZWxmLl9taW5fc2lsZW5jZV9kdXJhdGlvbl9tcyA9IG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zCiAgICAgICAgc2VsZi5fd2luZG93X3NpemVfc2FtcGxlcyA9IHdpbmRvd19zaXplX3NhbXBsZXMKICAgICAgICBzZWxmLl9zcGVlY2hfcGFkX21zID0gc3BlZWNoX3BhZF9tcwogICAgICAgIHNlbGYuX3JldHVybl9zZWNvbmRzID0gcmV0dXJuX3NlY29uZHMKICAgICAgICBzZWxmLl9wZXJfY2hhbm5lbCA9IHBlcl9jaGFubmVsCgogICAgICAgICMgUHJlcGFyZSB0aGUgbW9kZWwgdmFyaWFibGVzCiAgICAgICAgc2VsZi5fbW9kZWw6IHRvcmNoLk1vZHVsZSA9IE5vbmUKICAgICAgICBzZWxmLl9nZXRfc3BlZWNoX3RpbWVzdGFtcHM6IEZ1bmN0aW9uVHlwZSA9IE5vbmUKCiAgICBkZWYgbG9hZChzZWxmLCBmb3JjZV9yZWxvYWQ6IGJvb2wgPSBUcnVlKToKICAgICAgICAiIiIKICAgICAgICBMb2FkIHRoZSBWQUQgbW9kZWwuCgogICAgICAgIDpwYXJhbSBmb3JjZV9yZWxvYWQ6IFdoZXRoZXIgdG8gZm9yY2UgcmVsb2FkIHRoZSBtb2RlbCBldmVuIGlmIGl0IHdhcyBhbHJlYWR5IGxvYWRlZC4gRGVmYXVsdCBpcyBUcnVlLgogICAgICAgICIiIgogICAgICAgIG1vZGVsLCB1dGlscyA9IHRvcmNoLmh1Yi5sb2FkKAogICAgICAgICAgICByZXBvX29yX2Rpcj0ic25ha2VyczQvc2lsZXJvLXZhZCIsCiAgICAgICAgICAgIG1vZGVsPSJzaWxlcm9fdmFkIiwKICAgICAgICAgICAgZm9yY2VfcmVsb2FkPWZvcmNlX3JlbG9hZCwKICAgICAgICAgICAgb25ueD1zZWxmLl91c2Vfb25ueCwKICAgICAgICAgICAgZm9yY2Vfb25ueF9jcHU9c2VsZi5fZm9yY2Vfb25ueF9jcHUsCiAgICAgICAgKQogICAgICAgIHNlbGYuX21vZGVsID0gbW9kZWwKICAgICAgICAoCiAgICAgICAgICAgIHNlbGYuX2dldF9zcGVlY2hfdGltZXN0YW1wcywKICAgICAgICAgICAgXywgICMgc2F2ZV9hdWRpbywKICAgICAgICAgICAgXywgICMgcmVhZF9hdWRpbywKICAgICAgICAgICAgXywgICMgVkFESXRlcmF0b3IsCiAgICAgICAgICAgIF8sICAjIGNvbGxlY3RfY2h1bmtzCiAgICAgICAgKSA9IHV0aWxzCgogICAgZGVmIGRldGVjdF92b2ljZSgKICAgICAgICBzZWxmLAogICAgICAgIGF1ZGlvX2ZpbGU6IFBhdGgsCiAgICApIC0+IGxpc3RbZGljdFtzdHIsIGludF1dIHwgbGlzdFtsaXN0W2RpY3Rbc3RyLCBpbnRdXV06CiAgICAgICAgIiIiCiAgICAgICAgSW5mZXIgdGhlIGF1ZGlvIHRocm91Z2ggdGhlIFZBRCBtb2RlbCBhbmQgcmV0dXJuIHRoZSBzcGVlY2ggdGltZXN0YW1wcy4KCiAgICAgICAgOnBhcmFtIGF1ZGlvX2ZpbGU6IFRoZSBhdWRpbyBmaWxlIHRvIGluZmVyLgoKICAgICAgICA6cmV0dXJuczogVGhlIHNwZWVjaCB0aW1lc3RhbXBzIGluIHRoZSBhdWRpby4gQSBsaXN0IG9mIHRpbWVzdGFtcHMgd2hlcmUgZWFjaCB0aW1lc3RhbXAgaXMgYSBkaWN0aW9uYXJ5IHdpdGggdGhlCiAgICAgICAgICAgICAgICAgZm9sbG93aW5nIGtleXM6CgogICAgICAgICAgICAgICAgICogInN0YXJ0IjogVGhlIHN0YXJ0IHNhbXBsZSBpbmRleCBvZiB0aGUgc3BlZWNoIGluIHRoZSBhdWRpby4KICAgICAgICAgICAgICAgICAqICJlbmQiOiAgIFRoZSBlbmQgc2FtcGxlIGluZGV4IG9mIHRoZSBzcGVlY2ggaW4gdGhlIGF1ZGlvLgoKICAgICAgICAgICAgICAgICBJZiBgcGVyX2NoYW5uZWxgIGlzIFRydWUsIGEgbGlzdCBvZiB0aW1lc3RhbXBzIHBlciBjaGFubmVsIHdpbGwgYmUgcmV0dXJuZWQuCiAgICAgICAgIiIiCiAgICAgICAgIyBDYXN0IHRvIGEgbnVtcHkgYXJyYXk6CiAgICAgICAgYXVkaW8gPSBzZWxmLl9yZWFkX2F1ZGlvKGF1ZGlvX2ZpbGUpCgogICAgICAgICMgRGV0ZWN0IHNwZWVjaDoKICAgICAgICBpZiBub3Qgc2VsZi5fcGVyX2NoYW5uZWw6CiAgICAgICAgICAgIHJldHVybiBzZWxmLl9nZXRfc3BlZWNoX3RpbWVzdGFtcHMoCiAgICAgICAgICAgICAgICBhdWRpbywKICAgICAgICAgICAgICAgIHNlbGYuX21vZGVsLAogICAgICAgICAgICAgICAgdGhyZXNob2xkPXNlbGYuX3RocmVzaG9sZCwKICAgICAgICAgICAgICAgIG1pbl9zcGVlY2hfZHVyYXRpb25fbXM9c2VsZi5fbWluX3NwZWVjaF9kdXJhdGlvbl9tcywKICAgICAgICAgICAgICAgIG1heF9zcGVlY2hfZHVyYXRpb25fcz1zZWxmLl9tYXhfc3BlZWNoX2R1cmF0aW9uX3MsCiAgICAgICAgICAgICAgICBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcz1zZWxmLl9taW5fc2lsZW5jZV9kdXJhdGlvbl9tcywKICAgICAgICAgICAgICAgIHNwZWVjaF9wYWRfbXM9c2VsZi5fc3BlZWNoX3BhZF9tcywKICAgICAgICAgICAgICAgIHNhbXBsaW5nX3JhdGU9c2VsZi5fc2FtcGxpbmdfcmF0ZSwKICAgICAgICAgICAgICAgIHdpbmRvd19zaXplX3NhbXBsZXM9c2VsZi5fd2luZG93X3NpemVfc2FtcGxlcywKICAgICAgICAgICAgICAgIHJldHVybl9zZWNvbmRzPXNlbGYuX3JldHVybl9zZWNvbmRzLAogICAgICAgICAgICApCgogICAgICAgICMgUGVyIGNoYW5uZWw6CiAgICAgICAgc3BlZWNoX3RpbWVzdGFtcHMgPSBbXQogICAgICAgIGZvciBjaGFubmVsIGluIGF1ZGlvOgogICAgICAgICAgICBzcGVlY2hfdGltZXN0YW1wcy5hcHBlbmQoCiAgICAgICAgICAgICAgICBzZWxmLl9nZXRfc3BlZWNoX3RpbWVzdGFtcHMoCiAgICAgICAgICAgICAgICAgICAgY2hhbm5lbCwKICAgICAgICAgICAgICAgICAgICBzZWxmLl9tb2RlbCwKICAgICAgICAgICAgICAgICAgICB0aHJlc2hvbGQ9c2VsZi5fdGhyZXNob2xkLAogICAgICAgICAgICAgICAgICAgIG1pbl9zcGVlY2hfZHVyYXRpb25fbXM9c2VsZi5fbWluX3NwZWVjaF9kdXJhdGlvbl9tcywKICAgICAgICAgICAgICAgICAgICBtYXhfc3BlZWNoX2R1cmF0aW9uX3M9c2VsZi5fbWF4X3NwZWVjaF9kdXJhdGlvbl9zLAogICAgICAgICAgICAgICAgICAgIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zPXNlbGYuX21pbl9zaWxlbmNlX2R1cmF0aW9uX21zLAogICAgICAgICAgICAgICAgICAgIHNwZWVjaF9wYWRfbXM9c2VsZi5fc3BlZWNoX3BhZF9tcywKICAgICAgICAgICAgICAgICAgICBzYW1wbGluZ19yYXRlPXNlbGYuX3NhbXBsaW5nX3JhdGUsCiAgICAgICAgICAgICAgICAgICAgd2luZG93X3NpemVfc2FtcGxlcz1zZWxmLl93aW5kb3dfc2l6ZV9zYW1wbGVzLAogICAgICAgICAgICAgICAgICAgIHJldHVybl9zZWNvbmRzPXNlbGYuX3JldHVybl9zZWNvbmRzLAogICAgICAgICAgICAgICAgKQogICAgICAgICAgICApCgogICAgICAgIHJldHVybiBzcGVlY2hfdGltZXN0YW1wcwoKICAgIGRlZiBfcmVhZF9hdWRpbygKICAgICAgICBzZWxmLAogICAgICAgIHBhdGg6IFBhdGgsCiAgICApIC0+IHRvcmNoLlRlbnNvcjoKICAgICAgICAiIiIKICAgICAgICBSZWFkIHRoZSBhdWRpbyBmcm9tIHRoZSBnaXZlbiBwYXRoIGFuZCByZXR1cm4gaXQgYXMgYSB0ZW5zb3IuCgogICAgICAgIDpwYXJhbSBwYXRoOiBUaGUgcGF0aCB0byB0aGUgYXVkaW8gZmlsZS4KCiAgICAgICAgOnJldHVybnM6IFRoZSBhdWRpbyBhcyBhIHRlbnNvci4KICAgICAgICAiIiIKICAgICAgICAjIFJlYWQgdGhlIGF1ZGlvOgogICAgICAgIGF1ZGlvLCBzYW1wbGluZ19yYXRlID0gdG9yY2hhdWRpby5sb2FkKHN0cihwYXRoKSkKCiAgICAgICAgIyBDaGVjayBpZiB0aGUgYXVkaW8gaXMgc3RlcmVvIGFuZCBpZiBzbywgY29udmVydCBpdCB0byBtb25vIChvbmx5IGlmIG5vdCBwZXIgY2hhbm5lbCk6CiAgICAgICAgaWYgYXVkaW8uc2l6ZSgwKSA+IDEgYW5kIG5vdCBzZWxmLl9wZXJfY2hhbm5lbDoKICAgICAgICAgICAgYXVkaW8gPSBhdWRpby5tZWFuKGRpbT0wLCBrZWVwZGltPVRydWUpCgogICAgICAgICMgUmVzYW1wbGUgdGhlIGF1ZGlvIGlmIG5lZWRlZDoKICAgICAgICBpZiBzYW1wbGluZ19yYXRlICE9IHNlbGYuX3NhbXBsaW5nX3JhdGU6CiAgICAgICAgICAgIHRyYW5zZm9ybSA9IHRvcmNoYXVkaW8udHJhbnNmb3Jtcy5SZXNhbXBsZSgKICAgICAgICAgICAgICAgIG9yaWdfZnJlcT1zYW1wbGluZ19yYXRlLCBuZXdfZnJlcT1zZWxmLl9zYW1wbGluZ19yYXRlCiAgICAgICAgICAgICkKICAgICAgICAgICAgYXVkaW8gPSB0cmFuc2Zvcm0oYXVkaW8pCgogICAgICAgICMgUmV0dXJuIHRoZSBhdWRpbyAoc3F1ZWV6ZSBpZiBub3QgcGVyIGNoYW5uZWwpOgogICAgICAgIHJldHVybiBhdWRpbyBpZiBzZWxmLl9wZXJfY2hhbm5lbCBlbHNlIGF1ZGlvLnNxdWVlemUoMCkKCgojOiBUaGUgdmFsdWUgdG8gc2VuZCBpbnRvIG11bHRpcHJvY2Vzc2luZyBxdWV1ZXMgdG8gc3RvcCB0aGUgcHJvY2VzczoKX01VTFRJUFJPQ0VTU0lOR19TVE9QX01BUksgPSAiU1RPUCIKCgpkZWYgX211bHRpcHJvY2Vzc2luZ19jb21wbGV0ZV90YXNrcygKICAgIHZhZF9pbml0X2t3YXJnczogZGljdCwgdGFza3NfcXVldWU6IFF1ZXVlLCByZXN1bHRzX3F1ZXVlOiBRdWV1ZQopOgogICAgIiIiCiAgICBDb21wbGV0ZSB0aGUgdGFza3MgaW4gdGhlIGdpdmVuIHF1ZXVlIGFuZCBwdXQgdGhlIHJlc3VsdHMgaW4gdGhlIGdpdmVuIHJlc3VsdHMgcXVldWUuIFRoZSBmdW5jdGlvbiB3aWxsIHN0b3Agd2hlbgogICAgdGhlIGdpdmVuIHRhc2tzIHF1ZXVlIHdpbGwgcmVjZWl2ZSB0aGUgc3RvcCBtYXJrLiBJdCBpcyBhaW1lZCB0byBiZSB1c2VkIHdpdGggbXVsdGlwcm9jZXNzaW5nIGFzIGEgcHJvY2Vzcy4KCiAgICA6cGFyYW0gdmFkX2luaXRfa3dhcmdzOiBUaGUgVkFEIGluaXRpYWxpemF0aW9uIGt3YXJncy4KICAgIDpwYXJhbSB0YXNrc19xdWV1ZTogICAgIEEgcXVldWUgdG8gZ2V0IHRoZSB0YXNrcyBmcm9tLgogICAgOnBhcmFtIHJlc3VsdHNfcXVldWU6ICAgQSBxdWV1ZSB0byBwdXQgdGhlIHJlc3VsdHMgaW4uCiAgICAiIiIKICAgICMgSW5pdGlhbGl6ZSBhbmQgbG9hZCB0aGUgVkFEOgogICAgdmFkID0gVm9pY2VBY3Rpdml0eURldGVjdG9yKCoqdmFkX2luaXRfa3dhcmdzKQogICAgdmFkLmxvYWQoZm9yY2VfcmVsb2FkPUZhbHNlKQoKICAgICMgU3RhcnQgbGlzdGVuaW5nIHRvIHRoZSB0YXNrcyBxdWV1ZToKICAgIHdoaWxlIFRydWU6CiAgICAgICAgIyBHZXQgdGhlIHRhc2s6CiAgICAgICAgdGFzazogdHVwbGVbc3RyLCBkaWN0XSA9IHRhc2tzX3F1ZXVlLmdldCgpCiAgICAgICAgaWYgdGFzayA9PSBfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSzoKICAgICAgICAgICAgYnJlYWsKICAgICAgICB0cnk6CiAgICAgICAgICAgICMgQ3JlYXRlIHRoZSB0YXNrOgogICAgICAgICAgICB0YXNrID0gVGFza0NyZWF0b3IuZnJvbV90dXBsZSh0YXNrX3R1cGxlPXRhc2spCiAgICAgICAgICAgICMgUnVuIHRoZSBmaWxlIHRocm91Z2ggdGhlIFZBRDoKICAgICAgICAgICAgc3BlZWNoX3RpbWVzdGFtcHMgPSB2YWQuZGV0ZWN0X3ZvaWNlKGF1ZGlvX2ZpbGU9dGFzay5hdWRpb19maWxlKQogICAgICAgICAgICAjIENvbXBsZXRlIHRoZSB0YXNrOgogICAgICAgICAgICB0YXNrLmRvX3Rhc2soc3BlZWNoX3RpbWVzdGFtcHM9c3BlZWNoX3RpbWVzdGFtcHMpCiAgICAgICAgICAgICMgQnVpbGQgdGhlIHJlc3VsdDoKICAgICAgICAgICAgcmVzdWx0ID0gKEZhbHNlLCB0YXNrLmdldF9yZXN1bHQoKSkKICAgICAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGV4Y2VwdGlvbjoKICAgICAgICAgICAgIyBCdWlsZCB0aGUgZXJyb3I6CiAgICAgICAgICAgIHJlc3VsdCA9IChUcnVlLCAodGFzay5hdWRpb19maWxlLm5hbWUsIHN0cihleGNlcHRpb24pKSkKICAgICAgICAjIENvbGxlY3QgdGhlIHJlc3VsdCAvIGVycm9yOgogICAgICAgIHJlc3VsdHNfcXVldWUucHV0KHJlc3VsdCkKCiAgICAjIE1hcmsgdGhlIGVuZCBvZiB0aGUgdGFza3M6CiAgICByZXN1bHRzX3F1ZXVlLnB1dChfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSykKCgojIEdldCB0aGUgZ2xvYmFsIGxvZ2dlcjoKdHJ5OgogICAgaW1wb3J0IG1scnVuCgogICAgX0xPR0dFUiA9IG1scnVuLmdldF9vcl9jcmVhdGVfY3R4KCJzaWxlcm9fdmFkIikubG9nZ2VyCmV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgX0xPR0dFUiA9IGxvZ2dpbmcuZ2V0TG9nZ2VyKCkKCgpkZWYgZGV0ZWN0X3ZvaWNlKAogICAgIyBJbnB1dCBrd2FyZ3M6CiAgICBkYXRhX3BhdGg6IHN0ciB8IFBhdGggfCBsaXN0W3N0ciB8IFBhdGhdLAogICAgIyBNb2RlbCBsb2FkaW5nIGt3YXJnczoKICAgIHVzZV9vbm54OiBib29sID0gVHJ1ZSwKICAgIGZvcmNlX29ubnhfY3B1OiBib29sID0gVHJ1ZSwKICAgICMgRGV0ZWN0aW9uIGt3YXJnczoKICAgIHRocmVzaG9sZDogZmxvYXQgPSAwLjUsCiAgICBzYW1wbGluZ19yYXRlOiBpbnQgPSAxNl8wMDAsCiAgICBtaW5fc3BlZWNoX2R1cmF0aW9uX21zOiBpbnQgPSAyNTAsCiAgICBtYXhfc3BlZWNoX2R1cmF0aW9uX3M6IGZsb2F0ID0gZmxvYXQoImluZiIpLAogICAgbWluX3NpbGVuY2VfZHVyYXRpb25fbXM6IGludCA9IDEwMCwKICAgIHdpbmRvd19zaXplX3NhbXBsZXM6IGludCA9IDUxMiwKICAgIHNwZWVjaF9wYWRfbXM6IGludCA9IDMwLAogICAgcmV0dXJuX3NlY29uZHM6IGJvb2wgPSBGYWxzZSwKICAgIHBlcl9jaGFubmVsOiBib29sID0gRmFsc2UsCiAgICAjIE90aGVyIGt3YXJnczoKICAgIHVzZV9tdWx0aXByb2Nlc3Npbmc6IGludCA9IDAsCiAgICB2ZXJib3NlOiBib29sID0gRmFsc2UsCik6CiAgICAiIiIKICAgIFBlcmZvcm0gdm9pY2UgYWN0aXZpdHkgZGV0ZWN0aW9uIG9uIGdpdmVuIGF1ZGlvIGZpbGVzIHVzaW5nIHRoZSBzaWxlcm8gVkFEIG1vZGVsIC0KICAgIGh0dHBzOi8vZ2l0aHViLmNvbS9zbmFrZXJzNC9zaWxlcm8tdmFkLiBUaGUgZW5kIHJlc3VsdCBpcyBhIGRpY3Rpb25hcnkgd2l0aCB0aGUgZmlsZSBuYW1lcyBhcyBrZXlzIGFuZCB0aGVpcgogICAgVkFEIHRpbWVzdGFtcHMgZGljdGlvbmFyaWVzIGFzIHZhbHVlLgoKICAgIEZvciBleGFtcGxlOjoKCiAgICAgICAgewogICAgICAgICAgICAiZmlsZV8xLndhdiI6IFsKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAwLCAiZW5kIjogMTYwMDB9LAogICAgICAgICAgICAgICAgeyJzdGFydCI6IDE2MDAwLCAiZW5kIjogMzIwMDB9LAogICAgICAgICAgICAgICAgeyJzdGFydCI6IDMyMDAwLCAiZW5kIjogNDgwMDB9LAogICAgICAgICAgICAgICAgLi4uCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgICJmaWxlXzIud2F2IjogWwogICAgICAgICAgICAgICAgeyJzdGFydCI6IDAsICJlbmQiOiAxNjAwMH0sCiAgICAgICAgICAgICAgICB7InN0YXJ0IjogMTYwMDAsICJlbmQiOiAzMjAwMH0sCiAgICAgICAgICAgICAgICB7InN0YXJ0IjogMzIwMDAsICJlbmQiOiA0ODAwMH0sCiAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgXSwKICAgICAgICAgICAgLi4uCiAgICAgICAgfQoKCiAgICA6cGFyYW0gZGF0YV9wYXRoOiAgICAgICAgICAgICAgIFRoZSBwYXRoIHRvIHRoZSBhdWRpbyBmaWxlcyB0byBkaWFyaXplLiBDYW4gYmUgYSBwYXRoIHRvIGEgc2luZ2xlIGZpbGUsIGEgcGF0aCB0byBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRpcmVjdG9yeSBvciBhIGxpc3Qgb2YgcGF0aHMgdG8gZmlsZXMuCiAgICA6cGFyYW0gdXNlX29ubng6ICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gdXNlIE9OTlggZm9yIGluZmVyZW5jZS4gRGVmYXVsdCBpcyBUcnVlLgogICAgOnBhcmFtIGZvcmNlX29ubnhfY3B1OiAgICAgICAgICBXaGV0aGVyIHRvIGZvcmNlIE9OTlggdG8gdXNlIENQVSBmb3IgaW5mZXJlbmNlLiBEZWZhdWx0IGlzIFRydWUuCiAgICA6cGFyYW0gdGhyZXNob2xkOiAgICAgICAgICAgICAgIFNwZWVjaCB0aHJlc2hvbGQuIFNpbGVybyBWQUQgb3V0cHV0cyBzcGVlY2ggcHJvYmFiaWxpdGllcyBmb3IgZWFjaCBhdWRpbyBjaHVuaywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcHJvYmFiaWxpdGllcyBBQk9WRSB0aGlzIHZhbHVlIGFyZSBjb25zaWRlcmVkIGFzIFNQRUVDSC4gSXQgaXMgYmV0dGVyIHRvIHR1bmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhpcyBwYXJhbWV0ZXIgZm9yIGVhY2ggZGF0YXNldCBzZXBhcmF0ZWx5LCBidXQgImxhenkiIDAuNSBpcyBwcmV0dHkgZ29vZCBmb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9zdCBkYXRhc2V0cy4KICAgIDpwYXJhbSBzYW1wbGluZ19yYXRlOiAgICAgICAgICAgQ3VycmVudGx5LCBzaWxlcm8gVkFEIG1vZGVscyBzdXBwb3J0IDgwMDAgYW5kIDE2MDAwIHNhbXBsZSByYXRlcy4KICAgIDpwYXJhbSBtaW5fc3BlZWNoX2R1cmF0aW9uX21zOiAgRmluYWwgc3BlZWNoIGNodW5rcyBzaG9ydGVyIG1pbl9zcGVlY2hfZHVyYXRpb25fbXMgYXJlIHRocm93biBvdXQuCiAgICA6cGFyYW0gbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiAgIE1heGltdW0gZHVyYXRpb24gb2Ygc3BlZWNoIGNodW5rcyBpbiBzZWNvbmRzLiBDaHVua3MgbG9uZ2VyIHRoYW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYG1heF9zcGVlY2hfZHVyYXRpb25fc2Agd2lsbCBiZSBzcGxpdCBhdCB0aGUgdGltZXN0YW1wIG9mIHRoZSBsYXN0IHNpbGVuY2UgdGhhdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBsYXN0cyBtb3JlIHRoYW4gMTAwbXMgKGlmIGFueSksIHRvIHByZXZlbnQgYWdncmVzc2l2ZSBjdXR0aW5nLiBPdGhlcndpc2UsIHRoZXkgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSBzcGxpdCBhZ2dyZXNzaXZlbHkganVzdCBiZWZvcmUgbWF4X3NwZWVjaF9kdXJhdGlvbl9zLgogICAgOnBhcmFtIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zOiBJbiB0aGUgZW5kIG9mIGVhY2ggc3BlZWNoIGNodW5rIHdhaXQgZm9yIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zIGJlZm9yZSBzZXBhcmF0aW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGl0LgogICAgOnBhcmFtIHdpbmRvd19zaXplX3NhbXBsZXM6ICAgICBBdWRpbyBjaHVua3Mgb2Ygd2luZG93X3NpemVfc2FtcGxlcyBzaXplIGFyZSBmZWQgdG8gdGhlIHNpbGVybyBWQUQgbW9kZWwuCgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXQVJOSU5HISBTaWxlcm8gVkFEIG1vZGVscyB3ZXJlIHRyYWluZWQgdXNpbmcgNTEyLCAxMDI0LCAxNTM2IHNhbXBsZXMgZm9yIDE2MDAwCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSByYXRlIGFuZCAyNTYsIDUxMiwgNzY4IHNhbXBsZXMgZm9yIDgwMDAgc2FtcGxlIHJhdGUuIFZhbHVlcyBvdGhlciB0aGFuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZXNlIG1heSBhZmZlY3QgbW9kZWwgcGVyZm9ybWFuY2UhCiAgICA6cGFyYW0gc3BlZWNoX3BhZF9tczogICAgICAgICAgIEZpbmFsIHNwZWVjaCBjaHVua3MgYXJlIHBhZGRlZCBieSBzcGVlY2hfcGFkX21zIGVhY2ggc2lkZS4KICAgIDpwYXJhbSByZXR1cm5fc2Vjb25kczogICAgICAgICAgV2hldGhlciByZXR1cm4gdGltZXN0YW1wcyBpbiBzZWNvbmRzLiBGYWxzZSBtZWFucyB0byByZXR1cm4gdGltZXN0YW1wcyBpbiBzYW1wbGVzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChkZWZhdWx0IC0gRmFsc2UpLgogICAgOnBhcmFtIHBlcl9jaGFubmVsOiAgICAgICAgICAgICBXaGV0aGVyIHRvIHJldHVybiB0aW1lc3RhbXBzIHBlciBjaGFubmVsIChkZWZhdWx0IC0gRmFsc2UpLiBUaGlzIHdpbGwgcnVuIFZBRCBvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBlYWNoIGNoYW5uZWwgc2VwYXJhdGVseSBhbmQgcmV0dXJuIGEgbGlzdCBvZiB0aW1lc3RhbXBzIHBlciBjaGFubmVsLgogICAgOnBhcmFtIHVzZV9tdWx0aXByb2Nlc3Npbmc6ICAgICBUaGUgbnVtYmVyIG9mIHdvcmtlcnMgdG8gdXNlIGZvciBtdWx0aXByb2Nlc3NpbmcuIElmIDAsIG5vIG11bHRpcHJvY2Vzc2luZyB3aWxsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJlIHVzZWQuIERlZmF1bHQgaXMgMC4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICAgICAgVmVyYm9zaXR5LgogICAgIiIiCiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgR2V0IHRoZSBpbnB1dCBhdWRpbyBmaWxlcyB0byB0cmFuc2NyaWJlOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgYXVkaW8gZmlsZXMuIikKICAgIGF1ZGlvX2ZpbGVzID0gX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9ZGF0YV9wYXRoKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbihhdWRpb19maWxlcyl9IGF1ZGlvIGZpbGVzLiIpCgogICAgIyBJbml0aWFsaXplIHRoZSB0cmFuc2NyaXB0aW9uIHBpcGVsaW5lOgogICAgdmFkX2luaXRfa3dhcmdzID0gewogICAgICAgICJ1c2Vfb25ueCI6IHVzZV9vbm54LAogICAgICAgICJmb3JjZV9vbm54X2NwdSI6IGZvcmNlX29ubnhfY3B1LAogICAgICAgICJ0aHJlc2hvbGQiOiB0aHJlc2hvbGQsCiAgICAgICAgInNhbXBsaW5nX3JhdGUiOiBzYW1wbGluZ19yYXRlLAogICAgICAgICJtaW5fc3BlZWNoX2R1cmF0aW9uX21zIjogbWluX3NwZWVjaF9kdXJhdGlvbl9tcywKICAgICAgICAibWF4X3NwZWVjaF9kdXJhdGlvbl9zIjogbWF4X3NwZWVjaF9kdXJhdGlvbl9zLAogICAgICAgICJtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyI6IG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zLAogICAgICAgICJ3aW5kb3dfc2l6ZV9zYW1wbGVzIjogd2luZG93X3NpemVfc2FtcGxlcywKICAgICAgICAic3BlZWNoX3BhZF9tcyI6IHNwZWVjaF9wYWRfbXMsCiAgICAgICAgInJldHVybl9zZWNvbmRzIjogcmV0dXJuX3NlY29uZHMsCiAgICAgICAgInBlcl9jaGFubmVsIjogcGVyX2NoYW5uZWwsCiAgICB9CgogICAgIyBDcmVhdGUgdGhlIHRhc2sgY3JlYXRvcjoKICAgIHRhc2tfY3JlYXRvciA9IFRhc2tDcmVhdG9yKHRhc2tfdHlwZT1CYXNlVGFzaykKCiAgICAjIFJ1biB0aGUgdHJhbnNjcmlwdGlvbjoKICAgIGlmIHVzZV9tdWx0aXByb2Nlc3Npbmc6CiAgICAgICAgcmVzdWx0cyA9IF9wYXJhbGxlbF9ydW4oCiAgICAgICAgICAgIG5fd29ya2Vycz11c2VfbXVsdGlwcm9jZXNzaW5nLAogICAgICAgICAgICBhdWRpb19maWxlcz1hdWRpb19maWxlcywKICAgICAgICAgICAgZGVzY3JpcHRpb249IkRldGVjdGluZyB2b2ljZSIsCiAgICAgICAgICAgIHZhZF9pbml0X2t3YXJncz12YWRfaW5pdF9rd2FyZ3MsCiAgICAgICAgICAgIHRhc2tfY3JlYXRvcj10YXNrX2NyZWF0b3IsCiAgICAgICAgICAgIHZlcmJvc2U9dmVyYm9zZSwKICAgICAgICApCiAgICBlbHNlOgogICAgICAgIHJlc3VsdHMgPSBfcnVuKAogICAgICAgICAgICBhdWRpb19maWxlcz1hdWRpb19maWxlcywKICAgICAgICAgICAgZGVzY3JpcHRpb249IkRldGVjdGluZyB2b2ljZSIsCiAgICAgICAgICAgIHZhZF9pbml0X2t3YXJncz12YWRfaW5pdF9rd2FyZ3MsCiAgICAgICAgICAgIHRhc2tfY3JlYXRvcj10YXNrX2NyZWF0b3IsCiAgICAgICAgICAgIHZlcmJvc2U9dmVyYm9zZSwKICAgICAgICApCgogICAgIyBQcm9jZXNzIHRoZSByZXN1bHRzOgogICAgcmV0dXJuIF9wcm9jZXNzX3Jlc3VsdHMocmVzdWx0cz1yZXN1bHRzLCB2ZXJib3NlPXZlcmJvc2UpCgoKZGVmIGRpYXJpemUoCiAgICAjIElucHV0IC8gT3V0cHV0IGt3YXJnczoKICAgIGRhdGFfcGF0aDogc3RyIHwgUGF0aCB8IGxpc3Rbc3RyIHwgUGF0aF0sCiAgICAjIE1vZGVsIGxvYWRpbmcga3dhcmdzOgogICAgdXNlX29ubng6IGJvb2wgPSBUcnVlLAogICAgZm9yY2Vfb25ueF9jcHU6IGJvb2wgPSBUcnVlLAogICAgIyBEZXRlY3Rpb24ga3dhcmdzOgogICAgdGhyZXNob2xkOiBmbG9hdCA9IDAuNSwKICAgIHNhbXBsaW5nX3JhdGU6IGludCA9IDE2XzAwMCwKICAgIG1pbl9zcGVlY2hfZHVyYXRpb25fbXM6IGludCA9IDI1MCwKICAgIG1heF9zcGVlY2hfZHVyYXRpb25fczogZmxvYXQgPSBmbG9hdCgiaW5mIiksCiAgICBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogaW50ID0gMTAwLAogICAgd2luZG93X3NpemVfc2FtcGxlczogaW50ID0gNTEyLAogICAgc3BlZWNoX3BhZF9tczogaW50ID0gMzAsCiAgICAjIERpYXJpemF0aW9uIGt3YXJnczoKICAgIHNwZWFrZXJfbGFiZWxzOiBsaXN0W3N0cl0gPSBOb25lLAogICAgIyBPdGhlciBrd2FyZ3M6CiAgICB1c2VfbXVsdGlwcm9jZXNzaW5nOiBpbnQgPSAwLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBQZXJmb3JtIHNwZWVjaCBkaWFyaXphdGlvbiBvbiBnaXZlbiBhdWRpbyBmaWxlcyB1c2luZyB0aGUgc2lsZXJvIFZBRCBtb2RlbCAtIGh0dHBzOi8vZ2l0aHViLmNvbS9zbmFrZXJzNC9zaWxlcm8tdmFkLgogICAgVGhlIHNwZWVjaCBkaWFyaXphdGlvbiBpcyBwZXJmb3JtZWQgcGVyIGNoYW5uZWwgc28gdGhhdCBlYWNoIGNoYW5uZWwgaW4gdGhlIGF1ZGlvIGJlbG9uZyB0byBhIGRpZmZlcmVudCBzcGVha2VyLiBUaGUKICAgIGVuZCByZXN1bHQgaXMgYSBkaWN0aW9uYXJ5IHdpdGggdGhlIGZpbGUgbmFtZXMgYXMga2V5cyBhbmQgdGhlaXIgZGlhcml6YXRpb24gYXMgdmFsdWUuIEEgZGlhcml6YXRpb24gaXMgYSBsaXN0CiAgICBvZiB0dXBsZXM6IChzdGFydCwgZW5kLCBzcGVha2VyX2xhYmVsKS4KCiAgICBGb3IgZXhhbXBsZTo6CgogICAgICAgIHsKICAgICAgICAgICAgImZpbGVfMS53YXYiOiBbCiAgICAgICAgICAgICAgICAoMC4wLCAxLjAsICJzcGVha2VyXzAiKSwKICAgICAgICAgICAgICAgICgxLjAsIDIuMCwgInNwZWFrZXJfMSIpLAogICAgICAgICAgICAgICAgKDIuMCwgMy4wLCAic3BlYWtlcl8wIiksCiAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgXSwKICAgICAgICAgICAgImZpbGVfMi53YXYiOiBbCiAgICAgICAgICAgICAgICAoMC4wLCAxLjAsICJzcGVha2VyXzAiKSwKICAgICAgICAgICAgICAgICgxLjAsIDIuMCwgInNwZWFrZXJfMSIpLAogICAgICAgICAgICAgICAgKDIuMCwgMy4wLCAic3BlYWtlcl8wIiksCiAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgXSwKICAgICAgICAgICAgLi4uCiAgICAgICAgfQoKCiAgICA6cGFyYW0gZGF0YV9wYXRoOiAgICAgICAgICAgICAgIFRoZSBwYXRoIHRvIHRoZSBhdWRpbyBmaWxlcyB0byBkaWFyaXplLiBDYW4gYmUgYSBwYXRoIHRvIGEgc2luZ2xlIGZpbGUsIGEgcGF0aCB0byBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRpcmVjdG9yeSBvciBhIGxpc3Qgb2YgcGF0aHMgdG8gZmlsZXMuCiAgICA6cGFyYW0gdXNlX29ubng6ICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gdXNlIE9OTlggZm9yIGluZmVyZW5jZS4gRGVmYXVsdCBpcyBUcnVlLgogICAgOnBhcmFtIGZvcmNlX29ubnhfY3B1OiAgICAgICAgICBXaGV0aGVyIHRvIGZvcmNlIE9OTlggdG8gdXNlIENQVSBmb3IgaW5mZXJlbmNlLiBEZWZhdWx0IGlzIFRydWUuCiAgICA6cGFyYW0gdGhyZXNob2xkOiAgICAgICAgICAgICAgIFNwZWVjaCB0aHJlc2hvbGQuIFNpbGVybyBWQUQgb3V0cHV0cyBzcGVlY2ggcHJvYmFiaWxpdGllcyBmb3IgZWFjaCBhdWRpbyBjaHVuaywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcHJvYmFiaWxpdGllcyBBQk9WRSB0aGlzIHZhbHVlIGFyZSBjb25zaWRlcmVkIGFzIFNQRUVDSC4gSXQgaXMgYmV0dGVyIHRvIHR1bmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhpcyBwYXJhbWV0ZXIgZm9yIGVhY2ggZGF0YXNldCBzZXBhcmF0ZWx5LCBidXQgImxhenkiIDAuNSBpcyBwcmV0dHkgZ29vZCBmb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9zdCBkYXRhc2V0cy4KICAgIDpwYXJhbSBzYW1wbGluZ19yYXRlOiAgICAgICAgICAgQ3VycmVudGx5LCBzaWxlcm8gVkFEIG1vZGVscyBzdXBwb3J0IDgwMDAgYW5kIDE2MDAwIHNhbXBsZSByYXRlcy4KICAgIDpwYXJhbSBtaW5fc3BlZWNoX2R1cmF0aW9uX21zOiAgRmluYWwgc3BlZWNoIGNodW5rcyBzaG9ydGVyIG1pbl9zcGVlY2hfZHVyYXRpb25fbXMgYXJlIHRocm93biBvdXQuCiAgICA6cGFyYW0gbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiAgIE1heGltdW0gZHVyYXRpb24gb2Ygc3BlZWNoIGNodW5rcyBpbiBzZWNvbmRzLiBDaHVua3MgbG9uZ2VyIHRoYW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYG1heF9zcGVlY2hfZHVyYXRpb25fc2Agd2lsbCBiZSBzcGxpdCBhdCB0aGUgdGltZXN0YW1wIG9mIHRoZSBsYXN0IHNpbGVuY2UgdGhhdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBsYXN0cyBtb3JlIHRoYW4gMTAwbXMgKGlmIGFueSksIHRvIHByZXZlbnQgYWdncmVzc2l2ZSBjdXR0aW5nLiBPdGhlcndpc2UsIHRoZXkgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSBzcGxpdCBhZ2dyZXNzaXZlbHkganVzdCBiZWZvcmUgbWF4X3NwZWVjaF9kdXJhdGlvbl9zLgogICAgOnBhcmFtIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zOiBJbiB0aGUgZW5kIG9mIGVhY2ggc3BlZWNoIGNodW5rIHdhaXQgZm9yIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zIGJlZm9yZSBzZXBhcmF0aW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGl0LgogICAgOnBhcmFtIHdpbmRvd19zaXplX3NhbXBsZXM6ICAgICBBdWRpbyBjaHVua3Mgb2Ygd2luZG93X3NpemVfc2FtcGxlcyBzaXplIGFyZSBmZWQgdG8gdGhlIHNpbGVybyBWQUQgbW9kZWwuCgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXQVJOSU5HISBTaWxlcm8gVkFEIG1vZGVscyB3ZXJlIHRyYWluZWQgdXNpbmcgNTEyLCAxMDI0LCAxNTM2IHNhbXBsZXMgZm9yIDE2MDAwCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSByYXRlIGFuZCAyNTYsIDUxMiwgNzY4IHNhbXBsZXMgZm9yIDgwMDAgc2FtcGxlIHJhdGUuIFZhbHVlcyBvdGhlciB0aGFuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZXNlIG1heSBhZmZlY3QgbW9kZWwgcGVyZm9ybWFuY2UhCiAgICA6cGFyYW0gc3BlZWNoX3BhZF9tczogICAgICAgICAgIEZpbmFsIHNwZWVjaCBjaHVua3MgYXJlIHBhZGRlZCBieSBzcGVlY2hfcGFkX21zIGVhY2ggc2lkZS4KICAgIDpwYXJhbSBzcGVha2VyX2xhYmVsczogICAgICAgICAgVGhlIHNwZWFrZXIgbGFiZWxzIHRvIHVzZSBmb3IgdGhlIGRpYXJpemF0aW9uLiBJZiBub3QgZ2l2ZW4sIHRoZSBzcGVha2VycyB3aWxsIGJlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG5hbWVkICJzcGVha2VyXzAiLCAic3BlYWtlcl8xIiwgZXRjLgogICAgOnBhcmFtIHVzZV9tdWx0aXByb2Nlc3Npbmc6ICAgICBUaGUgbnVtYmVyIG9mIHdvcmtlcnMgdG8gdXNlIGZvciBtdWx0aXByb2Nlc3NpbmcuIElmIDAsIG5vIG11bHRpcHJvY2Vzc2luZyB3aWxsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJlIHVzZWQuIERlZmF1bHQgaXMgMC4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICAgICAgVmVyYm9zaXR5LgogICAgIiIiCiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgR2V0IHRoZSBpbnB1dCBhdWRpbyBmaWxlcyB0byB0cmFuc2NyaWJlOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgYXVkaW8gZmlsZXMuIikKICAgIGF1ZGlvX2ZpbGVzID0gX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9ZGF0YV9wYXRoKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbihhdWRpb19maWxlcyl9IGF1ZGlvIGZpbGVzLiIpCgogICAgIyBJbml0aWFsaXplIHRoZSB0cmFuc2NyaXB0aW9uIHBpcGVsaW5lOgogICAgdmFkX2luaXRfa3dhcmdzID0gewogICAgICAgICJ1c2Vfb25ueCI6IHVzZV9vbm54LAogICAgICAgICJmb3JjZV9vbm54X2NwdSI6IGZvcmNlX29ubnhfY3B1LAogICAgICAgICJ0aHJlc2hvbGQiOiB0aHJlc2hvbGQsCiAgICAgICAgInNhbXBsaW5nX3JhdGUiOiBzYW1wbGluZ19yYXRlLAogICAgICAgICJtaW5fc3BlZWNoX2R1cmF0aW9uX21zIjogbWluX3NwZWVjaF9kdXJhdGlvbl9tcywKICAgICAgICAibWF4X3NwZWVjaF9kdXJhdGlvbl9zIjogbWF4X3NwZWVjaF9kdXJhdGlvbl9zLAogICAgICAgICJtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyI6IG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zLAogICAgICAgICJ3aW5kb3dfc2l6ZV9zYW1wbGVzIjogd2luZG93X3NpemVfc2FtcGxlcywKICAgICAgICAic3BlZWNoX3BhZF9tcyI6IHNwZWVjaF9wYWRfbXMsCiAgICAgICAgInJldHVybl9zZWNvbmRzIjogVHJ1ZSwKICAgICAgICAicGVyX2NoYW5uZWwiOiBUcnVlLAogICAgfQoKICAgICMgQ3JlYXRlIHRoZSB0YXNrIGNyZWF0b3I6CiAgICB0YXNrX2NyZWF0b3IgPSBUYXNrQ3JlYXRvcigKICAgICAgICB0YXNrX3R5cGU9U3BlZWNoRGlhcml6YXRpb25UYXNrLCB0YXNrX2t3YXJncz17InNwZWFrZXJfbGFiZWxzIjogc3BlYWtlcl9sYWJlbHN9CiAgICApCgogICAgIyBSdW4gdGhlIHRyYW5zY3JpcHRpb246CiAgICBpZiB1c2VfbXVsdGlwcm9jZXNzaW5nOgogICAgICAgIHJlc3VsdHMgPSBfcGFyYWxsZWxfcnVuKAogICAgICAgICAgICBuX3dvcmtlcnM9dXNlX211bHRpcHJvY2Vzc2luZywKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEaWFyaXppbmciLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXN1bHRzID0gX3J1bigKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEaWFyaXppbmciLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQoKICAgICMgUHJvY2VzcyB0aGUgcmVzdWx0czoKICAgIHJldHVybiBfcHJvY2Vzc19yZXN1bHRzKHJlc3VsdHM9cmVzdWx0cywgdmVyYm9zZT12ZXJib3NlKQoKCmRlZiBfZ2V0X2F1ZGlvX2ZpbGVzKAogICAgZGF0YV9wYXRoOiBQYXRoIHwgc3RyIHwgbGlzdCwKKSAtPiBsaXN0W1BhdGhdOgogICAgIiIiCiAgICBHZXQgdGhlIGF1ZGlvIGZpbGVzIGZyb20gdGhlIGRhdGEgcGF0aC4gSWYgYSBwYXRoIHRvIGEgZGlyZWN0b3J5IGlzIGdpdmVuLCBhbGwgZmlsZXMgaW4gdGhlIGRpcmVjdG9yeSB3aWxsIGJlCiAgICBjb2xsZWN0ZWQuCgogICAgOnBhcmFtIGRhdGFfcGF0aDogVGhlIGRhdGEgcGF0aCB0byBjb2xsZWN0IHRoZSBhdWRpbyBmaWxlcyBmcm9tLgoKICAgIDpyZXR1cm5zOiBUaGUgYXVkaW8gZmlsZXMgbGlzdC4KICAgICIiIgogICAgIyBDaGVjayBpZiBnaXZlbiBhIGxpc3Qgb2YgcGF0aHM6CiAgICBpZiBpc2luc3RhbmNlKGRhdGFfcGF0aCwgbGlzdCk6CiAgICAgICAgYXVkaW9fZmlsZXMgPSBbXQogICAgICAgIGZvciBwYXRoIGluIGRhdGFfcGF0aDoKICAgICAgICAgICAgYXVkaW9fZmlsZXMuZXh0ZW5kKF9nZXRfYXVkaW9fZmlsZXMoZGF0YV9wYXRoPXBhdGgpKQogICAgICAgIHJldHVybiBhdWRpb19maWxlcwoKICAgICMgQ2hlY2sgaWYgZ2l2ZW4gYSBzaW5nbGUgc3RyaW5nIHBhdGggdG8gY2FzdCBpdCB0byBhIGBwYXRobGliLlBhdGhgOgogICAgaWYgaXNpbnN0YW5jZShkYXRhX3BhdGgsIHN0cik6CiAgICAgICAgZGF0YV9wYXRoID0gUGF0aChkYXRhX3BhdGgpLmFic29sdXRlKCkKCiAgICAjIENoZWNrIGlmIHRoZSBwYXRoIGlzIG9mIGEgZGlyZWN0b3J5IG9yIGEgZmlsZToKICAgIGlmIGRhdGFfcGF0aC5pc19kaXIoKToKICAgICAgICAjIEdldCBhbGwgZmlsZXMgaW5zaWRlIHRoZSBkaXJlY3Rvcnk6CiAgICAgICAgYXVkaW9fZmlsZXMgPSBsaXN0KGRhdGFfcGF0aC5nbG9iKCIqLioiKSkKICAgIGVsaWYgZGF0YV9wYXRoLmlzX2ZpbGUoKToKICAgICAgICBhdWRpb19maWxlcyA9IFtkYXRhX3BhdGhdCiAgICBlbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgIGYiVW5yZWNvZ25pemVkIGRhdGEgcGF0aC4gVGhlIHBhcmFtZXRlciBgZGF0YV9wYXRoYCBtdXN0IGJlIGEgdmFsaWQgcGF0aCB0byBlaXRoZXIgYSBkaXJlY3RvcnkgcGF0aCBvciBhICIKICAgICAgICAgICAgZiJmaWxlLiBHaXZlbjoge3N0cihkYXRhX3BhdGgpfSAiCiAgICAgICAgKQoKICAgIHJldHVybiBhdWRpb19maWxlcwoKCmRlZiBfcnVuKAogICAgYXVkaW9fZmlsZXM6IGxpc3RbUGF0aF0sCiAgICBkZXNjcmlwdGlvbjogc3RyLAogICAgdmFkX2luaXRfa3dhcmdzOiBkaWN0LAogICAgdGFza19jcmVhdG9yOiBUYXNrQ3JlYXRvciwKICAgIHZlcmJvc2U6IGJvb2wsCikgLT4gbGlzdFt0dXBsZVtib29sLCB0dXBsZVtzdHIsIGxpc3RdXV06CiAgICAiIiIKICAgIExvYWQgYSBWQUQgYW5kIHVzZSBpdCB0byBjb21wbGV0ZSB0aGUgdGFza3MgdGhhdCB3aWxsIGJlIGNyZWF0ZWQgb24gdGhlIHByb3ZpZGVkIGZpbGVzIHVzaW5nIHRoZSBnaXZlbiB0YXNrIGNyZWF0b3IuCgogICAgOnBhcmFtIGF1ZGlvX2ZpbGVzOiAgICAgVGhlIGF1ZGlvIGZpbGVzIHRvIHVzZS4KICAgIDpwYXJhbSBkZXNjcmlwdGlvbjogICAgIFRoZSBkZXNjcmlwdGlvbiB0byB1c2UgZm9yIHRoZSBwcm9ncmVzcyBiYXIuCiAgICA6cGFyYW0gdmFkX2luaXRfa3dhcmdzOiBUaGUgVkFEIGluaXRpYWxpemF0aW9uIGtleXdvcmQgYXJndW1lbnRzLgogICAgOnBhcmFtIHRhc2tfY3JlYXRvcjogICAgVGhlIHRhc2sgY3JlYXRvciB0byB1c2UgdG8gY3JlYXRlIHRoZSB0YXNrcy4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgIFZlcmJvc2l0eS4KCiAgICA6cmV0dXJuczogVGhlIGNvbGxlY3RlZCByZXN1bHRzLgogICAgIiIiCiAgICAjIExvYWQgdGhlIFZBRDoKICAgIHZhZCA9IFZvaWNlQWN0aXZpdHlEZXRlY3RvcigqKnZhZF9pbml0X2t3YXJncykKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJMb2FkaW5nIHRoZSBWQUQgbW9kZWwuIikKICAgIHZhZC5sb2FkKCkKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJWQUQgbW9kZWwgbG9hZGVkLiIpCgogICAgIyBSdW4gdGhlIFZBRCBvbiB0aGUgYXVkaW8gZmlsZXMgYW5kIGNvbGxlY3QgdGhlIHJlc3VsdHM6CiAgICByZXN1bHRzID0gW10KICAgIGZvciBhdWRpb19maWxlIGluIHRxZG0oCiAgICAgICAgYXVkaW9fZmlsZXMsCiAgICAgICAgZGVzYz1kZXNjcmlwdGlvbiwKICAgICAgICB1bml0PSJmaWxlIiwKICAgICAgICB0b3RhbD1sZW4oYXVkaW9fZmlsZXMpLAogICAgICAgIGRpc2FibGU9bm90IHZlcmJvc2UsCiAgICApOgogICAgICAgIHRyeToKICAgICAgICAgICAgIyBDcmVhdGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2sgPSB0YXNrX2NyZWF0b3IuY3JlYXRlX3Rhc2soYXVkaW9fZmlsZT1hdWRpb19maWxlKQogICAgICAgICAgICAjIFJ1biB0aGUgZmlsZSB0aHJvdWdoIHRoZSBWQUQ6CiAgICAgICAgICAgIHNwZWVjaF90aW1lc3RhbXBzID0gdmFkLmRldGVjdF92b2ljZShhdWRpb19maWxlPWF1ZGlvX2ZpbGUpCiAgICAgICAgICAgICMgQ29tcGxldGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2suZG9fdGFzayhzcGVlY2hfdGltZXN0YW1wcz1zcGVlY2hfdGltZXN0YW1wcykKICAgICAgICAgICAgIyBDb2xsZWN0IHRoZSByZXN1bHQ6CiAgICAgICAgICAgIHJlc3VsdHMuYXBwZW5kKChGYWxzZSwgdGFzay5nZXRfcmVzdWx0KCkpKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIENvbGxlY3QgdGhlIGVycm9yOgogICAgICAgICAgICByZXN1bHRzLmFwcGVuZCgoVHJ1ZSwgKGF1ZGlvX2ZpbGUubmFtZSwgc3RyKGV4Y2VwdGlvbikpKSkKCiAgICByZXR1cm4gcmVzdWx0cwoKCmRlZiBfcGFyYWxsZWxfcnVuKAogICAgbl93b3JrZXJzOiBpbnQsCiAgICBhdWRpb19maWxlczogbGlzdFtQYXRoXSwKICAgIGRlc2NyaXB0aW9uOiBzdHIsCiAgICB2YWRfaW5pdF9rd2FyZ3M6IGRpY3QsCiAgICB0YXNrX2NyZWF0b3I6IFRhc2tDcmVhdG9yLAogICAgdmVyYm9zZTogYm9vbCwKKSAtPiBsaXN0W3R1cGxlW2Jvb2wsIHR1cGxlW3N0ciwgbGlzdF1dXToKICAgICIiIgogICAgUnVuIG11bHRpcGxlIFZBRCB3b3JrZXJzIHdpdGggbXVsdGlwcm9jZXNzaW5nIHRvIGNvbXBsZXRlIHRoZSB0YXNrcyB0aGF0IHdpbGwgYmUgY3JlYXRlZCBvbiB0aGUgcHJvdmlkZWQgZmlsZXMgdXNpbmcKICAgIHRoZSBnaXZlbiB0YXNrIGNyZWF0b3IuCgogICAgOnBhcmFtIG5fd29ya2VyczogICAgICAgVGhlIG51bWJlciBvZiB3b3JrZXJzIHRvIHVzZS4KICAgIDpwYXJhbSBhdWRpb19maWxlczogICAgIFRoZSBhdWRpbyBmaWxlcyB0byB1c2UuCiAgICA6cGFyYW0gZGVzY3JpcHRpb246ICAgICBUaGUgZGVzY3JpcHRpb24gdG8gdXNlIGZvciB0aGUgcHJvZ3Jlc3MgYmFyLgogICAgOnBhcmFtIHZhZF9pbml0X2t3YXJnczogVGhlIFZBRCBpbml0aWFsaXphdGlvbiBrZXl3b3JkIGFyZ3VtZW50cy4KICAgIDpwYXJhbSB0YXNrX2NyZWF0b3I6ICAgIFRoZSB0YXNrIGNyZWF0b3IgdG8gdXNlIHRvIGNyZWF0ZSB0aGUgdGFza3MuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICBWZXJib3NpdHkuCgogICAgOnJldHVybnM6IFRoZSBjb2xsZWN0ZWQgcmVzdWx0cy4KICAgICIiIgogICAgIyBMb2FkIHRoZSBWQUQgKGRvd25sb2FkIG9uY2UsIGFuZCBpdCB3aWxsIGJlIGxvYWRlZCB0aGVuIHBlciBwcm9jZXNzIGxhdGVyIG9uKToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJMb2FkaW5nIHRoZSBWQUQgbW9kZWwuIikKICAgIHZhZCA9IFZvaWNlQWN0aXZpdHlEZXRlY3RvcigqKnZhZF9pbml0X2t3YXJncykKICAgIHZhZC5sb2FkKCkKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJWQUQgbW9kZWwgbG9hZGVkLiIpCgogICAgIyBDaGVjayB0aGUgbnVtYmVyIG9mIHdvcmtlcnM6CiAgICBpZiBuX3dvcmtlcnMgPiBsZW4oYXVkaW9fZmlsZXMpOgogICAgICAgIF9MT0dHRVIud2FybmluZygKICAgICAgICAgICAgZiJUaGUgbnVtYmVyIG9mIHdvcmtlcnMgKHtuX3dvcmtlcnN9KSBpcyBsYXJnZXIgdGhhbiB0aGUgbnVtYmVyIG9mIGF1ZGlvIGZpbGVzICh7bGVuKGF1ZGlvX2ZpbGVzKX0pLiAiCiAgICAgICAgICAgIGYiU2V0dGluZyB0aGUgbnVtYmVyIG9mIHdvcmtlcnMgdG8ge2xlbihhdWRpb19maWxlcyl9LiIKICAgICAgICApCiAgICAgICAgbl93b3JrZXJzID0gbGVuKGF1ZGlvX2ZpbGVzKQoKICAgICMgSW5pdGlhbGl6ZSB0aGUgbXVsdGlwcm9jZXNzaW5nIHF1ZXVlczoKICAgIHRhc2tzX3F1ZXVlID0gUXVldWUoKQogICAgcmVzdWx0c19xdWV1ZSA9IFF1ZXVlKCkKCiAgICAjIEluaXRpYWxpemUgdGhlIG11bHRpcHJvY2Vzc2luZyBwcm9jZXNzZXM6CiAgICB0YXNrX2NvbXBsZXRpb25fcHJvY2Vzc2VzID0gWwogICAgICAgIFByb2Nlc3MoCiAgICAgICAgICAgIHRhcmdldD1fbXVsdGlwcm9jZXNzaW5nX2NvbXBsZXRlX3Rhc2tzLAogICAgICAgICAgICBrd2FyZ3M9ewogICAgICAgICAgICAgICAgInZhZF9pbml0X2t3YXJncyI6IHZhZF9pbml0X2t3YXJncywKICAgICAgICAgICAgICAgICJ0YXNrc19xdWV1ZSI6IHRhc2tzX3F1ZXVlLAogICAgICAgICAgICAgICAgInJlc3VsdHNfcXVldWUiOiByZXN1bHRzX3F1ZXVlLAogICAgICAgICAgICB9LAogICAgICAgICkKICAgICAgICBmb3IgXyBpbiByYW5nZShuX3dvcmtlcnMpCiAgICBdCgogICAgIyBTdGFydCB0aGUgbXVsdGlwcm9jZXNzaW5nIHByb2Nlc3NlczoKICAgIGZvciBwIGluIHRhc2tfY29tcGxldGlvbl9wcm9jZXNzZXM6CiAgICAgICAgcC5zdGFydCgpCgogICAgIyBQdXQgdGhlIHRhc2tzIGluIHRoZSBxdWV1ZToKICAgIGZvciBhdWRpb19maWxlIGluIGF1ZGlvX2ZpbGVzOgogICAgICAgIHRhc2tzX3F1ZXVlLnB1dCh0YXNrX2NyZWF0b3IuY3JlYXRlX3Rhc2soYXVkaW9fZmlsZT1hdWRpb19maWxlKS50b190dXBsZSgpKQoKICAgICMgUHV0IHRoZSBzdG9wIG1hcmtzIGluIHRoZSBxdWV1ZToKICAgIGZvciBfIGluIHJhbmdlKG5fd29ya2Vycyk6CiAgICAgICAgdGFza3NfcXVldWUucHV0KF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLKQoKICAgICMgQ29sbGVjdCB0aGUgcmVzdWx0czoKICAgIHJlc3VsdHMgPSBbXQogICAgc3RvcF9tYXJrc19jb3VudGVyID0gMAogICAgd2l0aCB0cWRtKAogICAgICAgIGRlc2M9ZGVzY3JpcHRpb24sCiAgICAgICAgdW5pdD0iZmlsZSIsCiAgICAgICAgdG90YWw9bGVuKGF1ZGlvX2ZpbGVzKSwKICAgICAgICBkaXNhYmxlPW5vdCB2ZXJib3NlLAogICAgKSBhcyBwcm9ncmVzc2JhcjoKICAgICAgICB3aGlsZSBUcnVlOgogICAgICAgICAgICAjIEdldCBhIHJlc3VsdCBmcm9tIHRoZSBxdWV1ZToKICAgICAgICAgICAgcmVzdWx0OiB0dXBsZVtib29sLCB0dXBsZVtzdHIsIGxpc3RdXSA9IHJlc3VsdHNfcXVldWUuZ2V0KCkKICAgICAgICAgICAgaWYgcmVzdWx0ID09IF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLOgogICAgICAgICAgICAgICAgc3RvcF9tYXJrc19jb3VudGVyICs9IDEKICAgICAgICAgICAgICAgIGlmIHN0b3BfbWFya3NfY291bnRlciA9PSBuX3dvcmtlcnM6CiAgICAgICAgICAgICAgICAgICAgYnJlYWsKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgICMgQ29sbGVjdCB0aGUgcmVzdWx0OgogICAgICAgICAgICAgICAgcmVzdWx0cy5hcHBlbmQocmVzdWx0KQogICAgICAgICAgICAgICAgcHJvZ3Jlc3NiYXIudXBkYXRlKDEpCgogICAgIyBXYWl0IGZvciB0aGUgcHJvY2Vzc2VzIHRvIGZpbmlzaDoKICAgIGZvciBwIGluIHRhc2tfY29tcGxldGlvbl9wcm9jZXNzZXM6CiAgICAgICAgcC5qb2luKCkKCiAgICByZXR1cm4gcmVzdWx0cwoKCmRlZiBfcHJvY2Vzc19yZXN1bHRzKAogICAgcmVzdWx0czogbGlzdFt0dXBsZVtib29sLCB0dXBsZVtzdHIsIGxpc3RdXV0sIHZlcmJvc2U6IGJvb2wKKSAtPiB0dXBsZVtkaWN0LCBkaWN0XToKICAgICIiIgogICAgUHJvY2VzcyB0aGUgcmVzdWx0cyBvZiB0aGUgdGFza3MuCgogICAgOnBhcmFtIHJlc3VsdHM6IFRoZSByZXN1bHRzIHRvIHByb2Nlc3MuCiAgICA6cGFyYW0gdmVyYm9zZTogVmVyYm9zaXR5LgoKICAgIDpyZXR1cm5zOiBUaGUgcHJvY2Vzc2VkIHJlc3VsdHMgYXMgYSB0dXBsZSBvZiBzdWNjZXNzZXMgYW5kIGVycm9ycy4KICAgICIiIgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIlN1bW1hcml6aW5nIHRoZSByZXN1bHRzLiIpCiAgICBzdWNjZXNzZXMgPSB7fQogICAgZXJyb3JzID0ge30KICAgIGZvciBpc19lcnJvciwgcmVzdWx0IGluIHJlc3VsdHM6CiAgICAgICAgaWYgaXNfZXJyb3I6CiAgICAgICAgICAgIGVycm9yc1tyZXN1bHRbMF1dID0gcmVzdWx0WzFdCiAgICAgICAgZWxzZToKICAgICAgICAgICAgc3VjY2Vzc2VzW3Jlc3VsdFswXV0gPSByZXN1bHRbMV0KICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiRG9uZSAoe2xlbihzdWNjZXNzZXMpfS97bGVuKHN1Y2Nlc3NlcykgKyBsZW4oZXJyb3JzKX0pXG4iKQoKICAgIHJldHVybiBzdWNjZXNzZXMsIGVycm9ycwo= requirements: - torch - torchaudio - tqdm - onnxruntime - functionSourceCode: IyBDb3B5cmlnaHQgMjAyNCBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgbG9nZ2luZwpmcm9tIG11bHRpcHJvY2Vzc2luZyBpbXBvcnQgUHJvY2VzcywgUXVldWUKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwZXMgaW1wb3J0IEZ1bmN0aW9uVHlwZQpmcm9tIHR5cGluZyBpbXBvcnQgRGljdCwgTGlzdCwgVHVwbGUsIFR5cGUsIFVuaW9uCgppbXBvcnQgdG9yY2gKaW1wb3J0IHRvcmNoYXVkaW8KZnJvbSB0cWRtIGltcG9ydCB0cWRtCgoKY2xhc3MgQmFzZVRhc2s6CiAgICAiIiIKICAgIEEgYmFzZSBjbGFzcyBmb3IgYSB0YXNrIHRvIGNvbXBsZXRlIGFmdGVyIFZBRC4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXyhzZWxmLCBhdWRpb19maWxlOiBQYXRoKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSBiYXNlIHRhc2suCgogICAgICAgIDpwYXJhbSBhdWRpb19maWxlOiBUaGUgYXVkaW8gZmlsZSBhc3NpZ25lZCB0byB0aGUgdGFzay4KICAgICAgICAiIiIKICAgICAgICAjIFN0b3JlIHRoZSBhdWRpbyBmaWxlOgogICAgICAgIHNlbGYuX2F1ZGlvX2ZpbGUgPSBhdWRpb19maWxlCgogICAgICAgICMgUHJlcGFyZSB0aGUgcmVzdWx0OgogICAgICAgIHNlbGYuX3Jlc3VsdCA9IE5vbmUKCiAgICBAcHJvcGVydHkKICAgIGRlZiBhdWRpb19maWxlKHNlbGYpIC0+IFBhdGg6CiAgICAgICAgIiIiCiAgICAgICAgR2V0IHRoZSBhdWRpbyBmaWxlIG9mIHRoZSB0YXNrLgoKICAgICAgICA6cmV0dXJuczogVGhlIGF1ZGlvIGZpbGUgb2YgdGhlIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX2F1ZGlvX2ZpbGUKCiAgICBkZWYgZG9fdGFzaygKICAgICAgICBzZWxmLCBzcGVlY2hfdGltZXN0YW1wczogVW5pb25bTGlzdFtEaWN0W3N0ciwgaW50XV0sIExpc3RbTGlzdFtEaWN0W3N0ciwgaW50XV1dXQogICAgKToKICAgICAgICAiIiIKICAgICAgICBEbyB0aGUgdGFzayBvbiB0aGUgZ2l2ZW4gc3BlZWNoIHRpbWVzdGFtcHMuIFRoZSBiYXNlIHRhc2sgd2lsbCBzaW1wbHkgc2F2ZSB0aGUgc3BlZWNoIHRpbWVzdGFtcHMgYXMgdGhlIHJlc3VsdC4KCiAgICAgICAgOnBhcmFtIHNwZWVjaF90aW1lc3RhbXBzOiBUaGUgc3BlZWNoIHRpbWVzdGFtcHMgdG8gZG8gdGhlIHRhc2sgb24gYXMgb3V0cHV0dGVkIGZyb20gdGhlIFZBRC4KICAgICAgICAiIiIKICAgICAgICBzZWxmLl9yZXN1bHQgPSBzcGVlY2hfdGltZXN0YW1wcwoKICAgIGRlZiBnZXRfcmVzdWx0KHNlbGYpIC0+IFR1cGxlW3N0ciwgbGlzdF06CiAgICAgICAgIiIiCiAgICAgICAgR2V0IHRoZSByZXN1bHQgb2YgdGhlIHRhc2suIEEgdHVwbGUgb2YgdGhlIGF1ZGlvIGZpbGUgbmFtZSBhbmQgdGhlIHJlc3VsdC4KCiAgICAgICAgOnJldHVybnM6IFRoZSByZXN1bHQgb2YgdGhlIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX2F1ZGlvX2ZpbGUubmFtZSwgc2VsZi5fcmVzdWx0CgogICAgZGVmIHRvX3R1cGxlKHNlbGYpIC0+IFR1cGxlW3N0ciwgZGljdF06CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCB0aGUgdGFzayB0byBhIHR1cGxlIHRvIHJlY29uc3RydWN0IGl0IGxhdGVyICh1c2VkIGZvciBtdWx0aXByb2Nlc3NpbmcgdG8gcGFzcyBpbiBxdWV1ZSkuCgogICAgICAgIDpyZXR1cm5zOiBUaGUgY29udmVydGVkIHRhc2suCiAgICAgICAgIiIiCiAgICAgICAgcmV0dXJuIHNlbGYuX19jbGFzc19fLl9fbmFtZV9fLCB7ImF1ZGlvX2ZpbGUiOiBzZWxmLl9hdWRpb19maWxlfQoKCmNsYXNzIFNwZWVjaERpYXJpemF0aW9uVGFzayhCYXNlVGFzayk6CiAgICAiIiIKICAgIEEgc3BlZWNoIGRpYXJpemF0aW9uIHRhc2suIFRoZSB0YXNrIHdpbGwgZGlhcml6ZSB0aGUgVkFEIHNwZWVjaCB0aW1lc3RhbXBzIGludG8gc3BlYWtlcnMuCiAgICAiIiIKCiAgICBkZWYgX19pbml0X18oc2VsZiwgYXVkaW9fZmlsZTogUGF0aCwgc3BlYWtlcl9sYWJlbHM6IExpc3Rbc3RyXSk6CiAgICAgICAgIiIiCiAgICAgICAgSW5pdGlhbGl6ZSB0aGUgc3BlZWNoIGRpYXJpemF0aW9uIHRhc2suCgogICAgICAgIDpwYXJhbSBhdWRpb19maWxlOiAgICAgVGhlIGF1ZGlvIGZpbGUgYXNzaWduZWQgdG8gdGhlIHRhc2suCiAgICAgICAgOnBhcmFtIHNwZWFrZXJfbGFiZWxzOiBUaGUgc3BlYWtlciBsYWJlbHMgdG8gdXNlIGZvciB0aGUgZGlhcml6YXRpb24uIElmIG5vdCBnaXZlbiwgdGhlIHNwZWFrZXJzIHdpbGwgYmUgbmFtZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJzcGVha2VyXzAiLCAic3BlYWtlcl8xIiwgZXRjLgogICAgICAgICIiIgogICAgICAgIHN1cGVyKCkuX19pbml0X18oYXVkaW9fZmlsZT1hdWRpb19maWxlKQogICAgICAgIHNlbGYuX3NwZWFrZXJfbGFiZWxzID0gc3BlYWtlcl9sYWJlbHMKCiAgICBkZWYgZG9fdGFzayhzZWxmLCBzcGVlY2hfdGltZXN0YW1wczogTGlzdFtMaXN0W0RpY3Rbc3RyLCBpbnRdXV0pOgogICAgICAgICIiIgogICAgICAgIERvIHRoZSB0YXNrIG9uIHRoZSBnaXZlbiBzcGVlY2ggdGltZXN0YW1wcy4gVGhlIHRhc2sgd2lsbCBkaWFyaXplIHRoZSBWQUQgc3BlZWNoIHRpbWVzdGFtcHMgaW50byBzcGVha2Vycy4KCiAgICAgICAgOnBhcmFtIHNwZWVjaF90aW1lc3RhbXBzOiBUaGUgc3BlZWNoIHRpbWVzdGFtcHMgcGVyIGNoYW5uZWwgdG8gZG8gdGhlIHRhc2sgb24gYXMgb3V0cHV0dGVkIGZyb20gdGhlIFZBRC4KICAgICAgICAiIiIKICAgICAgICAjIEdldCB0aGUgc3BlYWtlciBsYWJlbHMgKHNldCBkZWZhdWx0IGlmIG5vdCBnaXZlbik6CiAgICAgICAgc3BlYWtlcl9sYWJlbHMgPSBzZWxmLl9zcGVha2VyX2xhYmVscyBvciBbCiAgICAgICAgICAgIGYic3BlYWtlcl97aX0iIGZvciBpIGluIHJhbmdlKGxlbihzcGVlY2hfdGltZXN0YW1wcykpCiAgICAgICAgXQoKICAgICAgICAjIERpYXJpemUgLSBvcmdhbml6ZSB0aGUgc3BlZWNoIHRpbWVzdGFtcHMgaW50byBhIHNpbmdsZSBsaXN0IG9mIHNwZWFrZXJzIGFuZCBzb3J0IGl0IGJ5IHN0YXJ0IHRpbWU6CiAgICAgICAgc3BlZWNoX2RpYXJpemF0aW9uID0gWwogICAgICAgICAgICAoc3BlZWNoX3RpbWVzdGFtcFsic3RhcnQiXSwgc3BlZWNoX3RpbWVzdGFtcFsiZW5kIl0sIHNwZWFrZXJfbGFiZWwpCiAgICAgICAgICAgIGZvciBzcGVha2VyX2xhYmVsLCBjaGFubmVsX3NwZWVjaF90aW1lc3RhbXBzIGluIHppcCgKICAgICAgICAgICAgICAgIHNwZWFrZXJfbGFiZWxzLCBzcGVlY2hfdGltZXN0YW1wcwogICAgICAgICAgICApCiAgICAgICAgICAgIGZvciBzcGVlY2hfdGltZXN0YW1wIGluIGNoYW5uZWxfc3BlZWNoX3RpbWVzdGFtcHMKICAgICAgICBdCiAgICAgICAgc3BlZWNoX2RpYXJpemF0aW9uLnNvcnQoKQogICAgICAgIHNlbGYuX3Jlc3VsdCA9IHNwZWVjaF9kaWFyaXphdGlvbgoKICAgIGRlZiB0b190dXBsZShzZWxmKSAtPiBUdXBsZVtzdHIsIGRpY3RdOgogICAgICAgICIiIgogICAgICAgIENvbnZlcnQgdGhlIHRhc2sgdG8gYSB0dXBsZSB0byByZWNvbnN0cnVjdCBpdCBsYXRlciAodXNlZCBmb3IgbXVsdGlwcm9jZXNzaW5nIHRvIHBhc3MgaW4gcXVldWUpLgoKICAgICAgICA6cmV0dXJuczogVGhlIGNvbnZlcnRlZCB0YXNrLgogICAgICAgICIiIgogICAgICAgIHRhc2tfY2xhc3MsIHRhc2tfa3dhcmdzID0gc3VwZXIoKS50b190dXBsZSgpCiAgICAgICAgcmV0dXJuIHRhc2tfY2xhc3MsIHsqKnRhc2tfa3dhcmdzLCAic3BlYWtlcl9sYWJlbHMiOiBzZWxmLl9zcGVha2VyX2xhYmVsc30KCgpjbGFzcyBUYXNrQ3JlYXRvcjoKICAgICIiIgogICAgQSB0YXNrIGNyZWF0b3IgdG8gY3JlYXRlIGRpZmZlcmVudCB0YXNrcyB0byBydW4gYWZ0ZXIgdGhlIFZBRC4KICAgICIiIgoKICAgICM6IEEgbWFwIGZyb20gdGFzayBjbGFzcyBuYW1lIHRvIHRhc2sgY2xhc3MgdG8gdXNlIGluIGBmcm9tX3R1cGxlYDoKICAgIF9NQVAgPSB7CiAgICAgICAgQmFzZVRhc2suX19uYW1lX186IEJhc2VUYXNrLAogICAgICAgIFNwZWVjaERpYXJpemF0aW9uVGFzay5fX25hbWVfXzogU3BlZWNoRGlhcml6YXRpb25UYXNrLAogICAgfQoKICAgIGRlZiBfX2luaXRfXyhzZWxmLCB0YXNrX3R5cGU6IFR5cGVbQmFzZVRhc2tdLCB0YXNrX2t3YXJnczogZGljdCA9IE5vbmUpOgogICAgICAgICIiIgogICAgICAgIEluaXRpYWxpemUgdGhlIHRhc2sgY3JlYXRvci4KICAgICAgICA6cGFyYW0gdGFza190eXBlOiBUaGUgdGFzayB0eXBlIC0gYSBgQmFzZVRhc2tgIHN1YmNsYXNzLgogICAgICAgIDpwYXJhbSB0YXNrX2t3YXJnczogQWRkaXRpb25hbCBrZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIHRvIHRoZSB0byBiZSBjcmVhdGVkIHRhc2tzLgogICAgICAgICIiIgogICAgICAgIHNlbGYuX3Rhc2tfdHlwZSA9IHRhc2tfdHlwZQogICAgICAgIHNlbGYuX3Rhc2tfa3dhcmdzID0gdGFza19rd2FyZ3Mgb3Ige30KCiAgICBkZWYgY3JlYXRlX3Rhc2soc2VsZiwgYXVkaW9fZmlsZTogUGF0aCkgLT4gQmFzZVRhc2s6CiAgICAgICAgIiIiCiAgICAgICAgQ3JlYXRlIGEgdGFzayB3aXRoIHRoZSBnaXZlbiBhdWRpbyBmaWxlLgoKICAgICAgICA6cGFyYW0gYXVkaW9fZmlsZTogVGhlIGF1ZGlvIGZpbGUgdG8gYXNzaWduIHRvIHRoZSB0YXNrLgoKICAgICAgICA6cmV0dXJuczogVGhlIGNyZWF0ZWQgdGFzay4KICAgICAgICAiIiIKICAgICAgICByZXR1cm4gc2VsZi5fdGFza190eXBlKGF1ZGlvX2ZpbGU9YXVkaW9fZmlsZSwgKipzZWxmLl90YXNrX2t3YXJncykKCiAgICBAY2xhc3NtZXRob2QKICAgIGRlZiBmcm9tX3R1cGxlKGNscywgdGFza190dXBsZTogVHVwbGVbc3RyLCBkaWN0XSkgLT4gQmFzZVRhc2s6CiAgICAgICAgIiIiCiAgICAgICAgQ3JlYXRlIGEgdGFzayBmcm9tIGEgdHVwbGUgb2YgdGhlIGF1ZGlvIGZpbGUgbmFtZSBhbmQgdGhlIHRhc2sga3dhcmdzLgoKICAgICAgICA6cGFyYW0gdGFza190dXBsZTogVGhlIHRhc2sgdHVwbGUgdG8gY3JlYXRlIHRoZSB0YXNrIGZyb20uCgogICAgICAgIDpyZXR1cm5zOiBUaGUgY3JlYXRlZCB0YXNrLgogICAgICAgICIiIgogICAgICAgIHRhc2tfY2xhc3MsIHRhc2tfa3dhcmdzID0gdGFza190dXBsZQogICAgICAgIHJldHVybiBjbHMuX01BUFt0YXNrX2NsYXNzXSgqKnRhc2tfa3dhcmdzKQoKCmNsYXNzIFZvaWNlQWN0aXZpdHlEZXRlY3RvcjoKICAgICIiIgogICAgQSB2b2ljZSBhY3Rpdml0eSBkZXRlY3Rpb24gd3JhcHBlciBmb3IgdGhlIHNpbGVybyBWQUQgbW9kZWwgLSBodHRwczovL2dpdGh1Yi5jb20vc25ha2VyczQvc2lsZXJvLXZhZC4KICAgICIiIgoKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgICMgTW9kZWwgbG9hZGluZyBrd2FyZ3M6CiAgICAgICAgdXNlX29ubng6IGJvb2wgPSBUcnVlLAogICAgICAgIGZvcmNlX29ubnhfY3B1OiBib29sID0gVHJ1ZSwKICAgICAgICAjIERldGVjdGlvbiBrd2FyZ3M6CiAgICAgICAgdGhyZXNob2xkOiBmbG9hdCA9IDAuNSwKICAgICAgICBzYW1wbGluZ19yYXRlOiBpbnQgPSAxNl8wMDAsCiAgICAgICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tczogaW50ID0gMjUwLAogICAgICAgIG1heF9zcGVlY2hfZHVyYXRpb25fczogZmxvYXQgPSBmbG9hdCgiaW5mIiksCiAgICAgICAgbWluX3NpbGVuY2VfZHVyYXRpb25fbXM6IGludCA9IDEwMCwKICAgICAgICB3aW5kb3dfc2l6ZV9zYW1wbGVzOiBpbnQgPSA1MTIsCiAgICAgICAgc3BlZWNoX3BhZF9tczogaW50ID0gMzAsCiAgICAgICAgcmV0dXJuX3NlY29uZHM6IGJvb2wgPSBGYWxzZSwKICAgICAgICBwZXJfY2hhbm5lbDogYm9vbCA9IEZhbHNlLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBJbml0aWFsaXplIHRoZSB2b2ljZSBhY3Rpdml0eSBkZXRlY3Rvci4KCiAgICAgICAgOnBhcmFtIHVzZV9vbm54OiAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHVzZSBPTk5YIGZvciBpbmZlcmVuY2UuIERlZmF1bHQgaXMgVHJ1ZS4KICAgICAgICA6cGFyYW0gZm9yY2Vfb25ueF9jcHU6ICAgICAgICAgIFdoZXRoZXIgdG8gZm9yY2UgT05OWCB0byB1c2UgQ1BVIGZvciBpbmZlcmVuY2UuIERlZmF1bHQgaXMgVHJ1ZS4KICAgICAgICA6cGFyYW0gdGhyZXNob2xkOiAgICAgICAgICAgICAgIFNwZWVjaCB0aHJlc2hvbGQuIFNpbGVybyBWQUQgb3V0cHV0cyBzcGVlY2ggcHJvYmFiaWxpdGllcyBmb3IgZWFjaCBhdWRpbyBjaHVuaywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHByb2JhYmlsaXRpZXMgQUJPVkUgdGhpcyB2YWx1ZSBhcmUgY29uc2lkZXJlZCBhcyBTUEVFQ0guIEl0IGlzIGJldHRlciB0byB0dW5lCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGlzIHBhcmFtZXRlciBmb3IgZWFjaCBkYXRhc2V0IHNlcGFyYXRlbHksIGJ1dCAibGF6eSIgMC41IGlzIHByZXR0eSBnb29kIGZvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9zdCBkYXRhc2V0cy4KICAgICAgICA6cGFyYW0gc2FtcGxpbmdfcmF0ZTogICAgICAgICAgIEN1cnJlbnRseSwgc2lsZXJvIFZBRCBtb2RlbHMgc3VwcG9ydCA4MDAwIGFuZCAxNjAwMCBzYW1wbGUgcmF0ZXMuCiAgICAgICAgOnBhcmFtIG1pbl9zcGVlY2hfZHVyYXRpb25fbXM6ICBGaW5hbCBzcGVlY2ggY2h1bmtzIHNob3J0ZXIgbWluX3NwZWVjaF9kdXJhdGlvbl9tcyBhcmUgdGhyb3duIG91dC4KICAgICAgICA6cGFyYW0gbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiAgIE1heGltdW0gZHVyYXRpb24gb2Ygc3BlZWNoIGNodW5rcyBpbiBzZWNvbmRzLiBDaHVua3MgbG9uZ2VyIHRoYW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtYXhfc3BlZWNoX2R1cmF0aW9uX3NgIHdpbGwgYmUgc3BsaXQgYXQgdGhlIHRpbWVzdGFtcCBvZiB0aGUgbGFzdCBzaWxlbmNlIHRoYXQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGxhc3RzIG1vcmUgdGhhbiAxMDBtcyAoaWYgYW55KSwgdG8gcHJldmVudCBhZ2dyZXNzaXZlIGN1dHRpbmcuIE90aGVyd2lzZSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZXkgd2lsbCBiZSBzcGxpdCBhZ2dyZXNzaXZlbHkganVzdCBiZWZvcmUgbWF4X3NwZWVjaF9kdXJhdGlvbl9zLgogICAgICAgIDpwYXJhbSBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogSW4gdGhlIGVuZCBvZiBlYWNoIHNwZWVjaCBjaHVuayB3YWl0IGZvciBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyBiZWZvcmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNlcGFyYXRpbmcgaXQuCiAgICAgICAgOnBhcmFtIHdpbmRvd19zaXplX3NhbXBsZXM6ICAgICBBdWRpbyBjaHVua3Mgb2Ygd2luZG93X3NpemVfc2FtcGxlcyBzaXplIGFyZSBmZWQgdG8gdGhlIHNpbGVybyBWQUQgbW9kZWwuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXQVJOSU5HISBTaWxlcm8gVkFEIG1vZGVscyB3ZXJlIHRyYWluZWQgdXNpbmcgNTEyLCAxMDI0LCAxNTM2IHNhbXBsZXMgZm9yIDE2MDAwCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgcmF0ZSBhbmQgMjU2LCA1MTIsIDc2OCBzYW1wbGVzIGZvciA4MDAwIHNhbXBsZSByYXRlLiBWYWx1ZXMgb3RoZXIgdGhhbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhlc2UgbWF5IGFmZmVjdCBtb2RlbCBwZXJmb3JtYW5jZSEKICAgICAgICA6cGFyYW0gc3BlZWNoX3BhZF9tczogICAgICAgICAgIEZpbmFsIHNwZWVjaCBjaHVua3MgYXJlIHBhZGRlZCBieSBzcGVlY2hfcGFkX21zIGVhY2ggc2lkZS4KICAgICAgICA6cGFyYW0gcmV0dXJuX3NlY29uZHM6ICAgICAgICAgIFdoZXRoZXIgcmV0dXJuIHRpbWVzdGFtcHMgaW4gc2Vjb25kcy4gRmFsc2UgbWVhbnMgdG8gcmV0dXJuIHRpbWVzdGFtcHMgaW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZXMgKGRlZmF1bHQgLSBGYWxzZSkuCiAgICAgICAgOnBhcmFtIHBlcl9jaGFubmVsOiAgICAgICAgICAgICBXaGV0aGVyIHRvIHJldHVybiB0aW1lc3RhbXBzIHBlciBjaGFubmVsIChkZWZhdWx0IC0gRmFsc2UpLiBUaGlzIHdpbGwgcnVuIFZBRAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb24gZWFjaCBjaGFubmVsIHNlcGFyYXRlbHkgYW5kIHJldHVybiBhIGxpc3Qgb2YgdGltZXN0YW1wcyBwZXIgY2hhbm5lbC4KICAgICAgICAiIiIKICAgICAgICAjIFN0b3JlIGNvbmZpZ3VyYXRpb25zOgogICAgICAgIHNlbGYuX3VzZV9vbm54ID0gdXNlX29ubngKICAgICAgICBzZWxmLl9mb3JjZV9vbm54X2NwdSA9IGZvcmNlX29ubnhfY3B1CiAgICAgICAgc2VsZi5fdGhyZXNob2xkID0gdGhyZXNob2xkCiAgICAgICAgc2VsZi5fc2FtcGxpbmdfcmF0ZSA9IHNhbXBsaW5nX3JhdGUKICAgICAgICBzZWxmLl9taW5fc3BlZWNoX2R1cmF0aW9uX21zID0gbWluX3NwZWVjaF9kdXJhdGlvbl9tcwogICAgICAgIHNlbGYuX21heF9zcGVlY2hfZHVyYXRpb25fcyA9IG1heF9zcGVlY2hfZHVyYXRpb25fcwogICAgICAgIHNlbGYuX21pbl9zaWxlbmNlX2R1cmF0aW9uX21zID0gbWluX3NpbGVuY2VfZHVyYXRpb25fbXMKICAgICAgICBzZWxmLl93aW5kb3dfc2l6ZV9zYW1wbGVzID0gd2luZG93X3NpemVfc2FtcGxlcwogICAgICAgIHNlbGYuX3NwZWVjaF9wYWRfbXMgPSBzcGVlY2hfcGFkX21zCiAgICAgICAgc2VsZi5fcmV0dXJuX3NlY29uZHMgPSByZXR1cm5fc2Vjb25kcwogICAgICAgIHNlbGYuX3Blcl9jaGFubmVsID0gcGVyX2NoYW5uZWwKCiAgICAgICAgIyBQcmVwYXJlIHRoZSBtb2RlbCB2YXJpYWJsZXMKICAgICAgICBzZWxmLl9tb2RlbDogdG9yY2guTW9kdWxlID0gTm9uZQogICAgICAgIHNlbGYuX2dldF9zcGVlY2hfdGltZXN0YW1wczogRnVuY3Rpb25UeXBlID0gTm9uZQoKICAgIGRlZiBsb2FkKHNlbGYsIGZvcmNlX3JlbG9hZDogYm9vbCA9IFRydWUpOgogICAgICAgICIiIgogICAgICAgIExvYWQgdGhlIFZBRCBtb2RlbC4KCiAgICAgICAgOnBhcmFtIGZvcmNlX3JlbG9hZDogV2hldGhlciB0byBmb3JjZSByZWxvYWQgdGhlIG1vZGVsIGV2ZW4gaWYgaXQgd2FzIGFscmVhZHkgbG9hZGVkLiBEZWZhdWx0IGlzIFRydWUuCiAgICAgICAgIiIiCiAgICAgICAgbW9kZWwsIHV0aWxzID0gdG9yY2guaHViLmxvYWQoCiAgICAgICAgICAgIHJlcG9fb3JfZGlyPSJzbmFrZXJzNC9zaWxlcm8tdmFkIiwKICAgICAgICAgICAgbW9kZWw9InNpbGVyb192YWQiLAogICAgICAgICAgICBmb3JjZV9yZWxvYWQ9Zm9yY2VfcmVsb2FkLAogICAgICAgICAgICBvbm54PXNlbGYuX3VzZV9vbm54LAogICAgICAgICAgICBmb3JjZV9vbm54X2NwdT1zZWxmLl9mb3JjZV9vbm54X2NwdSwKICAgICAgICApCiAgICAgICAgc2VsZi5fbW9kZWwgPSBtb2RlbAogICAgICAgICgKICAgICAgICAgICAgc2VsZi5fZ2V0X3NwZWVjaF90aW1lc3RhbXBzLAogICAgICAgICAgICBfLCAgIyBzYXZlX2F1ZGlvLAogICAgICAgICAgICBfLCAgIyByZWFkX2F1ZGlvLAogICAgICAgICAgICBfLCAgIyBWQURJdGVyYXRvciwKICAgICAgICAgICAgXywgICMgY29sbGVjdF9jaHVua3MKICAgICAgICApID0gdXRpbHMKCiAgICBkZWYgZGV0ZWN0X3ZvaWNlKAogICAgICAgIHNlbGYsCiAgICAgICAgYXVkaW9fZmlsZTogUGF0aCwKICAgICkgLT4gVW5pb25bTGlzdFtEaWN0W3N0ciwgaW50XV0sIExpc3RbTGlzdFtEaWN0W3N0ciwgaW50XV1dXToKICAgICAgICAiIiIKICAgICAgICBJbmZlciB0aGUgYXVkaW8gdGhyb3VnaCB0aGUgVkFEIG1vZGVsIGFuZCByZXR1cm4gdGhlIHNwZWVjaCB0aW1lc3RhbXBzLgoKICAgICAgICA6cGFyYW0gYXVkaW9fZmlsZTogVGhlIGF1ZGlvIGZpbGUgdG8gaW5mZXIuCgogICAgICAgIDpyZXR1cm5zOiBUaGUgc3BlZWNoIHRpbWVzdGFtcHMgaW4gdGhlIGF1ZGlvLiBBIGxpc3Qgb2YgdGltZXN0YW1wcyB3aGVyZSBlYWNoIHRpbWVzdGFtcCBpcyBhIGRpY3Rpb25hcnkgd2l0aCB0aGUKICAgICAgICAgICAgICAgICBmb2xsb3dpbmcga2V5czoKCiAgICAgICAgICAgICAgICAgKiAic3RhcnQiOiBUaGUgc3RhcnQgc2FtcGxlIGluZGV4IG9mIHRoZSBzcGVlY2ggaW4gdGhlIGF1ZGlvLgogICAgICAgICAgICAgICAgICogImVuZCI6ICAgVGhlIGVuZCBzYW1wbGUgaW5kZXggb2YgdGhlIHNwZWVjaCBpbiB0aGUgYXVkaW8uCgogICAgICAgICAgICAgICAgIElmIGBwZXJfY2hhbm5lbGAgaXMgVHJ1ZSwgYSBsaXN0IG9mIHRpbWVzdGFtcHMgcGVyIGNoYW5uZWwgd2lsbCBiZSByZXR1cm5lZC4KICAgICAgICAiIiIKICAgICAgICAjIENhc3QgdG8gYSBudW1weSBhcnJheToKICAgICAgICBhdWRpbyA9IHNlbGYuX3JlYWRfYXVkaW8oYXVkaW9fZmlsZSkKCiAgICAgICAgIyBEZXRlY3Qgc3BlZWNoOgogICAgICAgIGlmIG5vdCBzZWxmLl9wZXJfY2hhbm5lbDoKICAgICAgICAgICAgcmV0dXJuIHNlbGYuX2dldF9zcGVlY2hfdGltZXN0YW1wcygKICAgICAgICAgICAgICAgIGF1ZGlvLAogICAgICAgICAgICAgICAgc2VsZi5fbW9kZWwsCiAgICAgICAgICAgICAgICB0aHJlc2hvbGQ9c2VsZi5fdGhyZXNob2xkLAogICAgICAgICAgICAgICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tcz1zZWxmLl9taW5fc3BlZWNoX2R1cmF0aW9uX21zLAogICAgICAgICAgICAgICAgbWF4X3NwZWVjaF9kdXJhdGlvbl9zPXNlbGYuX21heF9zcGVlY2hfZHVyYXRpb25fcywKICAgICAgICAgICAgICAgIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zPXNlbGYuX21pbl9zaWxlbmNlX2R1cmF0aW9uX21zLAogICAgICAgICAgICAgICAgc3BlZWNoX3BhZF9tcz1zZWxmLl9zcGVlY2hfcGFkX21zLAogICAgICAgICAgICAgICAgc2FtcGxpbmdfcmF0ZT1zZWxmLl9zYW1wbGluZ19yYXRlLAogICAgICAgICAgICAgICAgd2luZG93X3NpemVfc2FtcGxlcz1zZWxmLl93aW5kb3dfc2l6ZV9zYW1wbGVzLAogICAgICAgICAgICAgICAgcmV0dXJuX3NlY29uZHM9c2VsZi5fcmV0dXJuX3NlY29uZHMsCiAgICAgICAgICAgICkKCiAgICAgICAgIyBQZXIgY2hhbm5lbDoKICAgICAgICBzcGVlY2hfdGltZXN0YW1wcyA9IFtdCiAgICAgICAgZm9yIGNoYW5uZWwgaW4gYXVkaW86CiAgICAgICAgICAgIHNwZWVjaF90aW1lc3RhbXBzLmFwcGVuZCgKICAgICAgICAgICAgICAgIHNlbGYuX2dldF9zcGVlY2hfdGltZXN0YW1wcygKICAgICAgICAgICAgICAgICAgICBjaGFubmVsLAogICAgICAgICAgICAgICAgICAgIHNlbGYuX21vZGVsLAogICAgICAgICAgICAgICAgICAgIHRocmVzaG9sZD1zZWxmLl90aHJlc2hvbGQsCiAgICAgICAgICAgICAgICAgICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tcz1zZWxmLl9taW5fc3BlZWNoX2R1cmF0aW9uX21zLAogICAgICAgICAgICAgICAgICAgIG1heF9zcGVlY2hfZHVyYXRpb25fcz1zZWxmLl9tYXhfc3BlZWNoX2R1cmF0aW9uX3MsCiAgICAgICAgICAgICAgICAgICAgbWluX3NpbGVuY2VfZHVyYXRpb25fbXM9c2VsZi5fbWluX3NpbGVuY2VfZHVyYXRpb25fbXMsCiAgICAgICAgICAgICAgICAgICAgc3BlZWNoX3BhZF9tcz1zZWxmLl9zcGVlY2hfcGFkX21zLAogICAgICAgICAgICAgICAgICAgIHNhbXBsaW5nX3JhdGU9c2VsZi5fc2FtcGxpbmdfcmF0ZSwKICAgICAgICAgICAgICAgICAgICB3aW5kb3dfc2l6ZV9zYW1wbGVzPXNlbGYuX3dpbmRvd19zaXplX3NhbXBsZXMsCiAgICAgICAgICAgICAgICAgICAgcmV0dXJuX3NlY29uZHM9c2VsZi5fcmV0dXJuX3NlY29uZHMsCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICkKCiAgICAgICAgcmV0dXJuIHNwZWVjaF90aW1lc3RhbXBzCgogICAgZGVmIF9yZWFkX2F1ZGlvKAogICAgICAgIHNlbGYsCiAgICAgICAgcGF0aDogUGF0aCwKICAgICkgLT4gdG9yY2guVGVuc29yOgogICAgICAgICIiIgogICAgICAgIFJlYWQgdGhlIGF1ZGlvIGZyb20gdGhlIGdpdmVuIHBhdGggYW5kIHJldHVybiBpdCBhcyBhIHRlbnNvci4KCiAgICAgICAgOnBhcmFtIHBhdGg6IFRoZSBwYXRoIHRvIHRoZSBhdWRpbyBmaWxlLgoKICAgICAgICA6cmV0dXJuczogVGhlIGF1ZGlvIGFzIGEgdGVuc29yLgogICAgICAgICIiIgogICAgICAgICMgUmVhZCB0aGUgYXVkaW86CiAgICAgICAgYXVkaW8sIHNhbXBsaW5nX3JhdGUgPSB0b3JjaGF1ZGlvLmxvYWQoc3RyKHBhdGgpKQoKICAgICAgICAjIENoZWNrIGlmIHRoZSBhdWRpbyBpcyBzdGVyZW8gYW5kIGlmIHNvLCBjb252ZXJ0IGl0IHRvIG1vbm8gKG9ubHkgaWYgbm90IHBlciBjaGFubmVsKToKICAgICAgICBpZiBhdWRpby5zaXplKDApID4gMSBhbmQgbm90IHNlbGYuX3Blcl9jaGFubmVsOgogICAgICAgICAgICBhdWRpbyA9IGF1ZGlvLm1lYW4oZGltPTAsIGtlZXBkaW09VHJ1ZSkKCiAgICAgICAgIyBSZXNhbXBsZSB0aGUgYXVkaW8gaWYgbmVlZGVkOgogICAgICAgIGlmIHNhbXBsaW5nX3JhdGUgIT0gc2VsZi5fc2FtcGxpbmdfcmF0ZToKICAgICAgICAgICAgdHJhbnNmb3JtID0gdG9yY2hhdWRpby50cmFuc2Zvcm1zLlJlc2FtcGxlKAogICAgICAgICAgICAgICAgb3JpZ19mcmVxPXNhbXBsaW5nX3JhdGUsIG5ld19mcmVxPXNlbGYuX3NhbXBsaW5nX3JhdGUKICAgICAgICAgICAgKQogICAgICAgICAgICBhdWRpbyA9IHRyYW5zZm9ybShhdWRpbykKCiAgICAgICAgIyBSZXR1cm4gdGhlIGF1ZGlvIChzcXVlZXplIGlmIG5vdCBwZXIgY2hhbm5lbCk6CiAgICAgICAgcmV0dXJuIGF1ZGlvIGlmIHNlbGYuX3Blcl9jaGFubmVsIGVsc2UgYXVkaW8uc3F1ZWV6ZSgwKQoKCiM6IFRoZSB2YWx1ZSB0byBzZW5kIGludG8gbXVsdGlwcm9jZXNzaW5nIHF1ZXVlcyB0byBzdG9wIHRoZSBwcm9jZXNzOgpfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSyA9ICJTVE9QIgoKCmRlZiBfbXVsdGlwcm9jZXNzaW5nX2NvbXBsZXRlX3Rhc2tzKAogICAgdmFkX2luaXRfa3dhcmdzOiBkaWN0LCB0YXNrc19xdWV1ZTogUXVldWUsIHJlc3VsdHNfcXVldWU6IFF1ZXVlCik6CiAgICAiIiIKICAgIENvbXBsZXRlIHRoZSB0YXNrcyBpbiB0aGUgZ2l2ZW4gcXVldWUgYW5kIHB1dCB0aGUgcmVzdWx0cyBpbiB0aGUgZ2l2ZW4gcmVzdWx0cyBxdWV1ZS4gVGhlIGZ1bmN0aW9uIHdpbGwgc3RvcCB3aGVuCiAgICB0aGUgZ2l2ZW4gdGFza3MgcXVldWUgd2lsbCByZWNlaXZlIHRoZSBzdG9wIG1hcmsuIEl0IGlzIGFpbWVkIHRvIGJlIHVzZWQgd2l0aCBtdWx0aXByb2Nlc3NpbmcgYXMgYSBwcm9jZXNzLgoKICAgIDpwYXJhbSB2YWRfaW5pdF9rd2FyZ3M6IFRoZSBWQUQgaW5pdGlhbGl6YXRpb24ga3dhcmdzLgogICAgOnBhcmFtIHRhc2tzX3F1ZXVlOiAgICAgQSBxdWV1ZSB0byBnZXQgdGhlIHRhc2tzIGZyb20uCiAgICA6cGFyYW0gcmVzdWx0c19xdWV1ZTogICBBIHF1ZXVlIHRvIHB1dCB0aGUgcmVzdWx0cyBpbi4KICAgICIiIgogICAgIyBJbml0aWFsaXplIGFuZCBsb2FkIHRoZSBWQUQ6CiAgICB2YWQgPSBWb2ljZUFjdGl2aXR5RGV0ZWN0b3IoKip2YWRfaW5pdF9rd2FyZ3MpCiAgICB2YWQubG9hZChmb3JjZV9yZWxvYWQ9RmFsc2UpCgogICAgIyBTdGFydCBsaXN0ZW5pbmcgdG8gdGhlIHRhc2tzIHF1ZXVlOgogICAgd2hpbGUgVHJ1ZToKICAgICAgICAjIEdldCB0aGUgdGFzazoKICAgICAgICB0YXNrOiBUdXBsZVtzdHIsIGRpY3RdID0gdGFza3NfcXVldWUuZ2V0KCkKICAgICAgICBpZiB0YXNrID09IF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLOgogICAgICAgICAgICBicmVhawogICAgICAgIHRyeToKICAgICAgICAgICAgIyBDcmVhdGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2sgPSBUYXNrQ3JlYXRvci5mcm9tX3R1cGxlKHRhc2tfdHVwbGU9dGFzaykKICAgICAgICAgICAgIyBSdW4gdGhlIGZpbGUgdGhyb3VnaCB0aGUgVkFEOgogICAgICAgICAgICBzcGVlY2hfdGltZXN0YW1wcyA9IHZhZC5kZXRlY3Rfdm9pY2UoYXVkaW9fZmlsZT10YXNrLmF1ZGlvX2ZpbGUpCiAgICAgICAgICAgICMgQ29tcGxldGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2suZG9fdGFzayhzcGVlY2hfdGltZXN0YW1wcz1zcGVlY2hfdGltZXN0YW1wcykKICAgICAgICAgICAgIyBCdWlsZCB0aGUgcmVzdWx0OgogICAgICAgICAgICByZXN1bHQgPSAoRmFsc2UsIHRhc2suZ2V0X3Jlc3VsdCgpKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIEJ1aWxkIHRoZSBlcnJvcjoKICAgICAgICAgICAgcmVzdWx0ID0gKFRydWUsICh0YXNrLmF1ZGlvX2ZpbGUubmFtZSwgc3RyKGV4Y2VwdGlvbikpKQogICAgICAgICMgQ29sbGVjdCB0aGUgcmVzdWx0IC8gZXJyb3I6CiAgICAgICAgcmVzdWx0c19xdWV1ZS5wdXQocmVzdWx0KQoKICAgICMgTWFyayB0aGUgZW5kIG9mIHRoZSB0YXNrczoKICAgIHJlc3VsdHNfcXVldWUucHV0KF9NVUxUSVBST0NFU1NJTkdfU1RPUF9NQVJLKQoKCiMgR2V0IHRoZSBnbG9iYWwgbG9nZ2VyOgp0cnk6CiAgICBpbXBvcnQgbWxydW4KCiAgICBfTE9HR0VSID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9jdHgoInNpbGVyb192YWQiKS5sb2dnZXIKZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3I6CiAgICBfTE9HR0VSID0gbG9nZ2luZy5nZXRMb2dnZXIoKQoKCmRlZiBkZXRlY3Rfdm9pY2UoCiAgICAjIElucHV0IGt3YXJnczoKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBQYXRoLCBMaXN0W1VuaW9uW3N0ciwgUGF0aF1dXSwKICAgICMgTW9kZWwgbG9hZGluZyBrd2FyZ3M6CiAgICB1c2Vfb25ueDogYm9vbCA9IFRydWUsCiAgICBmb3JjZV9vbm54X2NwdTogYm9vbCA9IFRydWUsCiAgICAjIERldGVjdGlvbiBrd2FyZ3M6CiAgICB0aHJlc2hvbGQ6IGZsb2F0ID0gMC41LAogICAgc2FtcGxpbmdfcmF0ZTogaW50ID0gMTZfMDAwLAogICAgbWluX3NwZWVjaF9kdXJhdGlvbl9tczogaW50ID0gMjUwLAogICAgbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiBmbG9hdCA9IGZsb2F0KCJpbmYiKSwKICAgIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zOiBpbnQgPSAxMDAsCiAgICB3aW5kb3dfc2l6ZV9zYW1wbGVzOiBpbnQgPSA1MTIsCiAgICBzcGVlY2hfcGFkX21zOiBpbnQgPSAzMCwKICAgIHJldHVybl9zZWNvbmRzOiBib29sID0gRmFsc2UsCiAgICBwZXJfY2hhbm5lbDogYm9vbCA9IEZhbHNlLAogICAgIyBPdGhlciBrd2FyZ3M6CiAgICB1c2VfbXVsdGlwcm9jZXNzaW5nOiBpbnQgPSAwLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBQZXJmb3JtIHZvaWNlIGFjdGl2aXR5IGRldGVjdGlvbiBvbiBnaXZlbiBhdWRpbyBmaWxlcyB1c2luZyB0aGUgc2lsZXJvIFZBRCBtb2RlbCAtCiAgICBodHRwczovL2dpdGh1Yi5jb20vc25ha2VyczQvc2lsZXJvLXZhZC4gVGhlIGVuZCByZXN1bHQgaXMgYSBkaWN0aW9uYXJ5IHdpdGggdGhlIGZpbGUgbmFtZXMgYXMga2V5cyBhbmQgdGhlaXIKICAgIFZBRCB0aW1lc3RhbXBzIGRpY3Rpb25hcmllcyBhcyB2YWx1ZS4KCiAgICBGb3IgZXhhbXBsZTo6CgogICAgICAgIHsKICAgICAgICAgICAgImZpbGVfMS53YXYiOiBbCiAgICAgICAgICAgICAgICB7InN0YXJ0IjogMCwgImVuZCI6IDE2MDAwfSwKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAxNjAwMCwgImVuZCI6IDMyMDAwfSwKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAzMjAwMCwgImVuZCI6IDQ4MDAwfSwKICAgICAgICAgICAgICAgIC4uLgogICAgICAgICAgICBdLAogICAgICAgICAgICAiZmlsZV8yLndhdiI6IFsKICAgICAgICAgICAgICAgIHsic3RhcnQiOiAwLCAiZW5kIjogMTYwMDB9LAogICAgICAgICAgICAgICAgeyJzdGFydCI6IDE2MDAwLCAiZW5kIjogMzIwMDB9LAogICAgICAgICAgICAgICAgeyJzdGFydCI6IDMyMDAwLCAiZW5kIjogNDgwMDB9LAogICAgICAgICAgICAgICAgLi4uCiAgICAgICAgICAgIF0sCiAgICAgICAgICAgIC4uLgogICAgICAgIH0KCgogICAgOnBhcmFtIGRhdGFfcGF0aDogICAgICAgICAgICAgICBUaGUgcGF0aCB0byB0aGUgYXVkaW8gZmlsZXMgdG8gZGlhcml6ZS4gQ2FuIGJlIGEgcGF0aCB0byBhIHNpbmdsZSBmaWxlLCBhIHBhdGggdG8gYQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBkaXJlY3Rvcnkgb3IgYSBsaXN0IG9mIHBhdGhzIHRvIGZpbGVzLgogICAgOnBhcmFtIHVzZV9vbm54OiAgICAgICAgICAgICAgICBXaGV0aGVyIHRvIHVzZSBPTk5YIGZvciBpbmZlcmVuY2UuIERlZmF1bHQgaXMgVHJ1ZS4KICAgIDpwYXJhbSBmb3JjZV9vbm54X2NwdTogICAgICAgICAgV2hldGhlciB0byBmb3JjZSBPTk5YIHRvIHVzZSBDUFUgZm9yIGluZmVyZW5jZS4gRGVmYXVsdCBpcyBUcnVlLgogICAgOnBhcmFtIHRocmVzaG9sZDogICAgICAgICAgICAgICBTcGVlY2ggdGhyZXNob2xkLiBTaWxlcm8gVkFEIG91dHB1dHMgc3BlZWNoIHByb2JhYmlsaXRpZXMgZm9yIGVhY2ggYXVkaW8gY2h1bmssCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHByb2JhYmlsaXRpZXMgQUJPVkUgdGhpcyB2YWx1ZSBhcmUgY29uc2lkZXJlZCBhcyBTUEVFQ0guIEl0IGlzIGJldHRlciB0byB0dW5lCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoaXMgcGFyYW1ldGVyIGZvciBlYWNoIGRhdGFzZXQgc2VwYXJhdGVseSwgYnV0ICJsYXp5IiAwLjUgaXMgcHJldHR5IGdvb2QgZm9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vc3QgZGF0YXNldHMuCiAgICA6cGFyYW0gc2FtcGxpbmdfcmF0ZTogICAgICAgICAgIEN1cnJlbnRseSwgc2lsZXJvIFZBRCBtb2RlbHMgc3VwcG9ydCA4MDAwIGFuZCAxNjAwMCBzYW1wbGUgcmF0ZXMuCiAgICA6cGFyYW0gbWluX3NwZWVjaF9kdXJhdGlvbl9tczogIEZpbmFsIHNwZWVjaCBjaHVua3Mgc2hvcnRlciBtaW5fc3BlZWNoX2R1cmF0aW9uX21zIGFyZSB0aHJvd24gb3V0LgogICAgOnBhcmFtIG1heF9zcGVlY2hfZHVyYXRpb25fczogICBNYXhpbXVtIGR1cmF0aW9uIG9mIHNwZWVjaCBjaHVua3MgaW4gc2Vjb25kcy4gQ2h1bmtzIGxvbmdlciB0aGFuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtYXhfc3BlZWNoX2R1cmF0aW9uX3NgIHdpbGwgYmUgc3BsaXQgYXQgdGhlIHRpbWVzdGFtcCBvZiB0aGUgbGFzdCBzaWxlbmNlIHRoYXQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGFzdHMgbW9yZSB0aGFuIDEwMG1zIChpZiBhbnkpLCB0byBwcmV2ZW50IGFnZ3Jlc3NpdmUgY3V0dGluZy4gT3RoZXJ3aXNlLCB0aGV5IHdpbGwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYmUgc3BsaXQgYWdncmVzc2l2ZWx5IGp1c3QgYmVmb3JlIG1heF9zcGVlY2hfZHVyYXRpb25fcy4KICAgIDpwYXJhbSBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogSW4gdGhlIGVuZCBvZiBlYWNoIHNwZWVjaCBjaHVuayB3YWl0IGZvciBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyBiZWZvcmUgc2VwYXJhdGluZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBpdC4KICAgIDpwYXJhbSB3aW5kb3dfc2l6ZV9zYW1wbGVzOiAgICAgQXVkaW8gY2h1bmtzIG9mIHdpbmRvd19zaXplX3NhbXBsZXMgc2l6ZSBhcmUgZmVkIHRvIHRoZSBzaWxlcm8gVkFEIG1vZGVsLgoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV0FSTklORyEgU2lsZXJvIFZBRCBtb2RlbHMgd2VyZSB0cmFpbmVkIHVzaW5nIDUxMiwgMTAyNCwgMTUzNiBzYW1wbGVzIGZvciAxNjAwMAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgcmF0ZSBhbmQgMjU2LCA1MTIsIDc2OCBzYW1wbGVzIGZvciA4MDAwIHNhbXBsZSByYXRlLiBWYWx1ZXMgb3RoZXIgdGhhbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0aGVzZSBtYXkgYWZmZWN0IG1vZGVsIHBlcmZvcm1hbmNlIQogICAgOnBhcmFtIHNwZWVjaF9wYWRfbXM6ICAgICAgICAgICBGaW5hbCBzcGVlY2ggY2h1bmtzIGFyZSBwYWRkZWQgYnkgc3BlZWNoX3BhZF9tcyBlYWNoIHNpZGUuCiAgICA6cGFyYW0gcmV0dXJuX3NlY29uZHM6ICAgICAgICAgIFdoZXRoZXIgcmV0dXJuIHRpbWVzdGFtcHMgaW4gc2Vjb25kcy4gRmFsc2UgbWVhbnMgdG8gcmV0dXJuIHRpbWVzdGFtcHMgaW4gc2FtcGxlcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoZGVmYXVsdCAtIEZhbHNlKS4KICAgIDpwYXJhbSBwZXJfY2hhbm5lbDogICAgICAgICAgICAgV2hldGhlciB0byByZXR1cm4gdGltZXN0YW1wcyBwZXIgY2hhbm5lbCAoZGVmYXVsdCAtIEZhbHNlKS4gVGhpcyB3aWxsIHJ1biBWQUQgb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgZWFjaCBjaGFubmVsIHNlcGFyYXRlbHkgYW5kIHJldHVybiBhIGxpc3Qgb2YgdGltZXN0YW1wcyBwZXIgY2hhbm5lbC4KICAgIDpwYXJhbSB1c2VfbXVsdGlwcm9jZXNzaW5nOiAgICAgVGhlIG51bWJlciBvZiB3b3JrZXJzIHRvIHVzZSBmb3IgbXVsdGlwcm9jZXNzaW5nLiBJZiAwLCBubyBtdWx0aXByb2Nlc3Npbmcgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSB1c2VkLiBEZWZhdWx0IGlzIDAuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICAgICAgIFZlcmJvc2l0eS4KICAgICIiIgogICAgZ2xvYmFsIF9MT0dHRVIKCiAgICAjIEdldCB0aGUgaW5wdXQgYXVkaW8gZmlsZXMgdG8gdHJhbnNjcmliZToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJDb2xsZWN0aW5nIGF1ZGlvIGZpbGVzLiIpCiAgICBhdWRpb19maWxlcyA9IF9nZXRfYXVkaW9fZmlsZXMoZGF0YV9wYXRoPWRhdGFfcGF0aCkKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiQ29sbGVjdGVkIHtsZW4oYXVkaW9fZmlsZXMpfSBhdWRpbyBmaWxlcy4iKQoKICAgICMgSW5pdGlhbGl6ZSB0aGUgdHJhbnNjcmlwdGlvbiBwaXBlbGluZToKICAgIHZhZF9pbml0X2t3YXJncyA9IHsKICAgICAgICAidXNlX29ubngiOiB1c2Vfb25ueCwKICAgICAgICAiZm9yY2Vfb25ueF9jcHUiOiBmb3JjZV9vbm54X2NwdSwKICAgICAgICAidGhyZXNob2xkIjogdGhyZXNob2xkLAogICAgICAgICJzYW1wbGluZ19yYXRlIjogc2FtcGxpbmdfcmF0ZSwKICAgICAgICAibWluX3NwZWVjaF9kdXJhdGlvbl9tcyI6IG1pbl9zcGVlY2hfZHVyYXRpb25fbXMsCiAgICAgICAgIm1heF9zcGVlY2hfZHVyYXRpb25fcyI6IG1heF9zcGVlY2hfZHVyYXRpb25fcywKICAgICAgICAibWluX3NpbGVuY2VfZHVyYXRpb25fbXMiOiBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcywKICAgICAgICAid2luZG93X3NpemVfc2FtcGxlcyI6IHdpbmRvd19zaXplX3NhbXBsZXMsCiAgICAgICAgInNwZWVjaF9wYWRfbXMiOiBzcGVlY2hfcGFkX21zLAogICAgICAgICJyZXR1cm5fc2Vjb25kcyI6IHJldHVybl9zZWNvbmRzLAogICAgICAgICJwZXJfY2hhbm5lbCI6IHBlcl9jaGFubmVsLAogICAgfQoKICAgICMgQ3JlYXRlIHRoZSB0YXNrIGNyZWF0b3I6CiAgICB0YXNrX2NyZWF0b3IgPSBUYXNrQ3JlYXRvcih0YXNrX3R5cGU9QmFzZVRhc2spCgogICAgIyBSdW4gdGhlIHRyYW5zY3JpcHRpb246CiAgICBpZiB1c2VfbXVsdGlwcm9jZXNzaW5nOgogICAgICAgIHJlc3VsdHMgPSBfcGFyYWxsZWxfcnVuKAogICAgICAgICAgICBuX3dvcmtlcnM9dXNlX211bHRpcHJvY2Vzc2luZywKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEZXRlY3Rpbmcgdm9pY2UiLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXN1bHRzID0gX3J1bigKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEZXRlY3Rpbmcgdm9pY2UiLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQoKICAgICMgUHJvY2VzcyB0aGUgcmVzdWx0czoKICAgIHJldHVybiBfcHJvY2Vzc19yZXN1bHRzKHJlc3VsdHM9cmVzdWx0cywgdmVyYm9zZT12ZXJib3NlKQoKCmRlZiBkaWFyaXplKAogICAgIyBJbnB1dCAvIE91dHB1dCBrd2FyZ3M6CiAgICBkYXRhX3BhdGg6IFVuaW9uW3N0ciwgUGF0aCwgTGlzdFtVbmlvbltzdHIsIFBhdGhdXV0sCiAgICAjIE1vZGVsIGxvYWRpbmcga3dhcmdzOgogICAgdXNlX29ubng6IGJvb2wgPSBUcnVlLAogICAgZm9yY2Vfb25ueF9jcHU6IGJvb2wgPSBUcnVlLAogICAgIyBEZXRlY3Rpb24ga3dhcmdzOgogICAgdGhyZXNob2xkOiBmbG9hdCA9IDAuNSwKICAgIHNhbXBsaW5nX3JhdGU6IGludCA9IDE2XzAwMCwKICAgIG1pbl9zcGVlY2hfZHVyYXRpb25fbXM6IGludCA9IDI1MCwKICAgIG1heF9zcGVlY2hfZHVyYXRpb25fczogZmxvYXQgPSBmbG9hdCgiaW5mIiksCiAgICBtaW5fc2lsZW5jZV9kdXJhdGlvbl9tczogaW50ID0gMTAwLAogICAgd2luZG93X3NpemVfc2FtcGxlczogaW50ID0gNTEyLAogICAgc3BlZWNoX3BhZF9tczogaW50ID0gMzAsCiAgICAjIERpYXJpemF0aW9uIGt3YXJnczoKICAgIHNwZWFrZXJfbGFiZWxzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgIyBPdGhlciBrd2FyZ3M6CiAgICB1c2VfbXVsdGlwcm9jZXNzaW5nOiBpbnQgPSAwLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopOgogICAgIiIiCiAgICBQZXJmb3JtIHNwZWVjaCBkaWFyaXphdGlvbiBvbiBnaXZlbiBhdWRpbyBmaWxlcyB1c2luZyB0aGUgc2lsZXJvIFZBRCBtb2RlbCAtIGh0dHBzOi8vZ2l0aHViLmNvbS9zbmFrZXJzNC9zaWxlcm8tdmFkLgogICAgVGhlIHNwZWVjaCBkaWFyaXphdGlvbiBpcyBwZXJmb3JtZWQgcGVyIGNoYW5uZWwgc28gdGhhdCBlYWNoIGNoYW5uZWwgaW4gdGhlIGF1ZGlvIGJlbG9uZyB0byBhIGRpZmZlcmVudCBzcGVha2VyLiBUaGUKICAgIGVuZCByZXN1bHQgaXMgYSBkaWN0aW9uYXJ5IHdpdGggdGhlIGZpbGUgbmFtZXMgYXMga2V5cyBhbmQgdGhlaXIgZGlhcml6YXRpb24gYXMgdmFsdWUuIEEgZGlhcml6YXRpb24gaXMgYSBsaXN0CiAgICBvZiB0dXBsZXM6IChzdGFydCwgZW5kLCBzcGVha2VyX2xhYmVsKS4KCiAgICBGb3IgZXhhbXBsZTo6CgogICAgICAgIHsKICAgICAgICAgICAgImZpbGVfMS53YXYiOiBbCiAgICAgICAgICAgICAgICAoMC4wLCAxLjAsICJzcGVha2VyXzAiKSwKICAgICAgICAgICAgICAgICgxLjAsIDIuMCwgInNwZWFrZXJfMSIpLAogICAgICAgICAgICAgICAgKDIuMCwgMy4wLCAic3BlYWtlcl8wIiksCiAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgXSwKICAgICAgICAgICAgImZpbGVfMi53YXYiOiBbCiAgICAgICAgICAgICAgICAoMC4wLCAxLjAsICJzcGVha2VyXzAiKSwKICAgICAgICAgICAgICAgICgxLjAsIDIuMCwgInNwZWFrZXJfMSIpLAogICAgICAgICAgICAgICAgKDIuMCwgMy4wLCAic3BlYWtlcl8wIiksCiAgICAgICAgICAgICAgICAuLi4KICAgICAgICAgICAgXSwKICAgICAgICAgICAgLi4uCiAgICAgICAgfQoKCiAgICA6cGFyYW0gZGF0YV9wYXRoOiAgICAgICAgICAgICAgIFRoZSBwYXRoIHRvIHRoZSBhdWRpbyBmaWxlcyB0byBkaWFyaXplLiBDYW4gYmUgYSBwYXRoIHRvIGEgc2luZ2xlIGZpbGUsIGEgcGF0aCB0byBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGRpcmVjdG9yeSBvciBhIGxpc3Qgb2YgcGF0aHMgdG8gZmlsZXMuCiAgICA6cGFyYW0gdXNlX29ubng6ICAgICAgICAgICAgICAgIFdoZXRoZXIgdG8gdXNlIE9OTlggZm9yIGluZmVyZW5jZS4gRGVmYXVsdCBpcyBUcnVlLgogICAgOnBhcmFtIGZvcmNlX29ubnhfY3B1OiAgICAgICAgICBXaGV0aGVyIHRvIGZvcmNlIE9OTlggdG8gdXNlIENQVSBmb3IgaW5mZXJlbmNlLiBEZWZhdWx0IGlzIFRydWUuCiAgICA6cGFyYW0gdGhyZXNob2xkOiAgICAgICAgICAgICAgIFNwZWVjaCB0aHJlc2hvbGQuIFNpbGVybyBWQUQgb3V0cHV0cyBzcGVlY2ggcHJvYmFiaWxpdGllcyBmb3IgZWFjaCBhdWRpbyBjaHVuaywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgcHJvYmFiaWxpdGllcyBBQk9WRSB0aGlzIHZhbHVlIGFyZSBjb25zaWRlcmVkIGFzIFNQRUVDSC4gSXQgaXMgYmV0dGVyIHRvIHR1bmUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhpcyBwYXJhbWV0ZXIgZm9yIGVhY2ggZGF0YXNldCBzZXBhcmF0ZWx5LCBidXQgImxhenkiIDAuNSBpcyBwcmV0dHkgZ29vZCBmb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbW9zdCBkYXRhc2V0cy4KICAgIDpwYXJhbSBzYW1wbGluZ19yYXRlOiAgICAgICAgICAgQ3VycmVudGx5LCBzaWxlcm8gVkFEIG1vZGVscyBzdXBwb3J0IDgwMDAgYW5kIDE2MDAwIHNhbXBsZSByYXRlcy4KICAgIDpwYXJhbSBtaW5fc3BlZWNoX2R1cmF0aW9uX21zOiAgRmluYWwgc3BlZWNoIGNodW5rcyBzaG9ydGVyIG1pbl9zcGVlY2hfZHVyYXRpb25fbXMgYXJlIHRocm93biBvdXQuCiAgICA6cGFyYW0gbWF4X3NwZWVjaF9kdXJhdGlvbl9zOiAgIE1heGltdW0gZHVyYXRpb24gb2Ygc3BlZWNoIGNodW5rcyBpbiBzZWNvbmRzLiBDaHVua3MgbG9uZ2VyIHRoYW4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYG1heF9zcGVlY2hfZHVyYXRpb25fc2Agd2lsbCBiZSBzcGxpdCBhdCB0aGUgdGltZXN0YW1wIG9mIHRoZSBsYXN0IHNpbGVuY2UgdGhhdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBsYXN0cyBtb3JlIHRoYW4gMTAwbXMgKGlmIGFueSksIHRvIHByZXZlbnQgYWdncmVzc2l2ZSBjdXR0aW5nLiBPdGhlcndpc2UsIHRoZXkgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSBzcGxpdCBhZ2dyZXNzaXZlbHkganVzdCBiZWZvcmUgbWF4X3NwZWVjaF9kdXJhdGlvbl9zLgogICAgOnBhcmFtIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zOiBJbiB0aGUgZW5kIG9mIGVhY2ggc3BlZWNoIGNodW5rIHdhaXQgZm9yIG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zIGJlZm9yZSBzZXBhcmF0aW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGl0LgogICAgOnBhcmFtIHdpbmRvd19zaXplX3NhbXBsZXM6ICAgICBBdWRpbyBjaHVua3Mgb2Ygd2luZG93X3NpemVfc2FtcGxlcyBzaXplIGFyZSBmZWQgdG8gdGhlIHNpbGVybyBWQUQgbW9kZWwuCgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXQVJOSU5HISBTaWxlcm8gVkFEIG1vZGVscyB3ZXJlIHRyYWluZWQgdXNpbmcgNTEyLCAxMDI0LCAxNTM2IHNhbXBsZXMgZm9yIDE2MDAwCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSByYXRlIGFuZCAyNTYsIDUxMiwgNzY4IHNhbXBsZXMgZm9yIDgwMDAgc2FtcGxlIHJhdGUuIFZhbHVlcyBvdGhlciB0aGFuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZXNlIG1heSBhZmZlY3QgbW9kZWwgcGVyZm9ybWFuY2UhCiAgICA6cGFyYW0gc3BlZWNoX3BhZF9tczogICAgICAgICAgIEZpbmFsIHNwZWVjaCBjaHVua3MgYXJlIHBhZGRlZCBieSBzcGVlY2hfcGFkX21zIGVhY2ggc2lkZS4KICAgIDpwYXJhbSBzcGVha2VyX2xhYmVsczogICAgICAgICAgVGhlIHNwZWFrZXIgbGFiZWxzIHRvIHVzZSBmb3IgdGhlIGRpYXJpemF0aW9uLiBJZiBub3QgZ2l2ZW4sIHRoZSBzcGVha2VycyB3aWxsIGJlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG5hbWVkICJzcGVha2VyXzAiLCAic3BlYWtlcl8xIiwgZXRjLgogICAgOnBhcmFtIHVzZV9tdWx0aXByb2Nlc3Npbmc6ICAgICBUaGUgbnVtYmVyIG9mIHdvcmtlcnMgdG8gdXNlIGZvciBtdWx0aXByb2Nlc3NpbmcuIElmIDAsIG5vIG11bHRpcHJvY2Vzc2luZyB3aWxsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGJlIHVzZWQuIERlZmF1bHQgaXMgMC4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICAgICAgVmVyYm9zaXR5LgogICAgIiIiCiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgR2V0IHRoZSBpbnB1dCBhdWRpbyBmaWxlcyB0byB0cmFuc2NyaWJlOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgYXVkaW8gZmlsZXMuIikKICAgIGF1ZGlvX2ZpbGVzID0gX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9ZGF0YV9wYXRoKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbihhdWRpb19maWxlcyl9IGF1ZGlvIGZpbGVzLiIpCgogICAgIyBJbml0aWFsaXplIHRoZSB0cmFuc2NyaXB0aW9uIHBpcGVsaW5lOgogICAgdmFkX2luaXRfa3dhcmdzID0gewogICAgICAgICJ1c2Vfb25ueCI6IHVzZV9vbm54LAogICAgICAgICJmb3JjZV9vbm54X2NwdSI6IGZvcmNlX29ubnhfY3B1LAogICAgICAgICJ0aHJlc2hvbGQiOiB0aHJlc2hvbGQsCiAgICAgICAgInNhbXBsaW5nX3JhdGUiOiBzYW1wbGluZ19yYXRlLAogICAgICAgICJtaW5fc3BlZWNoX2R1cmF0aW9uX21zIjogbWluX3NwZWVjaF9kdXJhdGlvbl9tcywKICAgICAgICAibWF4X3NwZWVjaF9kdXJhdGlvbl9zIjogbWF4X3NwZWVjaF9kdXJhdGlvbl9zLAogICAgICAgICJtaW5fc2lsZW5jZV9kdXJhdGlvbl9tcyI6IG1pbl9zaWxlbmNlX2R1cmF0aW9uX21zLAogICAgICAgICJ3aW5kb3dfc2l6ZV9zYW1wbGVzIjogd2luZG93X3NpemVfc2FtcGxlcywKICAgICAgICAic3BlZWNoX3BhZF9tcyI6IHNwZWVjaF9wYWRfbXMsCiAgICAgICAgInJldHVybl9zZWNvbmRzIjogVHJ1ZSwKICAgICAgICAicGVyX2NoYW5uZWwiOiBUcnVlLAogICAgfQoKICAgICMgQ3JlYXRlIHRoZSB0YXNrIGNyZWF0b3I6CiAgICB0YXNrX2NyZWF0b3IgPSBUYXNrQ3JlYXRvcigKICAgICAgICB0YXNrX3R5cGU9U3BlZWNoRGlhcml6YXRpb25UYXNrLCB0YXNrX2t3YXJncz17InNwZWFrZXJfbGFiZWxzIjogc3BlYWtlcl9sYWJlbHN9CiAgICApCgogICAgIyBSdW4gdGhlIHRyYW5zY3JpcHRpb246CiAgICBpZiB1c2VfbXVsdGlwcm9jZXNzaW5nOgogICAgICAgIHJlc3VsdHMgPSBfcGFyYWxsZWxfcnVuKAogICAgICAgICAgICBuX3dvcmtlcnM9dXNlX211bHRpcHJvY2Vzc2luZywKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEaWFyaXppbmciLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICByZXN1bHRzID0gX3J1bigKICAgICAgICAgICAgYXVkaW9fZmlsZXM9YXVkaW9fZmlsZXMsCiAgICAgICAgICAgIGRlc2NyaXB0aW9uPSJEaWFyaXppbmciLAogICAgICAgICAgICB2YWRfaW5pdF9rd2FyZ3M9dmFkX2luaXRfa3dhcmdzLAogICAgICAgICAgICB0YXNrX2NyZWF0b3I9dGFza19jcmVhdG9yLAogICAgICAgICAgICB2ZXJib3NlPXZlcmJvc2UsCiAgICAgICAgKQoKICAgICMgUHJvY2VzcyB0aGUgcmVzdWx0czoKICAgIHJldHVybiBfcHJvY2Vzc19yZXN1bHRzKHJlc3VsdHM9cmVzdWx0cywgdmVyYm9zZT12ZXJib3NlKQoKCmRlZiBfZ2V0X2F1ZGlvX2ZpbGVzKAogICAgZGF0YV9wYXRoOiBVbmlvbltQYXRoLCBzdHIsIGxpc3RdLAopIC0+IExpc3RbUGF0aF06CiAgICAiIiIKICAgIEdldCB0aGUgYXVkaW8gZmlsZXMgZnJvbSB0aGUgZGF0YSBwYXRoLiBJZiBhIHBhdGggdG8gYSBkaXJlY3RvcnkgaXMgZ2l2ZW4sIGFsbCBmaWxlcyBpbiB0aGUgZGlyZWN0b3J5IHdpbGwgYmUKICAgIGNvbGxlY3RlZC4KCiAgICA6cGFyYW0gZGF0YV9wYXRoOiBUaGUgZGF0YSBwYXRoIHRvIGNvbGxlY3QgdGhlIGF1ZGlvIGZpbGVzIGZyb20uCgogICAgOnJldHVybnM6IFRoZSBhdWRpbyBmaWxlcyBsaXN0LgogICAgIiIiCiAgICAjIENoZWNrIGlmIGdpdmVuIGEgbGlzdCBvZiBwYXRoczoKICAgIGlmIGlzaW5zdGFuY2UoZGF0YV9wYXRoLCBsaXN0KToKICAgICAgICBhdWRpb19maWxlcyA9IFtdCiAgICAgICAgZm9yIHBhdGggaW4gZGF0YV9wYXRoOgogICAgICAgICAgICBhdWRpb19maWxlcy5leHRlbmQoX2dldF9hdWRpb19maWxlcyhkYXRhX3BhdGg9cGF0aCkpCiAgICAgICAgcmV0dXJuIGF1ZGlvX2ZpbGVzCgogICAgIyBDaGVjayBpZiBnaXZlbiBhIHNpbmdsZSBzdHJpbmcgcGF0aCB0byBjYXN0IGl0IHRvIGEgYHBhdGhsaWIuUGF0aGA6CiAgICBpZiBpc2luc3RhbmNlKGRhdGFfcGF0aCwgc3RyKToKICAgICAgICBkYXRhX3BhdGggPSBQYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQoKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICBhdWRpb19maWxlcyA9IGxpc3QoZGF0YV9wYXRoLmdsb2IoIiouKiIpKQogICAgZWxpZiBkYXRhX3BhdGguaXNfZmlsZSgpOgogICAgICAgIGF1ZGlvX2ZpbGVzID0gW2RhdGFfcGF0aF0KICAgIGVsc2U6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJVbnJlY29nbml6ZWQgZGF0YSBwYXRoLiBUaGUgcGFyYW1ldGVyIGBkYXRhX3BhdGhgIG11c3QgYmUgYSB2YWxpZCBwYXRoIHRvIGVpdGhlciBhIGRpcmVjdG9yeSBwYXRoIG9yIGEgIgogICAgICAgICAgICBmImZpbGUuIEdpdmVuOiB7c3RyKGRhdGFfcGF0aCl9ICIKICAgICAgICApCgogICAgcmV0dXJuIGF1ZGlvX2ZpbGVzCgoKZGVmIF9ydW4oCiAgICBhdWRpb19maWxlczogTGlzdFtQYXRoXSwKICAgIGRlc2NyaXB0aW9uOiBzdHIsCiAgICB2YWRfaW5pdF9rd2FyZ3M6IGRpY3QsCiAgICB0YXNrX2NyZWF0b3I6IFRhc2tDcmVhdG9yLAogICAgdmVyYm9zZTogYm9vbCwKKSAtPiBMaXN0W1R1cGxlW2Jvb2wsIFR1cGxlW3N0ciwgbGlzdF1dXToKICAgICIiIgogICAgTG9hZCBhIFZBRCBhbmQgdXNlIGl0IHRvIGNvbXBsZXRlIHRoZSB0YXNrcyB0aGF0IHdpbGwgYmUgY3JlYXRlZCBvbiB0aGUgcHJvdmlkZWQgZmlsZXMgdXNpbmcgdGhlIGdpdmVuIHRhc2sgY3JlYXRvci4KCiAgICA6cGFyYW0gYXVkaW9fZmlsZXM6ICAgICBUaGUgYXVkaW8gZmlsZXMgdG8gdXNlLgogICAgOnBhcmFtIGRlc2NyaXB0aW9uOiAgICAgVGhlIGRlc2NyaXB0aW9uIHRvIHVzZSBmb3IgdGhlIHByb2dyZXNzIGJhci4KICAgIDpwYXJhbSB2YWRfaW5pdF9rd2FyZ3M6IFRoZSBWQUQgaW5pdGlhbGl6YXRpb24ga2V5d29yZCBhcmd1bWVudHMuCiAgICA6cGFyYW0gdGFza19jcmVhdG9yOiAgICBUaGUgdGFzayBjcmVhdG9yIHRvIHVzZSB0byBjcmVhdGUgdGhlIHRhc2tzLgogICAgOnBhcmFtIHZlcmJvc2U6ICAgICAgICAgVmVyYm9zaXR5LgoKICAgIDpyZXR1cm5zOiBUaGUgY29sbGVjdGVkIHJlc3VsdHMuCiAgICAiIiIKICAgICMgTG9hZCB0aGUgVkFEOgogICAgdmFkID0gVm9pY2VBY3Rpdml0eURldGVjdG9yKCoqdmFkX2luaXRfa3dhcmdzKQogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJMb2FkaW5nIHRoZSBWQUQgbW9kZWwuIikKICAgIHZhZC5sb2FkKCkKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJWQUQgbW9kZWwgbG9hZGVkLiIpCgogICAgIyBSdW4gdGhlIFZBRCBvbiB0aGUgYXVkaW8gZmlsZXMgYW5kIGNvbGxlY3QgdGhlIHJlc3VsdHM6CiAgICByZXN1bHRzID0gW10KICAgIGZvciBhdWRpb19maWxlIGluIHRxZG0oCiAgICAgICAgYXVkaW9fZmlsZXMsCiAgICAgICAgZGVzYz1kZXNjcmlwdGlvbiwKICAgICAgICB1bml0PSJmaWxlIiwKICAgICAgICB0b3RhbD1sZW4oYXVkaW9fZmlsZXMpLAogICAgICAgIGRpc2FibGU9bm90IHZlcmJvc2UsCiAgICApOgogICAgICAgIHRyeToKICAgICAgICAgICAgIyBDcmVhdGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2sgPSB0YXNrX2NyZWF0b3IuY3JlYXRlX3Rhc2soYXVkaW9fZmlsZT1hdWRpb19maWxlKQogICAgICAgICAgICAjIFJ1biB0aGUgZmlsZSB0aHJvdWdoIHRoZSBWQUQ6CiAgICAgICAgICAgIHNwZWVjaF90aW1lc3RhbXBzID0gdmFkLmRldGVjdF92b2ljZShhdWRpb19maWxlPWF1ZGlvX2ZpbGUpCiAgICAgICAgICAgICMgQ29tcGxldGUgdGhlIHRhc2s6CiAgICAgICAgICAgIHRhc2suZG9fdGFzayhzcGVlY2hfdGltZXN0YW1wcz1zcGVlY2hfdGltZXN0YW1wcykKICAgICAgICAgICAgIyBDb2xsZWN0IHRoZSByZXN1bHQ6CiAgICAgICAgICAgIHJlc3VsdHMuYXBwZW5kKChGYWxzZSwgdGFzay5nZXRfcmVzdWx0KCkpKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIENvbGxlY3QgdGhlIGVycm9yOgogICAgICAgICAgICByZXN1bHRzLmFwcGVuZCgoVHJ1ZSwgKGF1ZGlvX2ZpbGUubmFtZSwgc3RyKGV4Y2VwdGlvbikpKSkKCiAgICByZXR1cm4gcmVzdWx0cwoKCmRlZiBfcGFyYWxsZWxfcnVuKAogICAgbl93b3JrZXJzOiBpbnQsCiAgICBhdWRpb19maWxlczogTGlzdFtQYXRoXSwKICAgIGRlc2NyaXB0aW9uOiBzdHIsCiAgICB2YWRfaW5pdF9rd2FyZ3M6IGRpY3QsCiAgICB0YXNrX2NyZWF0b3I6IFRhc2tDcmVhdG9yLAogICAgdmVyYm9zZTogYm9vbCwKKSAtPiBMaXN0W1R1cGxlW2Jvb2wsIFR1cGxlW3N0ciwgbGlzdF1dXToKICAgICIiIgogICAgUnVuIG11bHRpcGxlIFZBRCB3b3JrZXJzIHdpdGggbXVsdGlwcm9jZXNzaW5nIHRvIGNvbXBsZXRlIHRoZSB0YXNrcyB0aGF0IHdpbGwgYmUgY3JlYXRlZCBvbiB0aGUgcHJvdmlkZWQgZmlsZXMgdXNpbmcKICAgIHRoZSBnaXZlbiB0YXNrIGNyZWF0b3IuCgogICAgOnBhcmFtIG5fd29ya2VyczogICAgICAgVGhlIG51bWJlciBvZiB3b3JrZXJzIHRvIHVzZS4KICAgIDpwYXJhbSBhdWRpb19maWxlczogICAgIFRoZSBhdWRpbyBmaWxlcyB0byB1c2UuCiAgICA6cGFyYW0gZGVzY3JpcHRpb246ICAgICBUaGUgZGVzY3JpcHRpb24gdG8gdXNlIGZvciB0aGUgcHJvZ3Jlc3MgYmFyLgogICAgOnBhcmFtIHZhZF9pbml0X2t3YXJnczogVGhlIFZBRCBpbml0aWFsaXphdGlvbiBrZXl3b3JkIGFyZ3VtZW50cy4KICAgIDpwYXJhbSB0YXNrX2NyZWF0b3I6ICAgIFRoZSB0YXNrIGNyZWF0b3IgdG8gdXNlIHRvIGNyZWF0ZSB0aGUgdGFza3MuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICBWZXJib3NpdHkuCgogICAgOnJldHVybnM6IFRoZSBjb2xsZWN0ZWQgcmVzdWx0cy4KICAgICIiIgogICAgIyBMb2FkIHRoZSBWQUQgKGRvd25sb2FkIG9uY2UsIGFuZCBpdCB3aWxsIGJlIGxvYWRlZCB0aGVuIHBlciBwcm9jZXNzIGxhdGVyIG9uKToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiTG9hZGluZyB0aGUgVkFEIG1vZGVsLiIpCiAgICB2YWQgPSBWb2ljZUFjdGl2aXR5RGV0ZWN0b3IoKip2YWRfaW5pdF9rd2FyZ3MpCiAgICB2YWQubG9hZCgpCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiVkFEIG1vZGVsIGxvYWRlZC4iKQoKICAgICMgQ2hlY2sgdGhlIG51bWJlciBvZiB3b3JrZXJzOgogICAgaWYgbl93b3JrZXJzID4gbGVuKGF1ZGlvX2ZpbGVzKToKICAgICAgICBfTE9HR0VSLndhcm5pbmcoCiAgICAgICAgICAgIGYiVGhlIG51bWJlciBvZiB3b3JrZXJzICh7bl93b3JrZXJzfSkgaXMgbGFyZ2VyIHRoYW4gdGhlIG51bWJlciBvZiBhdWRpbyBmaWxlcyAoe2xlbihhdWRpb19maWxlcyl9KS4gIgogICAgICAgICAgICBmIlNldHRpbmcgdGhlIG51bWJlciBvZiB3b3JrZXJzIHRvIHtsZW4oYXVkaW9fZmlsZXMpfS4iCiAgICAgICAgKQogICAgICAgIG5fd29ya2VycyA9IGxlbihhdWRpb19maWxlcykKCiAgICAjIEluaXRpYWxpemUgdGhlIG11bHRpcHJvY2Vzc2luZyBxdWV1ZXM6CiAgICB0YXNrc19xdWV1ZSA9IFF1ZXVlKCkKICAgIHJlc3VsdHNfcXVldWUgPSBRdWV1ZSgpCgogICAgIyBJbml0aWFsaXplIHRoZSBtdWx0aXByb2Nlc3NpbmcgcHJvY2Vzc2VzOgogICAgdGFza19jb21wbGV0aW9uX3Byb2Nlc3NlcyA9IFsKICAgICAgICBQcm9jZXNzKAogICAgICAgICAgICB0YXJnZXQ9X211bHRpcHJvY2Vzc2luZ19jb21wbGV0ZV90YXNrcywKICAgICAgICAgICAga3dhcmdzPXsKICAgICAgICAgICAgICAgICJ2YWRfaW5pdF9rd2FyZ3MiOiB2YWRfaW5pdF9rd2FyZ3MsCiAgICAgICAgICAgICAgICAidGFza3NfcXVldWUiOiB0YXNrc19xdWV1ZSwKICAgICAgICAgICAgICAgICJyZXN1bHRzX3F1ZXVlIjogcmVzdWx0c19xdWV1ZSwKICAgICAgICAgICAgfSwKICAgICAgICApCiAgICAgICAgZm9yIF8gaW4gcmFuZ2Uobl93b3JrZXJzKQogICAgXQoKICAgICMgU3RhcnQgdGhlIG11bHRpcHJvY2Vzc2luZyBwcm9jZXNzZXM6CiAgICBmb3IgcCBpbiB0YXNrX2NvbXBsZXRpb25fcHJvY2Vzc2VzOgogICAgICAgIHAuc3RhcnQoKQoKICAgICMgUHV0IHRoZSB0YXNrcyBpbiB0aGUgcXVldWU6CiAgICBmb3IgYXVkaW9fZmlsZSBpbiBhdWRpb19maWxlczoKICAgICAgICB0YXNrc19xdWV1ZS5wdXQodGFza19jcmVhdG9yLmNyZWF0ZV90YXNrKGF1ZGlvX2ZpbGU9YXVkaW9fZmlsZSkudG9fdHVwbGUoKSkKCiAgICAjIFB1dCB0aGUgc3RvcCBtYXJrcyBpbiB0aGUgcXVldWU6CiAgICBmb3IgXyBpbiByYW5nZShuX3dvcmtlcnMpOgogICAgICAgIHRhc2tzX3F1ZXVlLnB1dChfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSykKCiAgICAjIENvbGxlY3QgdGhlIHJlc3VsdHM6CiAgICByZXN1bHRzID0gW10KICAgIHN0b3BfbWFya3NfY291bnRlciA9IDAKICAgIHdpdGggdHFkbSgKICAgICAgICBkZXNjPWRlc2NyaXB0aW9uLAogICAgICAgIHVuaXQ9ImZpbGUiLAogICAgICAgIHRvdGFsPWxlbihhdWRpb19maWxlcyksCiAgICAgICAgZGlzYWJsZT1ub3QgdmVyYm9zZSwKICAgICkgYXMgcHJvZ3Jlc3NiYXI6CiAgICAgICAgd2hpbGUgVHJ1ZToKICAgICAgICAgICAgIyBHZXQgYSByZXN1bHQgZnJvbSB0aGUgcXVldWU6CiAgICAgICAgICAgIHJlc3VsdDogVHVwbGVbYm9vbCwgVHVwbGVbc3RyLCBsaXN0XV0gPSByZXN1bHRzX3F1ZXVlLmdldCgpCiAgICAgICAgICAgIGlmIHJlc3VsdCA9PSBfTVVMVElQUk9DRVNTSU5HX1NUT1BfTUFSSzoKICAgICAgICAgICAgICAgIHN0b3BfbWFya3NfY291bnRlciArPSAxCiAgICAgICAgICAgICAgICBpZiBzdG9wX21hcmtzX2NvdW50ZXIgPT0gbl93b3JrZXJzOgogICAgICAgICAgICAgICAgICAgIGJyZWFrCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAjIENvbGxlY3QgdGhlIHJlc3VsdDoKICAgICAgICAgICAgICAgIHJlc3VsdHMuYXBwZW5kKHJlc3VsdCkKICAgICAgICAgICAgICAgIHByb2dyZXNzYmFyLnVwZGF0ZSgxKQoKICAgICMgV2FpdCBmb3IgdGhlIHByb2Nlc3NlcyB0byBmaW5pc2g6CiAgICBmb3IgcCBpbiB0YXNrX2NvbXBsZXRpb25fcHJvY2Vzc2VzOgogICAgICAgIHAuam9pbigpCgogICAgcmV0dXJuIHJlc3VsdHMKCgpkZWYgX3Byb2Nlc3NfcmVzdWx0cygKICAgIHJlc3VsdHM6IExpc3RbVHVwbGVbYm9vbCwgVHVwbGVbc3RyLCBsaXN0XV1dLCB2ZXJib3NlOiBib29sCikgLT4gVHVwbGVbZGljdCwgZGljdF06CiAgICAiIiIKICAgIFByb2Nlc3MgdGhlIHJlc3VsdHMgb2YgdGhlIHRhc2tzLgoKICAgIDpwYXJhbSByZXN1bHRzOiBUaGUgcmVzdWx0cyB0byBwcm9jZXNzLgogICAgOnBhcmFtIHZlcmJvc2U6IFZlcmJvc2l0eS4KCiAgICA6cmV0dXJuczogVGhlIHByb2Nlc3NlZCByZXN1bHRzIGFzIGEgdHVwbGUgb2Ygc3VjY2Vzc2VzIGFuZCBlcnJvcnMuCiAgICAiIiIKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKCJTdW1tYXJpemluZyB0aGUgcmVzdWx0cy4iKQogICAgc3VjY2Vzc2VzID0ge30KICAgIGVycm9ycyA9IHt9CiAgICBmb3IgaXNfZXJyb3IsIHJlc3VsdCBpbiByZXN1bHRzOgogICAgICAgIGlmIGlzX2Vycm9yOgogICAgICAgICAgICBlcnJvcnNbcmVzdWx0WzBdXSA9IHJlc3VsdFsxXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHN1Y2Nlc3Nlc1tyZXN1bHRbMF1dID0gcmVzdWx0WzFdCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkRvbmUgKHtsZW4oc3VjY2Vzc2VzKX0ve2xlbihzdWNjZXNzZXMpICsgbGVuKGVycm9ycyl9KVxuIikKCiAgICByZXR1cm4gc3VjY2Vzc2VzLCBlcnJvcnMK - origin_filename: '' - image: '' - command: '' + code_origin: '' + base_image: mlrun/mlrun + filename: silero_vad.py entry_points: audio_file: - doc: Get the audio file of the task. - lineno: 43 - has_varargs: false outputs: - doc: The audio file of the task. type: Path parameters: - name: self - has_kwargs: false name: audio_file - do_task: - doc: Do the task on the given speech timestamps. The task will diarize the VAD - speech timestamps into speakers. - lineno: 94 + doc: Get the audio file of the task. + has_kwargs: false has_varargs: false + lineno: 42 + do_task: parameters: - name: self - name: speech_timestamps - type: List[List[Dict[str, int]]] + type: list[list[dict[str, int]]] doc: The speech timestamps per channel to do the task on as outputted from the VAD. - has_kwargs: false name: do_task - get_result: - doc: Get the result of the task. A tuple of the audio file name and the result. - lineno: 61 + doc: Do the task on the given speech timestamps. The task will diarize the VAD + speech timestamps into speakers. + has_kwargs: false has_varargs: false + lineno: 93 + get_result: outputs: - doc: The result of the task. - type: Tuple[str, list] + type: tuple[str, list] parameters: - name: self - has_kwargs: false name: get_result - to_tuple: - doc: Convert the task to a tuple to reconstruct it later (used for multiprocessing - to pass in queue). - lineno: 116 + doc: Get the result of the task. A tuple of the audio file name and the result. + has_kwargs: false has_varargs: false + lineno: 60 + to_tuple: outputs: - doc: The converted task. - type: Tuple[str, dict] + type: tuple[str, dict] parameters: - name: self - has_kwargs: false name: to_tuple - create_task: - doc: Create a task with the given audio file. - lineno: 146 + doc: Convert the task to a tuple to reconstruct it later (used for multiprocessing + to pass in queue). + has_kwargs: false has_varargs: false + lineno: 115 + create_task: outputs: - doc: The created task. type: BaseTask @@ -79,26 +77,26 @@ spec: - name: audio_file type: Path doc: The audio file to assign to the task. - has_kwargs: false name: create_task - from_tuple: - doc: Create a task from a tuple of the audio file name and the task kwargs. - lineno: 157 + doc: Create a task with the given audio file. + has_kwargs: false has_varargs: false + lineno: 145 + from_tuple: outputs: - doc: The created task. type: BaseTask parameters: - name: cls - name: task_tuple - type: Tuple[str, dict] + type: tuple[str, dict] doc: The task tuple to create the task from. - has_kwargs: false name: from_tuple - load: - doc: Load the VAD model. - lineno: 234 + doc: Create a task from a tuple of the audio file name and the task kwargs. + has_kwargs: false has_varargs: false + lineno: 156 + load: parameters: - name: self - name: force_reload @@ -106,24 +104,14 @@ spec: doc: Whether to force reload the model even if it was already loaded. Default is True. default: true - has_kwargs: false name: load - detect_voice: - doc: "Perform voice activity detection on given audio files using the silero\ - \ VAD model -\nhttps://github.com/snakers4/silero-vad. The end result is a\ - \ dictionary with the file names as keys and their\nVAD timestamps dictionaries\ - \ as value.\n\nFor example::\n\n {\n \"file_1.wav\": [\n \ - \ {\"start\": 0, \"end\": 16000},\n {\"start\": 16000, \"end\"\ - : 32000},\n {\"start\": 32000, \"end\": 48000},\n ...\n\ - \ ],\n \"file_2.wav\": [\n {\"start\": 0, \"end\"\ - : 16000},\n {\"start\": 16000, \"end\": 32000},\n {\"\ - start\": 32000, \"end\": 48000},\n ...\n ],\n ...\n\ - \ }" - lineno: 393 + doc: Load the VAD model. + has_kwargs: false has_varargs: false + lineno: 233 + detect_voice: parameters: - name: data_path - type: Union[str, Path, List[Union[str, Path]]] doc: The path to the audio files to diarize. Can be a path to a single file, a path to a directory or a list of paths to files. - name: use_onnx @@ -188,25 +176,23 @@ spec: type: bool doc: Verbosity. default: false - has_kwargs: false name: detect_voice - diarize: - doc: "Perform speech diarization on given audio files using the silero VAD model\ - \ - https://github.com/snakers4/silero-vad.\nThe speech diarization is performed\ - \ per channel so that each channel in the audio belong to a different speaker.\ - \ The\nend result is a dictionary with the file names as keys and their diarization\ - \ as value. A diarization is a list\nof tuples: (start, end, speaker_label).\n\ - \nFor example::\n\n {\n \"file_1.wav\": [\n (0.0, 1.0,\ - \ \"speaker_0\"),\n (1.0, 2.0, \"speaker_1\"),\n (2.0,\ - \ 3.0, \"speaker_0\"),\n ...\n ],\n \"file_2.wav\"\ - : [\n (0.0, 1.0, \"speaker_0\"),\n (1.0, 2.0, \"speaker_1\"\ - ),\n (2.0, 3.0, \"speaker_0\"),\n ...\n ],\n\ - \ ...\n }" - lineno: 517 + doc: "Perform voice activity detection on given audio files using the silero\ + \ VAD model -\nhttps://github.com/snakers4/silero-vad. The end result is a\ + \ dictionary with the file names as keys and their\nVAD timestamps dictionaries\ + \ as value.\n\nFor example::\n\n {\n \"file_1.wav\": [\n \ + \ {\"start\": 0, \"end\": 16000},\n {\"start\": 16000, \"end\"\ + : 32000},\n {\"start\": 32000, \"end\": 48000},\n ...\n\ + \ ],\n \"file_2.wav\": [\n {\"start\": 0, \"end\"\ + : 16000},\n {\"start\": 16000, \"end\": 32000},\n {\"\ + start\": 32000, \"end\": 48000},\n ...\n ],\n ...\n\ + \ }" + has_kwargs: false has_varargs: false + lineno: 392 + diarize: parameters: - name: data_path - type: Union[str, Path, List[Union[str, Path]]] doc: The path to the audio files to diarize. Can be a path to a single file, a path to a directory or a list of paths to files. - name: use_onnx @@ -253,7 +239,7 @@ spec: doc: Final speech chunks are padded by speech_pad_ms each side. default: 30 - name: speaker_labels - type: List[str] + type: list[str] doc: The speaker labels to use for the diarization. If not given, the speakers will be named "speaker_0", "speaker_1", etc. default: null @@ -266,8 +252,21 @@ spec: type: bool doc: Verbosity. default: false - has_kwargs: false name: diarize - disable_auto_mount: false + doc: "Perform speech diarization on given audio files using the silero VAD model\ + \ - https://github.com/snakers4/silero-vad.\nThe speech diarization is performed\ + \ per channel so that each channel in the audio belong to a different speaker.\ + \ The\nend result is a dictionary with the file names as keys and their diarization\ + \ as value. A diarization is a list\nof tuples: (start, end, speaker_label).\n\ + \nFor example::\n\n {\n \"file_1.wav\": [\n (0.0, 1.0,\ + \ \"speaker_0\"),\n (1.0, 2.0, \"speaker_1\"),\n (2.0,\ + \ 3.0, \"speaker_0\"),\n ...\n ],\n \"file_2.wav\"\ + : [\n (0.0, 1.0, \"speaker_0\"),\n (1.0, 2.0, \"speaker_1\"\ + ),\n (2.0, 3.0, \"speaker_0\"),\n ...\n ],\n\ + \ ...\n }" + has_kwargs: false + has_varargs: false + lineno: 516 + command: '' + description: Silero VAD (Voice Activity Detection) functions. default_handler: detect_voice -kind: job diff --git a/functions/src/silero_vad/silero_vad.py b/functions/src/silero_vad/silero_vad.py index a477d4ecf..877f49972 100644 --- a/functions/src/silero_vad/silero_vad.py +++ b/functions/src/silero_vad/silero_vad.py @@ -15,7 +15,6 @@ from multiprocessing import Process, Queue from pathlib import Path from types import FunctionType -from typing import Dict, List, Tuple, Type, Union import torch import torchaudio @@ -49,7 +48,7 @@ def audio_file(self) -> Path: return self._audio_file def do_task( - self, speech_timestamps: Union[List[Dict[str, int]], List[List[Dict[str, int]]]] + self, speech_timestamps: list[dict[str, int]] | list[list[dict[str, int]]] ): """ Do the task on the given speech timestamps. The base task will simply save the speech timestamps as the result. @@ -58,7 +57,7 @@ def do_task( """ self._result = speech_timestamps - def get_result(self) -> Tuple[str, list]: + def get_result(self) -> tuple[str, list]: """ Get the result of the task. A tuple of the audio file name and the result. @@ -66,7 +65,7 @@ def get_result(self) -> Tuple[str, list]: """ return self._audio_file.name, self._result - def to_tuple(self) -> Tuple[str, dict]: + def to_tuple(self) -> tuple[str, dict]: """ Convert the task to a tuple to reconstruct it later (used for multiprocessing to pass in queue). @@ -80,7 +79,7 @@ class SpeechDiarizationTask(BaseTask): A speech diarization task. The task will diarize the VAD speech timestamps into speakers. """ - def __init__(self, audio_file: Path, speaker_labels: List[str]): + def __init__(self, audio_file: Path, speaker_labels: list[str]): """ Initialize the speech diarization task. @@ -91,7 +90,7 @@ def __init__(self, audio_file: Path, speaker_labels: List[str]): super().__init__(audio_file=audio_file) self._speaker_labels = speaker_labels - def do_task(self, speech_timestamps: List[List[Dict[str, int]]]): + def do_task(self, speech_timestamps: list[list[dict[str, int]]]): """ Do the task on the given speech timestamps. The task will diarize the VAD speech timestamps into speakers. @@ -113,7 +112,7 @@ def do_task(self, speech_timestamps: List[List[Dict[str, int]]]): speech_diarization.sort() self._result = speech_diarization - def to_tuple(self) -> Tuple[str, dict]: + def to_tuple(self) -> tuple[str, dict]: """ Convert the task to a tuple to reconstruct it later (used for multiprocessing to pass in queue). @@ -134,7 +133,7 @@ class TaskCreator: SpeechDiarizationTask.__name__: SpeechDiarizationTask, } - def __init__(self, task_type: Type[BaseTask], task_kwargs: dict = None): + def __init__(self, task_type: type[BaseTask], task_kwargs: dict = None): """ Initialize the task creator. :param task_type: The task type - a `BaseTask` subclass. @@ -154,7 +153,7 @@ def create_task(self, audio_file: Path) -> BaseTask: return self._task_type(audio_file=audio_file, **self._task_kwargs) @classmethod - def from_tuple(cls, task_tuple: Tuple[str, dict]) -> BaseTask: + def from_tuple(cls, task_tuple: tuple[str, dict]) -> BaseTask: """ Create a task from a tuple of the audio file name and the task kwargs. @@ -256,7 +255,7 @@ def load(self, force_reload: bool = True): def detect_voice( self, audio_file: Path, - ) -> Union[List[Dict[str, int]], List[List[Dict[str, int]]]]: + ) -> list[dict[str, int]] | list[list[dict[str, int]]]: """ Infer the audio through the VAD model and return the speech timestamps. @@ -359,7 +358,7 @@ def _multiprocessing_complete_tasks( # Start listening to the tasks queue: while True: # Get the task: - task: Tuple[str, dict] = tasks_queue.get() + task: tuple[str, dict] = tasks_queue.get() if task == _MULTIPROCESSING_STOP_MARK: break try: @@ -392,7 +391,7 @@ def _multiprocessing_complete_tasks( def detect_voice( # Input kwargs: - data_path: Union[str, Path, List[Union[str, Path]]], + data_path: str | Path | list[str | Path], # Model loading kwargs: use_onnx: bool = True, force_onnx_cpu: bool = True, @@ -516,7 +515,7 @@ def detect_voice( def diarize( # Input / Output kwargs: - data_path: Union[str, Path, List[Union[str, Path]]], + data_path: str | Path | list[str | Path], # Model loading kwargs: use_onnx: bool = True, force_onnx_cpu: bool = True, @@ -529,7 +528,7 @@ def diarize( window_size_samples: int = 512, speech_pad_ms: int = 30, # Diarization kwargs: - speaker_labels: List[str] = None, + speaker_labels: list[str] = None, # Other kwargs: use_multiprocessing: int = 0, verbose: bool = False, @@ -640,8 +639,8 @@ def diarize( def _get_audio_files( - data_path: Union[Path, str, list], -) -> List[Path]: + data_path: Path | str | list, +) -> list[Path]: """ Get the audio files from the data path. If a path to a directory is given, all files in the directory will be collected. @@ -677,12 +676,12 @@ def _get_audio_files( def _run( - audio_files: List[Path], + audio_files: list[Path], description: str, vad_init_kwargs: dict, task_creator: TaskCreator, verbose: bool, -) -> List[Tuple[bool, Tuple[str, list]]]: +) -> list[tuple[bool, tuple[str, list]]]: """ Load a VAD and use it to complete the tasks that will be created on the provided files using the given task creator. @@ -697,7 +696,7 @@ def _run( # Load the VAD: vad = VoiceActivityDetector(**vad_init_kwargs) if verbose: - _LOGGER.info(f"Loading the VAD model.") + _LOGGER.info("Loading the VAD model.") vad.load() if verbose: _LOGGER.info("VAD model loaded.") @@ -729,12 +728,12 @@ def _run( def _parallel_run( n_workers: int, - audio_files: List[Path], + audio_files: list[Path], description: str, vad_init_kwargs: dict, task_creator: TaskCreator, verbose: bool, -) -> List[Tuple[bool, Tuple[str, list]]]: +) -> list[tuple[bool, tuple[str, list]]]: """ Run multiple VAD workers with multiprocessing to complete the tasks that will be created on the provided files using the given task creator. @@ -750,7 +749,7 @@ def _parallel_run( """ # Load the VAD (download once, and it will be loaded then per process later on): if verbose: - _LOGGER.info(f"Loading the VAD model.") + _LOGGER.info("Loading the VAD model.") vad = VoiceActivityDetector(**vad_init_kwargs) vad.load() if verbose: @@ -804,7 +803,7 @@ def _parallel_run( ) as progressbar: while True: # Get a result from the queue: - result: Tuple[bool, Tuple[str, list]] = results_queue.get() + result: tuple[bool, tuple[str, list]] = results_queue.get() if result == _MULTIPROCESSING_STOP_MARK: stop_marks_counter += 1 if stop_marks_counter == n_workers: @@ -822,8 +821,8 @@ def _parallel_run( def _process_results( - results: List[Tuple[bool, Tuple[str, list]]], verbose: bool -) -> Tuple[dict, dict]: + results: list[tuple[bool, tuple[str, list]]], verbose: bool +) -> tuple[dict, dict]: """ Process the results of the tasks. diff --git a/functions/src/sklearn_classifier/function.yaml b/functions/src/sklearn_classifier/function.yaml index 205df697d..80b257214 100644 --- a/functions/src/sklearn_classifier/function.yaml +++ b/functions/src/sklearn_classifier/function.yaml @@ -1,10 +1,23 @@ +metadata: + tag: '' + name: sklearn-classifier + categories: + - machine-learning + - model-training +verbose: false +kind: job spec: image: mlrun/mlrun - description: train any classifier using scikit-learn's API - default_handler: train_model + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgp3YXJuaW5ncy5zaW1wbGVmaWx0ZXIoYWN0aW9uPSJpZ25vcmUiLCBjYXRlZ29yeT1GdXR1cmVXYXJuaW5nKQoKCmltcG9ydCBwYW5kYXMgYXMgcGQKZnJvbSBjbG91ZHBpY2tsZSBpbXBvcnQgZHVtcHMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLm1sdXRpbHMuZGF0YSBpbXBvcnQgZ2V0X3NhbXBsZSwgZ2V0X3NwbGl0cwpmcm9tIG1scnVuLm1sdXRpbHMubW9kZWxzIGltcG9ydCBldmFsX21vZGVsX3YyLCBnZW5fc2tsZWFybl9tb2RlbApmcm9tIG1scnVuLnV0aWxzLmhlbHBlcnMgaW1wb3J0IGNyZWF0ZV9jbGFzcwoKCmRlZiB0cmFpbl9tb2RlbCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWxfcGtnX2NsYXNzOiBzdHIsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gImxhYmVscyIsCiAgICBlbmNvZGVfY29sczogbGlzdFtzdHJdID0gW10sCiAgICBzYW1wbGU6IGludCA9IC0xLAogICAgdGVzdF9zaXplOiBmbG9hdCA9IDAuMzAsCiAgICB0cmFpbl92YWxfc3BsaXQ6IGZsb2F0ID0gMC43MCwKICAgIHRlc3Rfc2V0X2tleTogc3RyID0gInRlc3Rfc2V0IiwKICAgIG1vZGVsX2V2YWx1YXRvcj1Ob25lLAogICAgbW9kZWxzX2Rlc3Q6IHN0ciA9ICIiLAogICAgcGxvdHNfZGVzdDogc3RyID0gInBsb3RzIiwKICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICBtb2RlbF9wa2dfZmlsZTogc3RyID0gIiIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDEsCikgLT4gTm9uZToKICAgICIiInRyYWluIGEgY2xhc3NpZmllcgoKICAgIEFuIG9wdGlvbmFsIGN1dG9tIG1vZGVsIGV2YWx1YXRvciBjYW4gYmUgc3VwcGxpZWQgdGhhdCBzaG91bGQgaGF2ZSB0aGUgc2lnbmF0dXJlOgogICAgYG15X2N1c3RvbV9ldmFsdWF0b3IoY29udGV4dCwgeHZhbGlkLCB5dmFsaWQsIG1vZGVsKWAgYW5kIHJldHVybiBhIGRpY3Rpb25hcnkgb2YKICAgIHNjYWxhciAicmVzdWx0cyIsIGEgInBsb3RzIiBrZXlzIHdpdGggYSBsaXN0IG9mIFBsb3RBcnRpZmFjdHMsIGFuZAogICAgYW5kICJ0YWJsZXMiIGtleSBjb250YWluaW5nIGEgcmV0dXJuZWQgbGlzdCBvZiBUYWJsZUFydGlmYWN0cy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgIHRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxfcGtnX2NsYXNzOiAgIHRoZSBtb2RlbCB0byB0cmFpbiwgZS5nLCAic2tsZWFybi5uZXVyYWxfbmV0d29ya3MuTUxQQ2xhc3NpZmllciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9yIGpzb24gbW9kZWwgY29uZmlnCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICgiZGF0YSIpIG5hbWUgb2YgcmF3IGRhdGEgZmlsZQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICBncm91bmQtdHJ1dGggKHkpIGxhYmVscwogICAgOnBhcmFtIGVuY29kZV9jb2xzOiAgICAgICBkaWN0aW9uYXJ5IG9mIG5hbWVzIGFuZCBwcmVmaXhlcyBmb3IgY29sdW1ucyB0aGF0IGFyZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0byBob3QgYmUgZW5jb2RlZC4KICAgIDpwYXJhbSBzYW1wbGU6ICAgICAgICAgICAgU2VsZWN0cyB0aGUgZmlyc3QgbiByb3dzLCBvciBzZWxlY3QgYSBzYW1wbGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RhcnRpbmcgZnJvbSB0aGUgZmlyc3QuIElmIG5lZ2F0aXZlIDwtMSwgc2VsZWN0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGEgcmFuZG9tIHNhbXBsZQogICAgOnBhcmFtIHRlc3Rfc2l6ZTogICAgICAgICAoMC4wNSkgdGVzdCBzZXQgc2l6ZQogICAgOnBhcmFtIHRyYWluX3ZhbF9zcGxpdDogICAoMC43NSkgT25jZSB0aGUgdGVzdCBzZXQgaGFzIGJlZW4gcmVtb3ZlZCB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJhaW5pbmcgc2V0IGdldHMgdGhpcyBwcm9wb3J0aW9uLgogICAgOnBhcmFtIHRlc3Rfc2V0X2tleTogICAgICBrZXkgb2YgaGVsZCBvdXQgZGF0YSBpbiBhcnRpZmFjdCBzdG9yZQogICAgOnBhcmFtIG1vZGVsX2V2YWx1YXRvcjogICAoTm9uZSkgYSBjdXN0b20gbW9kZWwgZXZhbHVhdG9yIGNhbiBiZSBzcGVjaWZpZWQKICAgIDpwYXJhbSBtb2RlbHNfZGVzdDogICAgICAgKCIiKSBtb2RlbHMgc3ViZm9sZGVyIG9uIGFydGlmYWN0IHBhdGgKICAgIDpwYXJhbSBwbG90c19kZXN0OiAgICAgICAgcGxvdCBzdWJmb2xkZXIgb24gYXJ0aWZhY3QgcGF0aAogICAgOnBhcmFtIGZpbGVfZXh0OiAgICAgICAgICAoInBhcnF1ZXQiKSBmb3JtYXQgZm9yIHRlc3Rfc2V0X2tleSBob2xkIG91dCBkYXRhCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICgxKSBza2xlYXJuIHJuZyBzZWVkCgogICAgIiIiCiAgICBtb2RlbHNfZGVzdCA9IG1vZGVsc19kZXN0IG9yICJtb2RlbCIKCiAgICByYXcsIGxhYmVscywgaGVhZGVyID0gZ2V0X3NhbXBsZShkYXRhc2V0LCBzYW1wbGUsIGxhYmVsX2NvbHVtbikKCiAgICBpZiBlbmNvZGVfY29sczoKICAgICAgICByYXcgPSBwZC5nZXRfZHVtbWllcygKICAgICAgICAgICAgcmF3LAogICAgICAgICAgICBjb2x1bW5zPWxpc3QoZW5jb2RlX2NvbHMua2V5cygpKSwKICAgICAgICAgICAgcHJlZml4PWxpc3QoZW5jb2RlX2NvbHMudmFsdWVzKCkpLAogICAgICAgICAgICBkcm9wX2ZpcnN0PVRydWUsCiAgICAgICAgKQoKICAgICh4dHJhaW4sIHl0cmFpbiksICh4dmFsaWQsIHl2YWxpZCksICh4dGVzdCwgeXRlc3QpID0gZ2V0X3NwbGl0cygKICAgICAgICByYXcsIGxhYmVscywgMywgdGVzdF9zaXplLCAxIC0gdHJhaW5fdmFsX3NwbGl0LCByYW5kb21fc3RhdGUKICAgICkKCiAgICB0ZXN0X3NldCA9IHBkLmNvbmNhdChbeHRlc3QsIHl0ZXN0LnRvX2ZyYW1lKCldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgIHRlc3Rfc2V0X2tleSwKICAgICAgICBkZj10ZXN0X3NldCwKICAgICAgICBmb3JtYXQ9ZmlsZV9leHQsCiAgICAgICAgaW5kZXg9RmFsc2UsCiAgICAgICAgbGFiZWxzPXsiZGF0YS10eXBlIjogImhlbGQtb3V0In0sCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1jb250ZXh0LmFydGlmYWN0X3N1YnBhdGgoImRhdGEiKSwKICAgICkKCiAgICBtb2RlbF9jb25maWcgPSBnZW5fc2tsZWFybl9tb2RlbChtb2RlbF9wa2dfY2xhc3MsIGNvbnRleHQucGFyYW1ldGVycy5pdGVtcygpKQoKICAgIG1vZGVsX2NvbmZpZ1siRklUIl0udXBkYXRlKHsiWCI6IHh0cmFpbiwgInkiOiB5dHJhaW4udmFsdWVzfSkKCiAgICBDbGFzc2lmaWVyQ2xhc3MgPSBjcmVhdGVfY2xhc3MobW9kZWxfY29uZmlnWyJNRVRBIl1bImNsYXNzIl0pCgogICAgbW9kZWwgPSBDbGFzc2lmaWVyQ2xhc3MoKiptb2RlbF9jb25maWdbIkNMQVNTIl0pCgogICAgbW9kZWwuZml0KCoqbW9kZWxfY29uZmlnWyJGSVQiXSkKCiAgICBhcnRpZmFjdF9wYXRoID0gY29udGV4dC5hcnRpZmFjdF9zdWJwYXRoKG1vZGVsc19kZXN0KQogICAgcGxvdHNfcGF0aCA9IGNvbnRleHQuYXJ0aWZhY3Rfc3VicGF0aChtb2RlbHNfZGVzdCwgcGxvdHNfZGVzdCkKICAgIGlmIG1vZGVsX2V2YWx1YXRvcjoKICAgICAgICBldmFsX21ldHJpY3MgPSBtb2RlbF9ldmFsdWF0b3IoCiAgICAgICAgICAgIGNvbnRleHQsIHh2YWxpZCwgeXZhbGlkLCBtb2RlbCwgcGxvdHNfYXJ0aWZhY3RfcGF0aD1wbG90c19wYXRoCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICBldmFsX21ldHJpY3MgPSBldmFsX21vZGVsX3YyKAogICAgICAgICAgICBjb250ZXh0LCB4dmFsaWQsIHl2YWxpZCwgbW9kZWwsIHBsb3RzX2FydGlmYWN0X3BhdGg9cGxvdHNfcGF0aAogICAgICAgICkKCiAgICBrd2FyZ3MgPSB7InRyYWluaW5nX3NldCI6IHRlc3Rfc2V0LCAibGFiZWxfY29sdW1uIjogbGFiZWxfY29sdW1ufQogICAgc3BsaXQgPSBtb2RlbF9wa2dfY2xhc3MucnNwbGl0KCIuIiwgMSkKICAgIGlmIHNwbGl0IGFuZCBsZW4oc3BsaXQpID09IDI6CiAgICAgICAga3dhcmdzWyJhbGdvcml0aG0iXSA9IHNwbGl0WzFdCgogICAgaWYgZGF0YXNldC5tZXRhIGFuZCBkYXRhc2V0Lm1ldGEua2luZCA9PSAiRmVhdHVyZVZlY3RvciI6CiAgICAgICAga3dhcmdzWyJmZWF0dXJlX3ZlY3RvciJdID0gZGF0YXNldC5tZXRhLnVyaQoKICAgIGNvbnRleHQuc2V0X2xhYmVsKCJjbGFzcyIsIG1vZGVsX3BrZ19jbGFzcykKICAgIGNvbnRleHQubG9nX21vZGVsKAogICAgICAgICJtb2RlbCIsCiAgICAgICAgYm9keT1kdW1wcyhtb2RlbCksCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1hcnRpZmFjdF9wYXRoLAogICAgICAgIGV4dHJhX2RhdGE9ZXZhbF9tZXRyaWNzLAogICAgICAgIG1vZGVsX2ZpbGU9Im1vZGVsLnBrbCIsCiAgICAgICAgbWV0cmljcz1jb250ZXh0LnJlc3VsdHMsCiAgICAgICAgbGFiZWxzPXsiY2xhc3MiOiBtb2RlbF9wa2dfY2xhc3N9LAogICAgICAgIGZyYW1ld29yaz0ic2tsZWFybiIsCiAgICAgICAgKiprd2FyZ3MsCiAgICApCg== + code_origin: '' + filename: sklearn_classifier.py entry_points: train_model: - has_varargs: false + outputs: + - type: None parameters: - name: context type: MLClientCtx @@ -21,14 +34,14 @@ spec: doc: ground-truth (y) labels default: labels - name: encode_cols - type: List[str] + type: list[str] doc: dictionary of names and prefixes for columns that are to hot be encoded. default: [] - name: sample type: int doc: Selects the first n rows, or select a sample starting from the first. If negative <-1, select a random sample - default: + default: - name: test_size type: float doc: (0.05) test set size @@ -76,21 +89,9 @@ spec: scalar "results", a "plots" keys with a list of PlotArtifacts, and and "tables" key containing a returned list of TableArtifacts.' - outputs: - - type: None - lineno: 32 has_kwargs: false - disable_auto_mount: false - build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgp3YXJuaW5ncy5zaW1wbGVmaWx0ZXIoYWN0aW9uPSJpZ25vcmUiLCBjYXRlZ29yeT1GdXR1cmVXYXJuaW5nKQoKCmZyb20gY2xvdWRwaWNrbGUgaW1wb3J0IGR1bXBzCmltcG9ydCBwYW5kYXMgYXMgcGQKZnJvbSB0eXBpbmcgaW1wb3J0IExpc3QKZnJvbSBtbHJ1bi5leGVjdXRpb24gaW1wb3J0IE1MQ2xpZW50Q3R4CmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLm1sdXRpbHMuZGF0YSBpbXBvcnQgZ2V0X3NhbXBsZSwgZ2V0X3NwbGl0cwpmcm9tIG1scnVuLm1sdXRpbHMubW9kZWxzIGltcG9ydCBnZW5fc2tsZWFybl9tb2RlbCwgZXZhbF9tb2RlbF92Mgpmcm9tIG1scnVuLnV0aWxzLmhlbHBlcnMgaW1wb3J0IGNyZWF0ZV9jbGFzcwoKCmRlZiB0cmFpbl9tb2RlbCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWxfcGtnX2NsYXNzOiBzdHIsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gImxhYmVscyIsCiAgICBlbmNvZGVfY29sczogTGlzdFtzdHJdID0gW10sCiAgICBzYW1wbGU6IGludCA9IC0xLAogICAgdGVzdF9zaXplOiBmbG9hdCA9IDAuMzAsCiAgICB0cmFpbl92YWxfc3BsaXQ6IGZsb2F0ID0gMC43MCwKICAgIHRlc3Rfc2V0X2tleTogc3RyID0gInRlc3Rfc2V0IiwKICAgIG1vZGVsX2V2YWx1YXRvcj1Ob25lLAogICAgbW9kZWxzX2Rlc3Q6IHN0ciA9ICIiLAogICAgcGxvdHNfZGVzdDogc3RyID0gInBsb3RzIiwKICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICBtb2RlbF9wa2dfZmlsZTogc3RyID0gIiIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDEsCikgLT4gTm9uZToKICAgICIiInRyYWluIGEgY2xhc3NpZmllcgoKICAgIEFuIG9wdGlvbmFsIGN1dG9tIG1vZGVsIGV2YWx1YXRvciBjYW4gYmUgc3VwcGxpZWQgdGhhdCBzaG91bGQgaGF2ZSB0aGUgc2lnbmF0dXJlOgogICAgYG15X2N1c3RvbV9ldmFsdWF0b3IoY29udGV4dCwgeHZhbGlkLCB5dmFsaWQsIG1vZGVsKWAgYW5kIHJldHVybiBhIGRpY3Rpb25hcnkgb2YKICAgIHNjYWxhciAicmVzdWx0cyIsIGEgInBsb3RzIiBrZXlzIHdpdGggYSBsaXN0IG9mIFBsb3RBcnRpZmFjdHMsIGFuZAogICAgYW5kICJ0YWJsZXMiIGtleSBjb250YWluaW5nIGEgcmV0dXJuZWQgbGlzdCBvZiBUYWJsZUFydGlmYWN0cy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgIHRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxfcGtnX2NsYXNzOiAgIHRoZSBtb2RlbCB0byB0cmFpbiwgZS5nLCAic2tsZWFybi5uZXVyYWxfbmV0d29ya3MuTUxQQ2xhc3NpZmllciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9yIGpzb24gbW9kZWwgY29uZmlnCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICgiZGF0YSIpIG5hbWUgb2YgcmF3IGRhdGEgZmlsZQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICBncm91bmQtdHJ1dGggKHkpIGxhYmVscwogICAgOnBhcmFtIGVuY29kZV9jb2xzOiAgICAgICBkaWN0aW9uYXJ5IG9mIG5hbWVzIGFuZCBwcmVmaXhlcyBmb3IgY29sdW1ucyB0aGF0IGFyZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0byBob3QgYmUgZW5jb2RlZC4KICAgIDpwYXJhbSBzYW1wbGU6ICAgICAgICAgICAgU2VsZWN0cyB0aGUgZmlyc3QgbiByb3dzLCBvciBzZWxlY3QgYSBzYW1wbGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RhcnRpbmcgZnJvbSB0aGUgZmlyc3QuIElmIG5lZ2F0aXZlIDwtMSwgc2VsZWN0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGEgcmFuZG9tIHNhbXBsZQogICAgOnBhcmFtIHRlc3Rfc2l6ZTogICAgICAgICAoMC4wNSkgdGVzdCBzZXQgc2l6ZQogICAgOnBhcmFtIHRyYWluX3ZhbF9zcGxpdDogICAoMC43NSkgT25jZSB0aGUgdGVzdCBzZXQgaGFzIGJlZW4gcmVtb3ZlZCB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJhaW5pbmcgc2V0IGdldHMgdGhpcyBwcm9wb3J0aW9uLgogICAgOnBhcmFtIHRlc3Rfc2V0X2tleTogICAgICBrZXkgb2YgaGVsZCBvdXQgZGF0YSBpbiBhcnRpZmFjdCBzdG9yZQogICAgOnBhcmFtIG1vZGVsX2V2YWx1YXRvcjogICAoTm9uZSkgYSBjdXN0b20gbW9kZWwgZXZhbHVhdG9yIGNhbiBiZSBzcGVjaWZpZWQKICAgIDpwYXJhbSBtb2RlbHNfZGVzdDogICAgICAgKCIiKSBtb2RlbHMgc3ViZm9sZGVyIG9uIGFydGlmYWN0IHBhdGgKICAgIDpwYXJhbSBwbG90c19kZXN0OiAgICAgICAgcGxvdCBzdWJmb2xkZXIgb24gYXJ0aWZhY3QgcGF0aAogICAgOnBhcmFtIGZpbGVfZXh0OiAgICAgICAgICAoInBhcnF1ZXQiKSBmb3JtYXQgZm9yIHRlc3Rfc2V0X2tleSBob2xkIG91dCBkYXRhCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICgxKSBza2xlYXJuIHJuZyBzZWVkCgogICAgIiIiCiAgICBtb2RlbHNfZGVzdCA9IG1vZGVsc19kZXN0IG9yICJtb2RlbCIKCiAgICByYXcsIGxhYmVscywgaGVhZGVyID0gZ2V0X3NhbXBsZShkYXRhc2V0LCBzYW1wbGUsIGxhYmVsX2NvbHVtbikKCiAgICBpZiBlbmNvZGVfY29sczoKICAgICAgICByYXcgPSBwZC5nZXRfZHVtbWllcygKICAgICAgICAgICAgcmF3LAogICAgICAgICAgICBjb2x1bW5zPWxpc3QoZW5jb2RlX2NvbHMua2V5cygpKSwKICAgICAgICAgICAgcHJlZml4PWxpc3QoZW5jb2RlX2NvbHMudmFsdWVzKCkpLAogICAgICAgICAgICBkcm9wX2ZpcnN0PVRydWUsCiAgICAgICAgKQoKICAgICh4dHJhaW4sIHl0cmFpbiksICh4dmFsaWQsIHl2YWxpZCksICh4dGVzdCwgeXRlc3QpID0gZ2V0X3NwbGl0cygKICAgICAgICByYXcsIGxhYmVscywgMywgdGVzdF9zaXplLCAxIC0gdHJhaW5fdmFsX3NwbGl0LCByYW5kb21fc3RhdGUKICAgICkKCiAgICB0ZXN0X3NldCA9IHBkLmNvbmNhdChbeHRlc3QsIHl0ZXN0LnRvX2ZyYW1lKCldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgIHRlc3Rfc2V0X2tleSwKICAgICAgICBkZj10ZXN0X3NldCwKICAgICAgICBmb3JtYXQ9ZmlsZV9leHQsCiAgICAgICAgaW5kZXg9RmFsc2UsCiAgICAgICAgbGFiZWxzPXsiZGF0YS10eXBlIjogImhlbGQtb3V0In0sCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1jb250ZXh0LmFydGlmYWN0X3N1YnBhdGgoImRhdGEiKSwKICAgICkKCiAgICBtb2RlbF9jb25maWcgPSBnZW5fc2tsZWFybl9tb2RlbChtb2RlbF9wa2dfY2xhc3MsIGNvbnRleHQucGFyYW1ldGVycy5pdGVtcygpKQoKICAgIG1vZGVsX2NvbmZpZ1siRklUIl0udXBkYXRlKHsiWCI6IHh0cmFpbiwgInkiOiB5dHJhaW4udmFsdWVzfSkKCiAgICBDbGFzc2lmaWVyQ2xhc3MgPSBjcmVhdGVfY2xhc3MobW9kZWxfY29uZmlnWyJNRVRBIl1bImNsYXNzIl0pCgogICAgbW9kZWwgPSBDbGFzc2lmaWVyQ2xhc3MoKiptb2RlbF9jb25maWdbIkNMQVNTIl0pCgogICAgbW9kZWwuZml0KCoqbW9kZWxfY29uZmlnWyJGSVQiXSkKCiAgICBhcnRpZmFjdF9wYXRoID0gY29udGV4dC5hcnRpZmFjdF9zdWJwYXRoKG1vZGVsc19kZXN0KQogICAgcGxvdHNfcGF0aCA9IGNvbnRleHQuYXJ0aWZhY3Rfc3VicGF0aChtb2RlbHNfZGVzdCwgcGxvdHNfZGVzdCkKICAgIGlmIG1vZGVsX2V2YWx1YXRvcjoKICAgICAgICBldmFsX21ldHJpY3MgPSBtb2RlbF9ldmFsdWF0b3IoCiAgICAgICAgICAgIGNvbnRleHQsIHh2YWxpZCwgeXZhbGlkLCBtb2RlbCwgcGxvdHNfYXJ0aWZhY3RfcGF0aD1wbG90c19wYXRoCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICBldmFsX21ldHJpY3MgPSBldmFsX21vZGVsX3YyKAogICAgICAgICAgICBjb250ZXh0LCB4dmFsaWQsIHl2YWxpZCwgbW9kZWwsIHBsb3RzX2FydGlmYWN0X3BhdGg9cGxvdHNfcGF0aAogICAgICAgICkKCiAgICBrd2FyZ3MgPSB7InRyYWluaW5nX3NldCI6IHRlc3Rfc2V0LCAibGFiZWxfY29sdW1uIjogbGFiZWxfY29sdW1ufQogICAgc3BsaXQgPSBtb2RlbF9wa2dfY2xhc3MucnNwbGl0KCIuIiwgMSkKICAgIGlmIHNwbGl0IGFuZCBsZW4oc3BsaXQpID09IDI6CiAgICAgICAga3dhcmdzWyJhbGdvcml0aG0iXSA9IHNwbGl0WzFdCgogICAgaWYgZGF0YXNldC5tZXRhIGFuZCBkYXRhc2V0Lm1ldGEua2luZCA9PSAiRmVhdHVyZVZlY3RvciI6CiAgICAgICAga3dhcmdzWyJmZWF0dXJlX3ZlY3RvciJdID0gZGF0YXNldC5tZXRhLnVyaQoKICAgIGNvbnRleHQuc2V0X2xhYmVsKCJjbGFzcyIsIG1vZGVsX3BrZ19jbGFzcykKICAgIGNvbnRleHQubG9nX21vZGVsKAogICAgICAgICJtb2RlbCIsCiAgICAgICAgYm9keT1kdW1wcyhtb2RlbCksCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1hcnRpZmFjdF9wYXRoLAogICAgICAgIGV4dHJhX2RhdGE9ZXZhbF9tZXRyaWNzLAogICAgICAgIG1vZGVsX2ZpbGU9Im1vZGVsLnBrbCIsCiAgICAgICAgbWV0cmljcz1jb250ZXh0LnJlc3VsdHMsCiAgICAgICAgbGFiZWxzPXsiY2xhc3MiOiBtb2RlbF9wa2dfY2xhc3N9LAogICAgICAgIGZyYW1ld29yaz0ic2tsZWFybiIsCiAgICAgICAgKiprd2FyZ3MKICAgICkK - origin_filename: '' - code_origin: '' + has_varargs: false + lineno: 31 command: '' -metadata: - tag: '' - name: sklearn-classifier - categories: - - machine-learning - - model-training -verbose: false -kind: job + description: train any classifier using scikit-learn's API + default_handler: train_model diff --git a/functions/src/sklearn_classifier/sklearn_classifier.py b/functions/src/sklearn_classifier/sklearn_classifier.py index 1a73d4045..daca4e4ad 100644 --- a/functions/src/sklearn_classifier/sklearn_classifier.py +++ b/functions/src/sklearn_classifier/sklearn_classifier.py @@ -19,13 +19,12 @@ warnings.simplefilter(action="ignore", category=FutureWarning) -from cloudpickle import dumps import pandas as pd -from typing import List -from mlrun.execution import MLClientCtx +from cloudpickle import dumps from mlrun.datastore import DataItem +from mlrun.execution import MLClientCtx from mlrun.mlutils.data import get_sample, get_splits -from mlrun.mlutils.models import gen_sklearn_model, eval_model_v2 +from mlrun.mlutils.models import eval_model_v2, gen_sklearn_model from mlrun.utils.helpers import create_class @@ -34,7 +33,7 @@ def train_model( model_pkg_class: str, dataset: DataItem, label_column: str = "labels", - encode_cols: List[str] = [], + encode_cols: list[str] = [], sample: int = -1, test_size: float = 0.30, train_val_split: float = 0.70, @@ -139,5 +138,5 @@ def train_model( metrics=context.results, labels={"class": model_pkg_class}, framework="sklearn", - **kwargs + **kwargs, ) diff --git a/functions/src/sklearn_classifier/test_sklearn_classifier.py b/functions/src/sklearn_classifier/test_sklearn_classifier.py index 5c29e85b3..2aa314b3d 100644 --- a/functions/src/sklearn_classifier/test_sklearn_classifier.py +++ b/functions/src/sklearn_classifier/test_sklearn_classifier.py @@ -12,22 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import mlrun import os import pickle + +import mlrun import pandas as pd def generate_data(): - fn = mlrun.import_function('../gen_class_data/function.yaml') - run = fn.run(params={'key': 'classifier-data', - 'n_samples': 10_000, - 'm_features': 5, - 'k_classes': 2, - 'header': None, - 'weight': [0.5, 0.5], - 'sk_params': {'n_informative': 2}, - 'file_ext': 'csv'}, local=True, artifact_path="./artifacts") + fn = mlrun.import_function("../gen_class_data/function.yaml") + run = fn.run( + params={ + "key": "classifier-data", + "n_samples": 10_000, + "m_features": 5, + "k_classes": 2, + "header": None, + "weight": [0.5, 0.5], + "sk_params": {"n_informative": 2}, + "file_ext": "csv", + }, + local=True, + artifact_path="./artifacts", + ) return run @@ -35,23 +42,31 @@ def test_import_sklearn_classifier(): acquire_run = generate_data() fn = mlrun.import_function("function.yaml") # define model - params = {"model_pkg_class": "sklearn.ensemble.RandomForestClassifier", - "label_column": "labels"} + params = { + "model_pkg_class": "sklearn.ensemble.RandomForestClassifier", + "label_column": "labels", + } - train_run = fn.run(params=params, - inputs={"dataset": acquire_run.status.artifacts[0]['spec']['target_path']}, - local=True, - artifact_path="./") + train_run = fn.run( + params=params, + inputs={"dataset": acquire_run.status.artifacts[0]["spec"]["target_path"]}, + local=True, + artifact_path="./", + ) for artifact in train_run.status.artifacts: - if artifact['kind'] == 'model': - assert os.path.exists(artifact['spec']['target_path']), 'Could not find model dir' + if artifact["kind"] == "model": + assert os.path.exists(artifact["spec"]["target_path"]), ( + "Could not find model dir" + ) break - assert os.path.exists(train_run.status.artifacts[0]['spec']['target_path']) - model = pickle.load(open(artifact['spec']['target_path'] + artifact['spec']['model_file'], 'rb')) - df = pd.read_csv(acquire_run.status.artifacts[0]['spec']['target_path']) - x = df.drop(['labels'], axis=1).iloc[0:1] - y_true = df['labels'][0] + assert os.path.exists(train_run.status.artifacts[0]["spec"]["target_path"]) + model = pickle.load( + open(artifact["spec"]["target_path"] + artifact["spec"]["model_file"], "rb") + ) + df = pd.read_csv(acquire_run.status.artifacts[0]["spec"]["target_path"]) + x = df.drop(["labels"], axis=1).iloc[0:1] + y_true = df["labels"][0] y_pred = model.predict_proba(x).argmax() assert y_pred == y_true, "Failed to predict correctly" diff --git a/functions/src/sklearn_classifier_dask/function.yaml b/functions/src/sklearn_classifier_dask/function.yaml index 46f733886..e202a6c2d 100644 --- a/functions/src/sklearn_classifier_dask/function.yaml +++ b/functions/src/sklearn_classifier_dask/function.yaml @@ -1,42 +1,34 @@ -kind: job metadata: - name: sklearn-classifier-dask tag: '' - hash: e542038fbb84f790b7144b529665f36d70d80906 - project: '' - labels: - author: Iguazio - framework: sklearn + name: sklearn-classifier-dask categories: - machine-learning - model-training +verbose: false +kind: job spec: - command: '' - args: [] image: mlrun/ml-models + disable_auto_mount: false build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IG1scnVuCgppbXBvcnQgd2FybmluZ3MKCndhcm5pbmdzLmZpbHRlcndhcm5pbmdzKCJpZ25vcmUiKQoKaW1wb3J0IGpvYmxpYgppbXBvcnQgbnVtcHkgYXMgbnAKaW1wb3J0IHBhbmRhcyBhcyBwZApmcm9tIGNsb3VkcGlja2xlIGltcG9ydCBkdW1wcwoKZnJvbSBkYXNrIGltcG9ydCBkYXRhZnJhbWUgYXMgZGQKZnJvbSBkYXNrLmRlbGF5ZWQgaW1wb3J0IGRlbGF5ZWQKZnJvbSBkYXNrX21sIGltcG9ydCBtb2RlbF9zZWxlY3Rpb24KZnJvbSBkYXNrX21sLnByZXByb2Nlc3NpbmcgaW1wb3J0IFN0YW5kYXJkU2NhbGVyLCBMYWJlbEVuY29kZXIKCmZyb20gbWxydW4uYXJ0aWZhY3RzIGltcG9ydCBQbG90QXJ0aWZhY3QKZnJvbSBtbHJ1bi5tbHV0aWxzLm1vZGVscyBpbXBvcnQgZ2VuX3NrbGVhcm5fbW9kZWwKZnJvbSBtbHJ1bi51dGlscy5oZWxwZXJzIGltcG9ydCBjcmVhdGVfY2xhc3MKCmltcG9ydCBtYXRwbG90bGliLnB5cGxvdCBhcyBwbHQKZnJvbSB5ZWxsb3dicmljay5jbGFzc2lmaWVyIGltcG9ydCBST0NBVUMsIENsYXNzaWZpY2F0aW9uUmVwb3J0LCBDb25mdXNpb25NYXRyaXgKZnJvbSB5ZWxsb3dicmljay5tb2RlbF9zZWxlY3Rpb24gaW1wb3J0IEZlYXR1cmVJbXBvcnRhbmNlcwoKCmRlZiB0cmFpbl9tb2RlbCgKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogbWxydW4uRGF0YUl0ZW0sCiAgICBtb2RlbF9wa2dfY2xhc3M6IHN0ciwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gImxhYmVsIiwKICAgIHRyYWluX3ZhbGlkYXRpb25fc2l6ZTogZmxvYXQgPSAwLjc1LAogICAgc2FtcGxlOiBmbG9hdCA9IDEuMCwKICAgIG1vZGVsc19kZXN0OiBzdHIgPSAibW9kZWxzIiwKICAgIHRlc3Rfc2V0X2tleTogc3RyID0gInRlc3Rfc2V0IiwKICAgIHBsb3RzX2Rlc3Q6IHN0ciA9ICJwbG90cyIsCiAgICBkYXNrX2Z1bmN0aW9uOiBzdHIgPSBOb25lLAogICAgZGFza19jbGllbnQ9Tm9uZSwKICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDQyLAopIC0+IE5vbmU6CgogICAgIiIiCiAgICBUcmFpbiBhIHNrbGVhcm4gY2xhc3NpZmllciB3aXRoIERhc2sKCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIEZ1bmN0aW9uIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFJhdyBkYXRhIGZpbGUuCiAgICA6cGFyYW0gbW9kZWxfcGtnX2NsYXNzOiAgICAgICAgIE1vZGVsIHRvIHRyYWluLCBlLmcsICJza2xlYXJuLmVuc2VtYmxlLlJhbmRvbUZvcmVzdENsYXNzaWZpZXIiLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvciBqc29uIG1vZGVsIGNvbmZpZy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW46ICAgICAgICAgICAgKGxhYmVsKSBHcm91bmQtdHJ1dGggeSBsYWJlbHMuCiAgICA6cGFyYW0gdHJhaW5fdmFsaWRhdGlvbl9zaXplOiAgICgwLjc1KSBUcmFpbiB2YWxpZGF0aW9uIHNldCBwcm9wb3J0aW9uIG91dCBvZiB0aGUgZnVsbCBkYXRhc2V0LgogICAgOnBhcmFtIHNhbXBsZTogICAgICAgICAgICAgICAgICAoMS4wKSBTZWxlY3Qgc2FtcGxlIGZyb20gZGF0YXNldCAobi1yb3dzLyUgb2YgdG90YWwpLCByYW5kb216aWUgcm93cyBhcyBkZWZhdWx0LgogICAgOnBhcmFtIG1vZGVsc19kZXN0OiAgICAgICAgICAgICAobW9kZWxzKSBNb2RlbHMgc3ViZm9sZGVyIG9uIGFydGlmYWN0IHBhdGguCiAgICA6cGFyYW0gdGVzdF9zZXRfa2V5OiAgICAgICAgICAgICh0ZXN0X3NldCkgTWxydW4gZGIga2V5IG9mIGhlbGQgb3V0IGRhdGEgaW4gYXJ0aWZhY3Qgc3RvcmUuCiAgICA6cGFyYW0gcGxvdHNfZGVzdDogICAgICAgICAgICAgIChwbG90cykgUGxvdCBzdWJmb2xkZXIgb24gYXJ0aWZhY3QgcGF0aC4KICAgIDpwYXJhbSBkYXNrX2Z1bmN0aW9uOiAgICAgICAgICAgZGFzayBmdW5jdGlvbiB1cmwgKGRiOi8vLi4pCiAgICA6cGFyYW0gZGFza19jbGllbnQ6ICAgICAgICAgICAgIGRhc2sgY2xpZW50IG9iamVjdAogICAgOnBhcmFtIGZpbGVfZXh0OiAgICAgICAgICAgICAgICAocGFycXVldCkgZm9ybWF0IGZvciB0ZXN0X3NldF9rZXkgaG9sZCBvdXQgZGF0YQogICAgOnBhcmFtIHJhbmRvbV9zdGF0ZTogICAgICAgICAgICAoNDIpIHNrbGVhcm4gc2VlZAogICAgIiIiCiAgICBpZiBkYXNrX2Z1bmN0aW9uOgogICAgICAgIGNsaWVudCA9IG1scnVuLmltcG9ydF9mdW5jdGlvbihkYXNrX2Z1bmN0aW9uKS5jbGllbnQKICAgIGVsaWYgZGFza19jbGllbnQ6CiAgICAgICAgY2xpZW50ID0gZGFza19jbGllbnQKICAgIGVsc2U6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigiZGFzayBjbGllbnQgd2FzIG5vdCBwcm92aWRlZCIpCgogICAgY29udGV4dC5sb2dnZXIuaW5mbygiUmVhZCBEYXRhIikKICAgIGRmID0gZGF0YXNldC5hc19kZihkZl9tb2R1bGU9ZGQpCgogICAgY29udGV4dC5sb2dnZXIuaW5mbygiUHJlcCBEYXRhIikKICAgIG51bWVyaWNzID0gWyJpbnQxNiIsICJpbnQzMiIsICJpbnQ2NCIsICJmbG9hdDE2IiwgImZsb2F0MzIiLCAiZmxvYXQ2NCJdCiAgICBkZiA9IGRmLnNlbGVjdF9kdHlwZXMoaW5jbHVkZT1udW1lcmljcykKCiAgICBpZiBkZi5pc25hKCkuYW55KCkuYW55KCkuY29tcHV0ZSgpID09IFRydWU6CiAgICAgICAgcmFpc2UgRXhjZXB0aW9uKCJOQXMgdmFsdXMgZm91bmQiKQoKICAgIGRmX2hlYWRlciA9IGRmLmNvbHVtbnMKCiAgICBkZiA9IGRmLnNhbXBsZShmcmFjPXNhbXBsZSkucmVzZXRfaW5kZXgoZHJvcD1UcnVlKQogICAgZW5jb2RlciA9IExhYmVsRW5jb2RlcigpCiAgICBlbmNvZGVyID0gZW5jb2Rlci5maXQoZGZbbGFiZWxfY29sdW1uXSkKICAgIFggPSBkZi5kcm9wKGxhYmVsX2NvbHVtbiwgYXhpcz0xKS50b19kYXNrX2FycmF5KGxlbmd0aHM9VHJ1ZSkKICAgIHkgPSBlbmNvZGVyLnRyYW5zZm9ybShkZltsYWJlbF9jb2x1bW5dKQoKICAgIGNsYXNzZXMgPSBkZltsYWJlbF9jb2x1bW5dLmRyb3BfZHVwbGljYXRlcygpICAjIG5vIHVuaXF1ZSB2YWx1ZXMgaW4gZGFzawogICAgY2xhc3NlcyA9IFtzdHIoaSkgZm9yIGkgaW4gY2xhc3Nlc10KCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJTcGxpdCBhbmQgVHJhaW4iKQogICAgWF90cmFpbiwgWF90ZXN0LCB5X3RyYWluLCB5X3Rlc3QgPSBtb2RlbF9zZWxlY3Rpb24udHJhaW5fdGVzdF9zcGxpdCgKICAgICAgICBYLCB5LCB0cmFpbl9zaXplPXRyYWluX3ZhbGlkYXRpb25fc2l6ZSwgcmFuZG9tX3N0YXRlPXJhbmRvbV9zdGF0ZQogICAgKQoKICAgIHNjYWxlciA9IFN0YW5kYXJkU2NhbGVyKCkKICAgIHNjYWxlciA9IHNjYWxlci5maXQoWF90cmFpbikKICAgIFhfdHJhaW5fdHJhbnNmb3JtZWQgPSBzY2FsZXIudHJhbnNmb3JtKFhfdHJhaW4pCiAgICBYX3Rlc3RfdHJhbnNmb3JtZWQgPSBzY2FsZXIudHJhbnNmb3JtKFhfdGVzdCkKCiAgICBtb2RlbF9jb25maWcgPSBnZW5fc2tsZWFybl9tb2RlbChtb2RlbF9wa2dfY2xhc3MsIGNvbnRleHQucGFyYW1ldGVycy5pdGVtcygpKQoKICAgIG1vZGVsX2NvbmZpZ1siRklUIl0udXBkYXRlKHsiWCI6IFhfdHJhaW5fdHJhbnNmb3JtZWQsICJ5IjogeV90cmFpbn0pCgogICAgQ2xhc3NpZmllckNsYXNzID0gY3JlYXRlX2NsYXNzKG1vZGVsX2NvbmZpZ1siTUVUQSJdWyJjbGFzcyJdKQoKICAgIG1vZGVsID0gQ2xhc3NpZmllckNsYXNzKCoqbW9kZWxfY29uZmlnWyJDTEFTUyJdKQoKICAgIHdpdGggam9ibGliLnBhcmFsbGVsX2JhY2tlbmQoImRhc2siKToKICAgICAgICBtb2RlbCA9IG1vZGVsLmZpdCgqKm1vZGVsX2NvbmZpZ1siRklUIl0pCgogICAgY29udGV4dC5sb2dnZXIuaW5mbygiRXZhbHVhdGUiKQogICAgZXh0cmFfZGF0YV9kaWN0ID0ge30KICAgIGZvciByZXBvcnQgaW4gKFJPQ0FVQywgQ2xhc3NpZmljYXRpb25SZXBvcnQsIENvbmZ1c2lvbk1hdHJpeCk6CiAgICAgICAgcmVwb3J0X25hbWUgPSBzdHIocmVwb3J0Ll9fbmFtZV9fKQogICAgICAgIHBsdC5jbGEoKQogICAgICAgIHBsdC5jbGYoKQogICAgICAgIHBsdC5jbG9zZSgpCgogICAgICAgIHZpeiA9IHJlcG9ydChtb2RlbCwgY2xhc3Nlcz1jbGFzc2VzLCBwZXJfY2xhc3M9VHJ1ZSwgaXNfZml0dGVkPVRydWUpCiAgICAgICAgdml6LmZpdChYX3RyYWluX3RyYW5zZm9ybWVkLCB5X3RyYWluKSAgIyBGaXQgdGhlIHRyYWluaW5nIGRhdGEgdG8gdGhlIHZpc3VhbGl6ZXIKICAgICAgICB2aXouc2NvcmUoCiAgICAgICAgICAgIFhfdGVzdF90cmFuc2Zvcm1lZCwgeV90ZXN0LmNvbXB1dGUoKQogICAgICAgICkgICMgRXZhbHVhdGUgdGhlIG1vZGVsIG9uIHRoZSB0ZXN0IGRhdGEKCiAgICAgICAgcGxvdCA9IGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgICAgICBQbG90QXJ0aWZhY3QocmVwb3J0X25hbWUsIGJvZHk9dml6LmZpZywgdGl0bGU9cmVwb3J0X25hbWUpLCBkYl9rZXk9RmFsc2UKICAgICAgICApCiAgICAgICAgZXh0cmFfZGF0YV9kaWN0W3N0cihyZXBvcnQpXSA9IHBsb3QKCiAgICAgICAgaWYgcmVwb3J0X25hbWUgPT0gIlJPQ0FVQyI6CiAgICAgICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdHMoCiAgICAgICAgICAgICAgICB7Im1pY3JvIjogdml6LnJvY19hdWMuZ2V0KCJtaWNybyIpLCAibWFjcm8iOiB2aXoucm9jX2F1Yy5nZXQoIm1hY3JvIil9CiAgICAgICAgICAgICkKCiAgICAgICAgZWxpZiByZXBvcnRfbmFtZSA9PSAiQ2xhc3NpZmljYXRpb25SZXBvcnQiOgogICAgICAgICAgICBmb3Igc2NvcmVfbmFtZSBpbiB2aXouc2NvcmVzXzoKICAgICAgICAgICAgICAgIGZvciBzY29yZV9jbGFzcyBpbiB2aXouc2NvcmVzX1tzY29yZV9uYW1lXToKCiAgICAgICAgICAgICAgICAgICAgY29udGV4dC5sb2dfcmVzdWx0cygKICAgICAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAgICAgc2NvcmVfbmFtZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgKyAiLSIKICAgICAgICAgICAgICAgICAgICAgICAgICAgICsgc2NvcmVfY2xhc3M6IHZpei5zY29yZXNfW3Njb3JlX25hbWVdLmdldChzY29yZV9jbGFzcykKICAgICAgICAgICAgICAgICAgICAgICAgfQogICAgICAgICAgICAgICAgICAgICkKCiAgICB2aXogPSBGZWF0dXJlSW1wb3J0YW5jZXMoCiAgICAgICAgbW9kZWwsCiAgICAgICAgY2xhc3Nlcz1jbGFzc2VzLAogICAgICAgIHBlcl9jbGFzcz1UcnVlLAogICAgICAgIGlzX2ZpdHRlZD1UcnVlLAogICAgICAgIGxhYmVscz1kZl9oZWFkZXIuZGVsZXRlKGRmX2hlYWRlci5nZXRfbG9jKGxhYmVsX2NvbHVtbikpLAogICAgKQogICAgdml6LmZpdChYX3RyYWluX3RyYW5zZm9ybWVkLCB5X3RyYWluKQogICAgdml6LnNjb3JlKFhfdGVzdF90cmFuc2Zvcm1lZCwgeV90ZXN0KQoKICAgIHBsb3QgPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBQbG90QXJ0aWZhY3QoIkZlYXR1cmVJbXBvcnRhbmNlcyIsIGJvZHk9dml6LmZpZywgdGl0bGU9IkZlYXR1cmVJbXBvcnRhbmNlcyIpLAogICAgICAgIGRiX2tleT1GYWxzZSwKICAgICkKICAgIGV4dHJhX2RhdGFfZGljdFsiRmVhdHVyZUltcG9ydGFuY2VzIl0gPSBwbG90CgogICAgcGx0LmNsYSgpCiAgICBwbHQuY2xmKCkKICAgIHBsdC5jbG9zZSgpCgogICAgY29udGV4dC5sb2dnZXIuaW5mbygiTG9nIGFydGlmYWN0cyIpCiAgICBhcnRpZmFjdF9wYXRoID0gY29udGV4dC5hcnRpZmFjdF9zdWJwYXRoKG1vZGVsc19kZXN0KQoKICAgIGNvbnRleHQuc2V0X2xhYmVsKCJjbGFzcyIsIG1vZGVsX3BrZ19jbGFzcykKCiAgICBjb250ZXh0LmxvZ19tb2RlbCgKICAgICAgICAibW9kZWwiLAogICAgICAgIGJvZHk9ZHVtcHMobW9kZWwpLAogICAgICAgIGFydGlmYWN0X3BhdGg9YXJ0aWZhY3RfcGF0aCwKICAgICAgICBtb2RlbF9maWxlPSJtb2RlbC5wa2wiLAogICAgICAgIGV4dHJhX2RhdGE9ZXh0cmFfZGF0YV9kaWN0LAogICAgICAgIG1ldHJpY3M9Y29udGV4dC5yZXN1bHRzLAogICAgICAgIGxhYmVscz17ImNsYXNzIjogbW9kZWxfcGtnX2NsYXNzfSwKICAgICkKCiAgICBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICAic3RhbmRhcmRfc2NhbGVyIiwKICAgICAgICBib2R5PWR1bXBzKHNjYWxlciksCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1hcnRpZmFjdF9wYXRoLAogICAgKQoKICAgIGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgICJsYWJlbF9lbmNvZGVyIiwKICAgICAgICBib2R5PWR1bXBzKGVuY29kZXIpLAogICAgICAgIGFydGlmYWN0X3BhdGg9YXJ0aWZhY3RfcGF0aCwKICAgICkKCiAgICBkZl90b19zYXZlID0gZGVsYXllZChucC5jb2x1bW5fc3RhY2spKChYX3Rlc3QsIHlfdGVzdCkpLmNvbXB1dGUoKQogICAgY29udGV4dC5sb2dfZGF0YXNldCgKICAgICAgICB0ZXN0X3NldF9rZXksCiAgICAgICAgZGY9cGQuRGF0YUZyYW1lKGRmX3RvX3NhdmUsIGNvbHVtbnM9ZGZfaGVhZGVyKSwgICMgaW1wcm92ZSBsb2cgZGF0YXNldCBhYmlsaXR5CiAgICAgICAgZm9ybWF0PWZpbGVfZXh0LAogICAgICAgIGluZGV4PUZhbHNlLAogICAgICAgIGxhYmVscz17ImRhdGEtdHlwZSI6ICJoZWxkLW91dCJ9LAogICAgICAgIGFydGlmYWN0X3BhdGg9Y29udGV4dC5hcnRpZmFjdF9zdWJwYXRoKCJkYXRhIiksCiAgICApCgogICAgY29udGV4dC5sb2dnZXIuaW5mbygiRG9uZSEiKQo= - commands: [] - code_origin: https://github.com/guy1992l/functions.git#75359393bff0aaf27fb04c00d5d0037a1d1e32db:/Users/guyl/Projects/functions/sklearn_classifier_dask/sklearn_classifier_dask.py - origin_filename: /Users/guyl/Projects/functions/sklearn_classifier_dask/sklearn_classifier_dask.py + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgppbXBvcnQgbWxydW4KCndhcm5pbmdzLmZpbHRlcndhcm5pbmdzKCJpZ25vcmUiKQoKaW1wb3J0IGpvYmxpYgppbXBvcnQgbWF0cGxvdGxpYi5weXBsb3QgYXMgcGx0CmltcG9ydCBudW1weSBhcyBucAppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gY2xvdWRwaWNrbGUgaW1wb3J0IGR1bXBzCmZyb20gZGFzayBpbXBvcnQgZGF0YWZyYW1lIGFzIGRkCmZyb20gZGFzay5kZWxheWVkIGltcG9ydCBkZWxheWVkCmZyb20gZGFza19tbCBpbXBvcnQgbW9kZWxfc2VsZWN0aW9uCmZyb20gZGFza19tbC5wcmVwcm9jZXNzaW5nIGltcG9ydCBMYWJlbEVuY29kZXIsIFN0YW5kYXJkU2NhbGVyCmZyb20gbWxydW4uYXJ0aWZhY3RzIGltcG9ydCBQbG90QXJ0aWZhY3QKZnJvbSBtbHJ1bi5tbHV0aWxzLm1vZGVscyBpbXBvcnQgZ2VuX3NrbGVhcm5fbW9kZWwKZnJvbSBtbHJ1bi51dGlscy5oZWxwZXJzIGltcG9ydCBjcmVhdGVfY2xhc3MKZnJvbSB5ZWxsb3dicmljay5jbGFzc2lmaWVyIGltcG9ydCBST0NBVUMsIENsYXNzaWZpY2F0aW9uUmVwb3J0LCBDb25mdXNpb25NYXRyaXgKZnJvbSB5ZWxsb3dicmljay5tb2RlbF9zZWxlY3Rpb24gaW1wb3J0IEZlYXR1cmVJbXBvcnRhbmNlcwoKCmRlZiB0cmFpbl9tb2RlbCgKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogbWxydW4uRGF0YUl0ZW0sCiAgICBtb2RlbF9wa2dfY2xhc3M6IHN0ciwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gImxhYmVsIiwKICAgIHRyYWluX3ZhbGlkYXRpb25fc2l6ZTogZmxvYXQgPSAwLjc1LAogICAgc2FtcGxlOiBmbG9hdCA9IDEuMCwKICAgIG1vZGVsc19kZXN0OiBzdHIgPSAibW9kZWxzIiwKICAgIHRlc3Rfc2V0X2tleTogc3RyID0gInRlc3Rfc2V0IiwKICAgIHBsb3RzX2Rlc3Q6IHN0ciA9ICJwbG90cyIsCiAgICBkYXNrX2Z1bmN0aW9uOiBzdHIgPSBOb25lLAogICAgZGFza19jbGllbnQ9Tm9uZSwKICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDQyLAopIC0+IE5vbmU6CiAgICAiIiIKICAgIFRyYWluIGEgc2tsZWFybiBjbGFzc2lmaWVyIHdpdGggRGFzawoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgRnVuY3Rpb24gY29udGV4dC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgUmF3IGRhdGEgZmlsZS4KICAgIDpwYXJhbSBtb2RlbF9wa2dfY2xhc3M6ICAgICAgICAgTW9kZWwgdG8gdHJhaW4sIGUuZywgInNrbGVhcm4uZW5zZW1ibGUuUmFuZG9tRm9yZXN0Q2xhc3NpZmllciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9yIGpzb24gbW9kZWwgY29uZmlnLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICAgICAgICAobGFiZWwpIEdyb3VuZC10cnV0aCB5IGxhYmVscy4KICAgIDpwYXJhbSB0cmFpbl92YWxpZGF0aW9uX3NpemU6ICAgKDAuNzUpIFRyYWluIHZhbGlkYXRpb24gc2V0IHByb3BvcnRpb24gb3V0IG9mIHRoZSBmdWxsIGRhdGFzZXQuCiAgICA6cGFyYW0gc2FtcGxlOiAgICAgICAgICAgICAgICAgICgxLjApIFNlbGVjdCBzYW1wbGUgZnJvbSBkYXRhc2V0IChuLXJvd3MvJSBvZiB0b3RhbCksIHJhbmRvbXppZSByb3dzIGFzIGRlZmF1bHQuCiAgICA6cGFyYW0gbW9kZWxzX2Rlc3Q6ICAgICAgICAgICAgIChtb2RlbHMpIE1vZGVscyBzdWJmb2xkZXIgb24gYXJ0aWZhY3QgcGF0aC4KICAgIDpwYXJhbSB0ZXN0X3NldF9rZXk6ICAgICAgICAgICAgKHRlc3Rfc2V0KSBNbHJ1biBkYiBrZXkgb2YgaGVsZCBvdXQgZGF0YSBpbiBhcnRpZmFjdCBzdG9yZS4KICAgIDpwYXJhbSBwbG90c19kZXN0OiAgICAgICAgICAgICAgKHBsb3RzKSBQbG90IHN1YmZvbGRlciBvbiBhcnRpZmFjdCBwYXRoLgogICAgOnBhcmFtIGRhc2tfZnVuY3Rpb246ICAgICAgICAgICBkYXNrIGZ1bmN0aW9uIHVybCAoZGI6Ly8uLikKICAgIDpwYXJhbSBkYXNrX2NsaWVudDogICAgICAgICAgICAgZGFzayBjbGllbnQgb2JqZWN0CiAgICA6cGFyYW0gZmlsZV9leHQ6ICAgICAgICAgICAgICAgIChwYXJxdWV0KSBmb3JtYXQgZm9yIHRlc3Rfc2V0X2tleSBob2xkIG91dCBkYXRhCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICAgICAgICg0Mikgc2tsZWFybiBzZWVkCiAgICAiIiIKICAgIGlmIGRhc2tfZnVuY3Rpb246CiAgICAgICAgY2xpZW50ID0gbWxydW4uaW1wb3J0X2Z1bmN0aW9uKGRhc2tfZnVuY3Rpb24pLmNsaWVudAogICAgZWxpZiBkYXNrX2NsaWVudDoKICAgICAgICBjbGllbnQgPSBkYXNrX2NsaWVudAogICAgZWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKCJkYXNrIGNsaWVudCB3YXMgbm90IHByb3ZpZGVkIikKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJSZWFkIERhdGEiKQogICAgZGYgPSBkYXRhc2V0LmFzX2RmKGRmX21vZHVsZT1kZCkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJQcmVwIERhdGEiKQogICAgbnVtZXJpY3MgPSBbImludDE2IiwgImludDMyIiwgImludDY0IiwgImZsb2F0MTYiLCAiZmxvYXQzMiIsICJmbG9hdDY0Il0KICAgIGRmID0gZGYuc2VsZWN0X2R0eXBlcyhpbmNsdWRlPW51bWVyaWNzKQoKICAgIGlmIGRmLmlzbmEoKS5hbnkoKS5hbnkoKS5jb21wdXRlKCkgPT0gVHJ1ZToKICAgICAgICByYWlzZSBFeGNlcHRpb24oIk5BcyB2YWx1cyBmb3VuZCIpCgogICAgZGZfaGVhZGVyID0gZGYuY29sdW1ucwoKICAgIGRmID0gZGYuc2FtcGxlKGZyYWM9c2FtcGxlKS5yZXNldF9pbmRleChkcm9wPVRydWUpCiAgICBlbmNvZGVyID0gTGFiZWxFbmNvZGVyKCkKICAgIGVuY29kZXIgPSBlbmNvZGVyLmZpdChkZltsYWJlbF9jb2x1bW5dKQogICAgWCA9IGRmLmRyb3AobGFiZWxfY29sdW1uLCBheGlzPTEpLnRvX2Rhc2tfYXJyYXkobGVuZ3Rocz1UcnVlKQogICAgeSA9IGVuY29kZXIudHJhbnNmb3JtKGRmW2xhYmVsX2NvbHVtbl0pCgogICAgY2xhc3NlcyA9IGRmW2xhYmVsX2NvbHVtbl0uZHJvcF9kdXBsaWNhdGVzKCkgICMgbm8gdW5pcXVlIHZhbHVlcyBpbiBkYXNrCiAgICBjbGFzc2VzID0gW3N0cihpKSBmb3IgaSBpbiBjbGFzc2VzXQoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIlNwbGl0IGFuZCBUcmFpbiIpCiAgICBYX3RyYWluLCBYX3Rlc3QsIHlfdHJhaW4sIHlfdGVzdCA9IG1vZGVsX3NlbGVjdGlvbi50cmFpbl90ZXN0X3NwbGl0KAogICAgICAgIFgsIHksIHRyYWluX3NpemU9dHJhaW5fdmFsaWRhdGlvbl9zaXplLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlCiAgICApCgogICAgc2NhbGVyID0gU3RhbmRhcmRTY2FsZXIoKQogICAgc2NhbGVyID0gc2NhbGVyLmZpdChYX3RyYWluKQogICAgWF90cmFpbl90cmFuc2Zvcm1lZCA9IHNjYWxlci50cmFuc2Zvcm0oWF90cmFpbikKICAgIFhfdGVzdF90cmFuc2Zvcm1lZCA9IHNjYWxlci50cmFuc2Zvcm0oWF90ZXN0KQoKICAgIG1vZGVsX2NvbmZpZyA9IGdlbl9za2xlYXJuX21vZGVsKG1vZGVsX3BrZ19jbGFzcywgY29udGV4dC5wYXJhbWV0ZXJzLml0ZW1zKCkpCgogICAgbW9kZWxfY29uZmlnWyJGSVQiXS51cGRhdGUoeyJYIjogWF90cmFpbl90cmFuc2Zvcm1lZCwgInkiOiB5X3RyYWlufSkKCiAgICBDbGFzc2lmaWVyQ2xhc3MgPSBjcmVhdGVfY2xhc3MobW9kZWxfY29uZmlnWyJNRVRBIl1bImNsYXNzIl0pCgogICAgbW9kZWwgPSBDbGFzc2lmaWVyQ2xhc3MoKiptb2RlbF9jb25maWdbIkNMQVNTIl0pCgogICAgd2l0aCBqb2JsaWIucGFyYWxsZWxfYmFja2VuZCgiZGFzayIpOgogICAgICAgIG1vZGVsID0gbW9kZWwuZml0KCoqbW9kZWxfY29uZmlnWyJGSVQiXSkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJFdmFsdWF0ZSIpCiAgICBleHRyYV9kYXRhX2RpY3QgPSB7fQogICAgZm9yIHJlcG9ydCBpbiAoUk9DQVVDLCBDbGFzc2lmaWNhdGlvblJlcG9ydCwgQ29uZnVzaW9uTWF0cml4KToKICAgICAgICByZXBvcnRfbmFtZSA9IHN0cihyZXBvcnQuX19uYW1lX18pCiAgICAgICAgcGx0LmNsYSgpCiAgICAgICAgcGx0LmNsZigpCiAgICAgICAgcGx0LmNsb3NlKCkKCiAgICAgICAgdml6ID0gcmVwb3J0KG1vZGVsLCBjbGFzc2VzPWNsYXNzZXMsIHBlcl9jbGFzcz1UcnVlLCBpc19maXR0ZWQ9VHJ1ZSkKICAgICAgICB2aXouZml0KFhfdHJhaW5fdHJhbnNmb3JtZWQsIHlfdHJhaW4pICAjIEZpdCB0aGUgdHJhaW5pbmcgZGF0YSB0byB0aGUgdmlzdWFsaXplcgogICAgICAgIHZpei5zY29yZSgKICAgICAgICAgICAgWF90ZXN0X3RyYW5zZm9ybWVkLCB5X3Rlc3QuY29tcHV0ZSgpCiAgICAgICAgKSAgIyBFdmFsdWF0ZSB0aGUgbW9kZWwgb24gdGhlIHRlc3QgZGF0YQoKICAgICAgICBwbG90ID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgICAgIFBsb3RBcnRpZmFjdChyZXBvcnRfbmFtZSwgYm9keT12aXouZmlnLCB0aXRsZT1yZXBvcnRfbmFtZSksIGRiX2tleT1GYWxzZQogICAgICAgICkKICAgICAgICBleHRyYV9kYXRhX2RpY3Rbc3RyKHJlcG9ydCldID0gcGxvdAoKICAgICAgICBpZiByZXBvcnRfbmFtZSA9PSAiUk9DQVVDIjoKICAgICAgICAgICAgY29udGV4dC5sb2dfcmVzdWx0cygKICAgICAgICAgICAgICAgIHsibWljcm8iOiB2aXoucm9jX2F1Yy5nZXQoIm1pY3JvIiksICJtYWNybyI6IHZpei5yb2NfYXVjLmdldCgibWFjcm8iKX0KICAgICAgICAgICAgKQoKICAgICAgICBlbGlmIHJlcG9ydF9uYW1lID09ICJDbGFzc2lmaWNhdGlvblJlcG9ydCI6CiAgICAgICAgICAgIGZvciBzY29yZV9uYW1lIGluIHZpei5zY29yZXNfOgogICAgICAgICAgICAgICAgZm9yIHNjb3JlX2NsYXNzIGluIHZpei5zY29yZXNfW3Njb3JlX25hbWVdOgogICAgICAgICAgICAgICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdHMoCiAgICAgICAgICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNjb3JlX25hbWUgKyAiLSIgKyBzY29yZV9jbGFzczogdml6LnNjb3Jlc19bc2NvcmVfbmFtZV0uZ2V0KAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNjb3JlX2NsYXNzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICApCgogICAgdml6ID0gRmVhdHVyZUltcG9ydGFuY2VzKAogICAgICAgIG1vZGVsLAogICAgICAgIGNsYXNzZXM9Y2xhc3NlcywKICAgICAgICBwZXJfY2xhc3M9VHJ1ZSwKICAgICAgICBpc19maXR0ZWQ9VHJ1ZSwKICAgICAgICBsYWJlbHM9ZGZfaGVhZGVyLmRlbGV0ZShkZl9oZWFkZXIuZ2V0X2xvYyhsYWJlbF9jb2x1bW4pKSwKICAgICkKICAgIHZpei5maXQoWF90cmFpbl90cmFuc2Zvcm1lZCwgeV90cmFpbikKICAgIHZpei5zY29yZShYX3Rlc3RfdHJhbnNmb3JtZWQsIHlfdGVzdCkKCiAgICBwbG90ID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdEFydGlmYWN0KCJGZWF0dXJlSW1wb3J0YW5jZXMiLCBib2R5PXZpei5maWcsIHRpdGxlPSJGZWF0dXJlSW1wb3J0YW5jZXMiKSwKICAgICAgICBkYl9rZXk9RmFsc2UsCiAgICApCiAgICBleHRyYV9kYXRhX2RpY3RbIkZlYXR1cmVJbXBvcnRhbmNlcyJdID0gcGxvdAoKICAgIHBsdC5jbGEoKQogICAgcGx0LmNsZigpCiAgICBwbHQuY2xvc2UoKQoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkxvZyBhcnRpZmFjdHMiKQogICAgYXJ0aWZhY3RfcGF0aCA9IGNvbnRleHQuYXJ0aWZhY3Rfc3VicGF0aChtb2RlbHNfZGVzdCkKCiAgICBjb250ZXh0LnNldF9sYWJlbCgiY2xhc3MiLCBtb2RlbF9wa2dfY2xhc3MpCgogICAgY29udGV4dC5sb2dfbW9kZWwoCiAgICAgICAgIm1vZGVsIiwKICAgICAgICBib2R5PWR1bXBzKG1vZGVsKSwKICAgICAgICBhcnRpZmFjdF9wYXRoPWFydGlmYWN0X3BhdGgsCiAgICAgICAgbW9kZWxfZmlsZT0ibW9kZWwucGtsIiwKICAgICAgICBleHRyYV9kYXRhPWV4dHJhX2RhdGFfZGljdCwKICAgICAgICBtZXRyaWNzPWNvbnRleHQucmVzdWx0cywKICAgICAgICBsYWJlbHM9eyJjbGFzcyI6IG1vZGVsX3BrZ19jbGFzc30sCiAgICApCgogICAgY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgInN0YW5kYXJkX3NjYWxlciIsCiAgICAgICAgYm9keT1kdW1wcyhzY2FsZXIpLAogICAgICAgIGFydGlmYWN0X3BhdGg9YXJ0aWZhY3RfcGF0aCwKICAgICkKCiAgICBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICAibGFiZWxfZW5jb2RlciIsCiAgICAgICAgYm9keT1kdW1wcyhlbmNvZGVyKSwKICAgICAgICBhcnRpZmFjdF9wYXRoPWFydGlmYWN0X3BhdGgsCiAgICApCgogICAgZGZfdG9fc2F2ZSA9IGRlbGF5ZWQobnAuY29sdW1uX3N0YWNrKSgoWF90ZXN0LCB5X3Rlc3QpKS5jb21wdXRlKCkKICAgIGNvbnRleHQubG9nX2RhdGFzZXQoCiAgICAgICAgdGVzdF9zZXRfa2V5LAogICAgICAgIGRmPXBkLkRhdGFGcmFtZShkZl90b19zYXZlLCBjb2x1bW5zPWRmX2hlYWRlciksICAjIGltcHJvdmUgbG9nIGRhdGFzZXQgYWJpbGl0eQogICAgICAgIGZvcm1hdD1maWxlX2V4dCwKICAgICAgICBpbmRleD1GYWxzZSwKICAgICAgICBsYWJlbHM9eyJkYXRhLXR5cGUiOiAiaGVsZC1vdXQifSwKICAgICAgICBhcnRpZmFjdF9wYXRoPWNvbnRleHQuYXJ0aWZhY3Rfc3VicGF0aCgiZGF0YSIpLAogICAgKQoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oIkRvbmUhIikK + code_origin: '' + filename: sklearn_classifier_dask.py entry_points: train_model: - name: train_model - doc: Train a sklearn classifier with Dask + outputs: + - type: None parameters: - name: context type: MLClientCtx doc: Function context. - default: '' - name: dataset type: DataItem doc: Raw data file. - default: '' - name: model_pkg_class type: str doc: Model to train, e.g, "sklearn.ensemble.RandomForestClassifier", or json model config. - default: '' - name: label_column type: str doc: (label) Ground-truth y labels. @@ -77,16 +69,11 @@ spec: type: int doc: (42) sklearn seed default: 42 - outputs: - - default: '' - lineno: 42 + name: train_model + doc: Train a sklearn classifier with Dask + has_kwargs: false + has_varargs: false + lineno: 39 + command: '' description: train any classifier using scikit-learn's API over Dask default_handler: train_model - disable_auto_mount: false - env: [] - priority_class_name: '' - preemption_mode: prevent - affinity: null - tolerations: null - security_context: {} -verbose: false diff --git a/functions/src/sklearn_classifier_dask/sklearn_classifier_dask.py b/functions/src/sklearn_classifier_dask/sklearn_classifier_dask.py index 39ec34716..73042ca45 100644 --- a/functions/src/sklearn_classifier_dask/sklearn_classifier_dask.py +++ b/functions/src/sklearn_classifier_dask/sklearn_classifier_dask.py @@ -14,27 +14,24 @@ # # Generated by nuclio.export.NuclioExporter -import mlrun - import warnings +import mlrun + warnings.filterwarnings("ignore") import joblib +import matplotlib.pyplot as plt import numpy as np import pandas as pd from cloudpickle import dumps - from dask import dataframe as dd from dask.delayed import delayed from dask_ml import model_selection -from dask_ml.preprocessing import StandardScaler, LabelEncoder - +from dask_ml.preprocessing import LabelEncoder, StandardScaler from mlrun.artifacts import PlotArtifact from mlrun.mlutils.models import gen_sklearn_model from mlrun.utils.helpers import create_class - -import matplotlib.pyplot as plt from yellowbrick.classifier import ROCAUC, ClassificationReport, ConfusionMatrix from yellowbrick.model_selection import FeatureImportances @@ -54,7 +51,6 @@ def train_model( file_ext: str = "parquet", random_state: int = 42, ) -> None: - """ Train a sklearn classifier with Dask @@ -149,12 +145,11 @@ def train_model( elif report_name == "ClassificationReport": for score_name in viz.scores_: for score_class in viz.scores_[score_name]: - context.log_results( { - score_name - + "-" - + score_class: viz.scores_[score_name].get(score_class) + score_name + "-" + score_class: viz.scores_[score_name].get( + score_class + ) } ) diff --git a/functions/src/structured_data_generator/function.yaml b/functions/src/structured_data_generator/function.yaml index 4e8a35626..e473c87f5 100644 --- a/functions/src/structured_data_generator/function.yaml +++ b/functions/src/structured_data_generator/function.yaml @@ -1,21 +1,27 @@ +metadata: + tag: '' + name: structured-data-generator + categories: + - data-generation + - genai +verbose: false +kind: job spec: + image: '' + disable_auto_mount: false build: origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgYXN0CmltcG9ydCBvcwoKaW1wb3J0IHRxZG0KZnJvbSBsYW5nY2hhaW4uY2hhdF9tb2RlbHMgaW1wb3J0IENoYXRPcGVuQUkKCgpkZWYgX3NldF9vcGVuYWlfc2VjcmV0cygpIC0+IGJvb2w6CiAgICBrZXkgPSAiT1BFTkFJX0FQSV9LRVkiCiAgICBiYXNlID0gIk9QRU5BSV9BUElfQkFTRSIKICAgICMgQ2hlY2sgaWYgdGhlIGtleSBpcyBhbHJlYWR5IGluIHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZXM6CiAgICBpZiBrZXkgaW4gb3MuZW52aXJvbiBhbmQgYmFzZSBpbiBvcy5lbnZpcm9uOgogICAgICAgIHJldHVybiBUcnVlCiAgICAjIENoZWNrIGlmIG1scnVuIGlzIGluc3RhbGxlZDoKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgICAgIHJhaXNlIE9TRXJyb3IoCiAgICAgICAgICAgIGYiT25lIG9yIG1vcmUgb2YgdGhlIE9wZW5BSSByZXF1aXJlZCBlbnZpcm9ubWVudCB2YXJpYWJsZXMgKCd7a2V5fScsICd7YmFzZX0nKSBhcmUgbWlzc2luZy4iCiAgICAgICAgICAgIGYiUGxlYXNlIHNldCB0aGVtIGFzIGVudmlyb25tZW50IHZhcmlhYmxlcyBvciBpbnN0YWxsIG1scnVuIChgcGlwIGluc3RhbGwgbWxydW5gKSIKICAgICAgICAgICAgZiJhbmQgc2V0IHRoZW0gYXMgcHJvamVjdCBzZWNyZXRzIHVzaW5nIGBwcm9qZWN5LnNldF9zZWNyZXRzYC4iCiAgICAgICAgKQoKICAgICMgQ2hlY2sgaWYgdGhlIGtleSBpcyBpbiB0aGUgc2VjcmV0czoKICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJjb250ZXh0IikKICAgIG9wZW5haV9rZXkgPSBjb250ZXh0LmdldF9zZWNyZXQoa2V5KQogICAgb3BlbmFpX2Jhc2UgPSBjb250ZXh0LmdldF9zZWNyZXQoYmFzZSkKCiAgICAjIElmIHRoZSBrZXkgaXMgbm90IGluIHRoZSBzZWNyZXRzLCByZXR1cm4gRmFsc2U6CiAgICBpZiBub3Qgb3BlbmFpX2tleToKICAgICAgICByYWlzZSBPU0Vycm9yKAogICAgICAgICAgICBmIkNvdWxkIG5vdCBmaW5kIE9wZW5BSSBBUEkga2V5IGluIHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZXMgb3Igc2VjcmV0cywiCiAgICAgICAgICAgIGYiIHBsZWFzZSBzZXQgaXQgYXM6IHtrZXl9LiIKICAgICAgICApCiAgICBpZiBub3Qgb3BlbmFpX2Jhc2U6CiAgICAgICAgcmFpc2UgT1NFcnJvcigKICAgICAgICAgICAgZiJDb3VsZCBub3QgZmluZCBPcGVuQUkgQVBJIGJhc2UgaW4gdGhlIGVudmlyb25tZW50IHZhcmlhYmxlcyBvciBzZWNyZXRzLCIKICAgICAgICAgICAgZiIgcGxlYXNlIHNldCBpdCBhczoge2Jhc2V9LiIKICAgICAgICApCiAgICAjIElmIHRoZSBrZXkgaXMgaW4gdGhlIHNlY3JldHMsIHNldCBpdCBpbiB0aGUgZW52aXJvbm1lbnQgdmFyaWFibGVzIGFuZCByZXR1cm4gVHJ1ZToKICAgIG9zLmVudmlyb25ba2V5XSA9IG9wZW5haV9rZXkKICAgIG9zLmVudmlyb25bYmFzZV0gPSBvcGVuYWlfYmFzZQogICAgcmV0dXJuIFRydWUKCgpkZWYgZ2VuZXJhdGVfZGF0YSgKICAgIGZpZWxkczogbGlzdCwKICAgIGFtb3VudDogaW50ID0gMTAsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSAiZ3B0LTMuNS10dXJibyIsCiAgICBsYW5ndWFnZTogc3RyID0gImVuIiwKICAgIGNodW5rX3NpemU6IGludCA9IDUwLAopIC0+IGxpc3Q6CiAgICAiIiIKICAgIFN0cnVjdHVyZWQgZGF0YSBvZiBlbGVtZW50cyBhY2NvcmRpbmcgdG8gdGhlIGdpdmVuIHBhcmFtZXRlcnMuCiAgICBUaGUgZGF0YSBjYW4gYmUgbGF0ZXIgbG9nZ2VkIGFzIGEgc3RydWN0dXJlZCBmaWxlIHdpdGggTUxSdW4ncyBgcmV0dXJuc2AgcGFyYW1ldGVyLgoKICAgIDpwYXJhbSBmaWVsZHM6IEEgbGlzdCBvZiBmaWVsZHMgdG8gcmFuZG9tbHkgZ2VuZXJhdGUuCiAgICA6cGFyYW0gYW1vdW50OiBUaGUgbnVtYmVyIG9mIHZhcmlhbnRzIHRvIGdlbmVyYXRlLgogICAgOnBhcmFtIG1vZGVsX25hbWU6IFRoZSBuYW1lIG9mIHRoZSBtb2RlbCB0byB1c2UgZm9yIGNvbnZlcnNhdGlvbiBnZW5lcmF0aW9uLgogICAgICAgICAgICAgICAgICAgICAgIFlvdSBzaG91bGQgY2hvb3NlIG9uZSBvZiBHUFQtNCBvciBHUFQtMy41IGZyb20gdGhlIGxpc3QgaGVyZTogaHR0cHM6Ly9wbGF0Zm9ybS5vcGVuYWkuY29tL2RvY3MvbW9kZWxzLgogICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHQ6ICdncHQtMy41LXR1cmJvJy4KICAgIDpwYXJhbSBsYW5ndWFnZTogVGhlIGxhbmd1YWdlIHRvIHVzZSBmb3IgdGhlIGdlbmVyYXRlZCBjb252ZXJzYXRpb24gdGV4dC4KICAgIDpwYXJhbSBjaHVua19zaXplOiBOdW1iZXIgb2Ygc2FtcGxlcyBnZW5lcmF0ZWQgYXQgZWFjaCBHUFQgcXVlcnkuCiAgICAiIiIKICAgIGluc3RydWN0aW9ucyA9ICIiCiAgICBmb3IgZmllbGQgaW4gZmllbGRzOgogICAgICAgICMgU3BsaXQgdGhlIGZpZWxkIHRvIGtleSBhbmQgaW5zdHJ1Y3Rpb246CiAgICAgICAgaWYgIjoiIGluIGZpZWxkOgogICAgICAgICAgICBrZXksIGluc3RydWN0aW9uID0gZmllbGQuc3BsaXQoIjoiLCAxKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGtleSwgaW5zdHJ1Y3Rpb24gPSBmaWVsZCwgIm5vIHNwZWNpYWwgaW5zdHJ1Y3Rpb24iCiAgICAgICAgIyBSZXBsYWNlIHNwYWNlcyB3aXRoIHVuZGVyc2NvcmVzIGZvciB0aGUga2V5IHRvIGJlIHVzZWQgYXMgYSBqc29uIGtleToKICAgICAgICBrZXkgPSBrZXkuc3RyaXAoKS5yZXBsYWNlKCIgIiwgIl8iKQogICAgICAgIGluc3RydWN0aW9ucyArPSBmIioge2tleX06IHtpbnN0cnVjdGlvbn1cbiIKCiAgICAjIENyZWF0ZSB0aGUgcHJvbXB0IHN0cnVjdHVyZToKICAgIHByb21wdF9zdHJ1Y3R1cmUgPSAoCiAgICAgICAgZiJnZW5lcmF0ZSB0aGUgZm9sbG93aW5nIHZhbHVlcyB7YW1vdW50fSB0aW1lcyByYW5kb21seSwgaW4gYW4gb3JkZXIgdGhhdCBjcmVhdGVzIGEganNvbiB0YWJsZS5cbiIKICAgICAgICBmIlVzZSB0aGUgZm9sbG93aW5nIGtleXMgYW5kIGluc3RydWN0aW9ucyAoZXhhbXBsZTogJ2tleTogaW5zdHJ1Y3Rpb24gb3Igbm8gc3BlY2lhbCBpbnN0cnVjdGlvbicpOiAiCiAgICAgICAgZiJ7aW5zdHJ1Y3Rpb25zfS5cbiIKICAgICAgICBmIlBsZWFzZSBnZW5lcmF0ZSB0aGUgdmFsdWVzIGluIHtsYW5ndWFnZX0gbGFuZ3VhZ2UuIFxuIgogICAgICAgIGYiTWFrZSBzdXJlIHRoZSBuYW1lcyBvZiB0aGUga2V5cyBhcmUgdGhlIHNhbWUgYXMgdGhlIGdpdmVuIGZpZWxkIG5hbWUuXG4iCiAgICAgICAgZiJQbGVhc2UgcmV0dXJuIG9ubHkgdGhlIGpzb24gZm9ybWF0IHdpdGhvdXQgYW55IGludHJvZHVjdGlvbiBhbmQgZW5kaW5nIgogICAgKQoKICAgICMgU2V0IHRoZSBPcGVuQUkgc2VjcmV0czoKICAgIF9zZXRfb3BlbmFpX3NlY3JldHMoKQoKICAgICMgTG9hZCB0aGUgT3BlbkFJIG1vZGVsIHVzaW5nIGxhbmdjaGFpbjoKICAgIGxsbSA9IENoYXRPcGVuQUkobW9kZWw9bW9kZWxfbmFtZSkKCiAgICAjIFN0YXJ0IGdlbmVyYXRpbmcgZGF0YToKICAgIGRhdGEgPSBbXQogICAgZm9yIF8gaW4gdHFkbS50cWRtKHJhbmdlKChhbW91bnQgLy8gY2h1bmtfc2l6ZSkgKyAxKSwgZGVzYz0iR2VuZXJhdGluZyIpOgogICAgICAgICMgV2UgdHJ5IHRvIGdlbmVyYXRlIHRoZSBkYXRhIDMgdGltZXMsIGlmIHdlIGZhaWwgd2UgcmFpc2UgYW4gZXJyb3I6CiAgICAgICAgZm9yIHRyeW91dCBpbiByYW5nZSgzKToKICAgICAgICAgICAgIyBJZiB0aGUgYW1vdW50IHdhbnRlZCBpcyBiaWdnZXIgdGhhbiB0aGUgY2h1bmsgc2l6ZSwgd2UgZ2VuZXJhdGUgYSBjaHVuayBvZiBkYXRhIGluIHRoZSBzaXplIG9mIHRoZSBjaHVuawogICAgICAgICAgICAjIGFuZCBkZWNyZWFzZSB0aGUgYW1vdW50IGJ5IHRoZSBjaHVuayBzaXplLgogICAgICAgICAgICAjIG90aGVyd2lzZSB3ZSBnZW5lcmF0ZSBhIGNodW5rIG9mIGRhdGEgaW4gdGhlIHNpemUgb2YgdGhlIGFtb3VudDoKICAgICAgICAgICAgaWYgYW1vdW50ID4gY2h1bmtfc2l6ZToKICAgICAgICAgICAgICAgIGN1cnJlbnRfY2h1bmtfc2l6ZSA9IGNodW5rX3NpemUKICAgICAgICAgICAgICAgIGFtb3VudCAtPSBjaHVua19zaXplCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICBjdXJyZW50X2NodW5rX3NpemUgPSBhbW91bnQKCiAgICAgICAgICAgICMgQ3JlYXRlIHRoZSBwcm9tcHQ6CiAgICAgICAgICAgIHByb21wdCA9IHByb21wdF9zdHJ1Y3R1cmUuZm9ybWF0KAogICAgICAgICAgICAgICAgYW1vdW50PWN1cnJlbnRfY2h1bmtfc2l6ZSwKICAgICAgICAgICAgKQoKICAgICAgICAgICAgIyBHZW5lcmF0ZSBhIGNodW5rIG9mIGRhdGE6CiAgICAgICAgICAgIGNodW5rX2RhdGEgPSBsbG0ucHJlZGljdCh0ZXh0PXByb21wdCkKCiAgICAgICAgICAgICMgVmFsaWRhdGUgdGhlIHJlc3BvbnNlIGZvciBjb3JyZWN0IHB5dGhvbiBgbGlzdGAgc3RydWN0dXJlCiAgICAgICAgICAgIGNodW5rX2RhdGEgPSBjaHVua19kYXRhW2NodW5rX2RhdGEuZmluZCgiWyIpIDogY2h1bmtfZGF0YS5yZmluZCgiXSIpICsgMV0KICAgICAgICAgICAgaWYgY2h1bmtfZGF0YS5jb3VudCgiWyIpICE9IGNodW5rX2RhdGEuY291bnQoIl0iKToKICAgICAgICAgICAgICAgIHByaW50KAogICAgICAgICAgICAgICAgICAgICJGYWlsZWQgdG8gZ2V0IHByb3BlciBqc29uIGZvcm1hdCBmcm9tIG1vZGVsLCBudW1iZXIgb2YgJ1snIGRvZXNuJ3QgbWF0Y2ggbnVtYmVyIG9mICddJy4iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICBjaHVua19kYXRhID0gYXN0LmxpdGVyYWxfZXZhbChjaHVua19kYXRhKQogICAgICAgICAgICBkYXRhICs9IGNodW5rX2RhdGEKICAgICAgICAgICAgYnJlYWsKICAgICAgICBpZiB0cnlvdXQgPT0gMzoKICAgICAgICAgICAgcmFpc2UgUnVudGltZUVycm9yKAogICAgICAgICAgICAgICAgZiJDb3VsZCBub3QgZ2VuZXJhdGUgYSBwcm9wZXIganNvbiBmb3JtYXQgZm9yIHRoZSBnaXZlbiBmaWVsZHMsIHVzaW5nIGdpdmVuIG1vZGVsOiB7bW9kZWxfbmFtZX0uIgogICAgICAgICAgICAgICAgZiIgSGludDogR3B0LTQgd29ya3MgYmVzdCBmb3IgbW9zdCBzY2VuYXJpb3MuIgogICAgICAgICAgICApCiAgICByZXR1cm4gZGF0YQo= requirements: - langchain - tqdm code_origin: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgYXN0CmltcG9ydCBvcwoKaW1wb3J0IHRxZG0KZnJvbSBsYW5nY2hhaW4uY2hhdF9tb2RlbHMgaW1wb3J0IENoYXRPcGVuQUkKCgpkZWYgX3NldF9vcGVuYWlfc2VjcmV0cygpIC0+IGJvb2w6CiAgICBrZXkgPSAiT1BFTkFJX0FQSV9LRVkiCiAgICBiYXNlID0gIk9QRU5BSV9BUElfQkFTRSIKICAgICMgQ2hlY2sgaWYgdGhlIGtleSBpcyBhbHJlYWR5IGluIHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZXM6CiAgICBpZiBrZXkgaW4gb3MuZW52aXJvbiBhbmQgYmFzZSBpbiBvcy5lbnZpcm9uOgogICAgICAgIHJldHVybiBUcnVlCiAgICAjIENoZWNrIGlmIG1scnVuIGlzIGluc3RhbGxlZDoKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgICAgIHJhaXNlIEVudmlyb25tZW50RXJyb3IoCiAgICAgICAgICAgIGYiT25lIG9yIG1vcmUgb2YgdGhlIE9wZW5BSSByZXF1aXJlZCBlbnZpcm9ubWVudCB2YXJpYWJsZXMgKCd7a2V5fScsICd7YmFzZX0nKSBhcmUgbWlzc2luZy4iCiAgICAgICAgICAgIGYiUGxlYXNlIHNldCB0aGVtIGFzIGVudmlyb25tZW50IHZhcmlhYmxlcyBvciBpbnN0YWxsIG1scnVuIChgcGlwIGluc3RhbGwgbWxydW5gKSIKICAgICAgICAgICAgZiJhbmQgc2V0IHRoZW0gYXMgcHJvamVjdCBzZWNyZXRzIHVzaW5nIGBwcm9qZWN5LnNldF9zZWNyZXRzYC4iCiAgICAgICAgKQoKICAgICMgQ2hlY2sgaWYgdGhlIGtleSBpcyBpbiB0aGUgc2VjcmV0czoKICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJjb250ZXh0IikKICAgIG9wZW5haV9rZXkgPSBjb250ZXh0LmdldF9zZWNyZXQoa2V5KQogICAgb3BlbmFpX2Jhc2UgPSBjb250ZXh0LmdldF9zZWNyZXQoYmFzZSkKCiAgICAjIElmIHRoZSBrZXkgaXMgbm90IGluIHRoZSBzZWNyZXRzLCByZXR1cm4gRmFsc2U6CiAgICBpZiBub3Qgb3BlbmFpX2tleToKICAgICAgICByYWlzZSBFbnZpcm9ubWVudEVycm9yKAogICAgICAgICAgICBmIkNvdWxkIG5vdCBmaW5kIE9wZW5BSSBBUEkga2V5IGluIHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZXMgb3Igc2VjcmV0cywiCiAgICAgICAgICAgIGYiIHBsZWFzZSBzZXQgaXQgYXM6IHtrZXl9LiIKICAgICAgICApCiAgICBpZiBub3Qgb3BlbmFpX2Jhc2U6CiAgICAgICAgcmFpc2UgRW52aXJvbm1lbnRFcnJvcigKICAgICAgICAgICAgZiJDb3VsZCBub3QgZmluZCBPcGVuQUkgQVBJIGJhc2UgaW4gdGhlIGVudmlyb25tZW50IHZhcmlhYmxlcyBvciBzZWNyZXRzLCIKICAgICAgICAgICAgZiIgcGxlYXNlIHNldCBpdCBhczoge2Jhc2V9LiIKICAgICAgICApCiAgICAjIElmIHRoZSBrZXkgaXMgaW4gdGhlIHNlY3JldHMsIHNldCBpdCBpbiB0aGUgZW52aXJvbm1lbnQgdmFyaWFibGVzIGFuZCByZXR1cm4gVHJ1ZToKICAgIG9zLmVudmlyb25ba2V5XSA9IG9wZW5haV9rZXkKICAgIG9zLmVudmlyb25bYmFzZV0gPSBvcGVuYWlfYmFzZQogICAgcmV0dXJuIFRydWUKCgpkZWYgZ2VuZXJhdGVfZGF0YSgKICAgIGZpZWxkczogbGlzdCwKICAgIGFtb3VudDogaW50ID0gMTAsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSAiZ3B0LTMuNS10dXJibyIsCiAgICBsYW5ndWFnZTogc3RyID0gImVuIiwKICAgIGNodW5rX3NpemU6IGludCA9IDUwLAopIC0+IGxpc3Q6CiAgICAiIiIKICAgIFN0cnVjdHVyZWQgZGF0YSBvZiBlbGVtZW50cyBhY2NvcmRpbmcgdG8gdGhlIGdpdmVuIHBhcmFtZXRlcnMuCiAgICBUaGUgZGF0YSBjYW4gYmUgbGF0ZXIgbG9nZ2VkIGFzIGEgc3RydWN0dXJlZCBmaWxlIHdpdGggTUxSdW4ncyBgcmV0dXJuc2AgcGFyYW1ldGVyLgoKICAgIDpwYXJhbSBmaWVsZHM6IEEgbGlzdCBvZiBmaWVsZHMgdG8gcmFuZG9tbHkgZ2VuZXJhdGUuCiAgICA6cGFyYW0gYW1vdW50OiBUaGUgbnVtYmVyIG9mIHZhcmlhbnRzIHRvIGdlbmVyYXRlLgogICAgOnBhcmFtIG1vZGVsX25hbWU6IFRoZSBuYW1lIG9mIHRoZSBtb2RlbCB0byB1c2UgZm9yIGNvbnZlcnNhdGlvbiBnZW5lcmF0aW9uLgogICAgICAgICAgICAgICAgICAgICAgIFlvdSBzaG91bGQgY2hvb3NlIG9uZSBvZiBHUFQtNCBvciBHUFQtMy41IGZyb20gdGhlIGxpc3QgaGVyZTogaHR0cHM6Ly9wbGF0Zm9ybS5vcGVuYWkuY29tL2RvY3MvbW9kZWxzLgogICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHQ6ICdncHQtMy41LXR1cmJvJy4KICAgIDpwYXJhbSBsYW5ndWFnZTogVGhlIGxhbmd1YWdlIHRvIHVzZSBmb3IgdGhlIGdlbmVyYXRlZCBjb252ZXJzYXRpb24gdGV4dC4KICAgIDpwYXJhbSBjaHVua19zaXplOiBOdW1iZXIgb2Ygc2FtcGxlcyBnZW5lcmF0ZWQgYXQgZWFjaCBHUFQgcXVlcnkuCiAgICAiIiIKICAgIGluc3RydWN0aW9ucyA9ICIiCiAgICBmb3IgZmllbGQgaW4gZmllbGRzOgogICAgICAgICMgU3BsaXQgdGhlIGZpZWxkIHRvIGtleSBhbmQgaW5zdHJ1Y3Rpb246CiAgICAgICAgaWYgIjoiIGluIGZpZWxkOgogICAgICAgICAgICBrZXksIGluc3RydWN0aW9uID0gZmllbGQuc3BsaXQoIjoiLCAxKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGtleSwgaW5zdHJ1Y3Rpb24gPSBmaWVsZCwgIm5vIHNwZWNpYWwgaW5zdHJ1Y3Rpb24iCiAgICAgICAgIyBSZXBsYWNlIHNwYWNlcyB3aXRoIHVuZGVyc2NvcmVzIGZvciB0aGUga2V5IHRvIGJlIHVzZWQgYXMgYSBqc29uIGtleToKICAgICAgICBrZXkgPSBrZXkuc3RyaXAoKS5yZXBsYWNlKCIgIiwgIl8iKQogICAgICAgIGluc3RydWN0aW9ucyArPSBmIioge2tleX06IHtpbnN0cnVjdGlvbn1cbiIKCiAgICAjIENyZWF0ZSB0aGUgcHJvbXB0IHN0cnVjdHVyZToKICAgIHByb21wdF9zdHJ1Y3R1cmUgPSAoCiAgICAgICAgZiJnZW5lcmF0ZSB0aGUgZm9sbG93aW5nIHZhbHVlcyB7YW1vdW50fSB0aW1lcyByYW5kb21seSwgaW4gYW4gb3JkZXIgdGhhdCBjcmVhdGVzIGEganNvbiB0YWJsZS5cbiIKICAgICAgICBmIlVzZSB0aGUgZm9sbG93aW5nIGtleXMgYW5kIGluc3RydWN0aW9ucyAoZXhhbXBsZTogJ2tleTogaW5zdHJ1Y3Rpb24gb3Igbm8gc3BlY2lhbCBpbnN0cnVjdGlvbicpOiAiCiAgICAgICAgZiJ7aW5zdHJ1Y3Rpb25zfS5cbiIKICAgICAgICBmIlBsZWFzZSBnZW5lcmF0ZSB0aGUgdmFsdWVzIGluIHtsYW5ndWFnZX0gbGFuZ3VhZ2UuIFxuIgogICAgICAgIGYiTWFrZSBzdXJlIHRoZSBuYW1lcyBvZiB0aGUga2V5cyBhcmUgdGhlIHNhbWUgYXMgdGhlIGdpdmVuIGZpZWxkIG5hbWUuXG4iCiAgICAgICAgZiJQbGVhc2UgcmV0dXJuIG9ubHkgdGhlIGpzb24gZm9ybWF0IHdpdGhvdXQgYW55IGludHJvZHVjdGlvbiBhbmQgZW5kaW5nIgogICAgKQoKICAgICMgU2V0IHRoZSBPcGVuQUkgc2VjcmV0czoKICAgIF9zZXRfb3BlbmFpX3NlY3JldHMoKQoKICAgICMgTG9hZCB0aGUgT3BlbkFJIG1vZGVsIHVzaW5nIGxhbmdjaGFpbjoKICAgIGxsbSA9IENoYXRPcGVuQUkobW9kZWw9bW9kZWxfbmFtZSkKCiAgICAjIFN0YXJ0IGdlbmVyYXRpbmcgZGF0YToKICAgIGRhdGEgPSBbXQogICAgZm9yIF8gaW4gdHFkbS50cWRtKHJhbmdlKChhbW91bnQgLy8gY2h1bmtfc2l6ZSkgKyAxKSwgZGVzYz0iR2VuZXJhdGluZyIpOgogICAgICAgICMgV2UgdHJ5IHRvIGdlbmVyYXRlIHRoZSBkYXRhIDMgdGltZXMsIGlmIHdlIGZhaWwgd2UgcmFpc2UgYW4gZXJyb3I6CiAgICAgICAgZm9yIHRyeW91dCBpbiByYW5nZSgzKToKICAgICAgICAgICAgIyBJZiB0aGUgYW1vdW50IHdhbnRlZCBpcyBiaWdnZXIgdGhhbiB0aGUgY2h1bmsgc2l6ZSwgd2UgZ2VuZXJhdGUgYSBjaHVuayBvZiBkYXRhIGluIHRoZSBzaXplIG9mIHRoZSBjaHVuawogICAgICAgICAgICAjIGFuZCBkZWNyZWFzZSB0aGUgYW1vdW50IGJ5IHRoZSBjaHVuayBzaXplLgogICAgICAgICAgICAjIG90aGVyd2lzZSB3ZSBnZW5lcmF0ZSBhIGNodW5rIG9mIGRhdGEgaW4gdGhlIHNpemUgb2YgdGhlIGFtb3VudDoKICAgICAgICAgICAgaWYgYW1vdW50ID4gY2h1bmtfc2l6ZToKICAgICAgICAgICAgICAgIGN1cnJlbnRfY2h1bmtfc2l6ZSA9IGNodW5rX3NpemUKICAgICAgICAgICAgICAgIGFtb3VudCAtPSBjaHVua19zaXplCiAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICBjdXJyZW50X2NodW5rX3NpemUgPSBhbW91bnQKCiAgICAgICAgICAgICMgQ3JlYXRlIHRoZSBwcm9tcHQ6CiAgICAgICAgICAgIHByb21wdCA9IHByb21wdF9zdHJ1Y3R1cmUuZm9ybWF0KAogICAgICAgICAgICAgICAgYW1vdW50PWN1cnJlbnRfY2h1bmtfc2l6ZSwKICAgICAgICAgICAgKQoKICAgICAgICAgICAgIyBHZW5lcmF0ZSBhIGNodW5rIG9mIGRhdGE6CiAgICAgICAgICAgIGNodW5rX2RhdGEgPSBsbG0ucHJlZGljdCh0ZXh0PXByb21wdCkKCiAgICAgICAgICAgICMgVmFsaWRhdGUgdGhlIHJlc3BvbnNlIGZvciBjb3JyZWN0IHB5dGhvbiBgbGlzdGAgc3RydWN0dXJlCiAgICAgICAgICAgIGNodW5rX2RhdGEgPSBjaHVua19kYXRhW2NodW5rX2RhdGEuZmluZCgiWyIpIDogY2h1bmtfZGF0YS5yZmluZCgiXSIpICsgMV0KICAgICAgICAgICAgaWYgY2h1bmtfZGF0YS5jb3VudCgiWyIpICE9IGNodW5rX2RhdGEuY291bnQoIl0iKToKICAgICAgICAgICAgICAgIHByaW50KAogICAgICAgICAgICAgICAgICAgICJGYWlsZWQgdG8gZ2V0IHByb3BlciBqc29uIGZvcm1hdCBmcm9tIG1vZGVsLCBudW1iZXIgb2YgJ1snIGRvZXNuJ3QgbWF0Y2ggbnVtYmVyIG9mICddJy4iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBjb250aW51ZQogICAgICAgICAgICBjaHVua19kYXRhID0gYXN0LmxpdGVyYWxfZXZhbChjaHVua19kYXRhKQogICAgICAgICAgICBkYXRhICs9IGNodW5rX2RhdGEKICAgICAgICAgICAgYnJlYWsKICAgICAgICBpZiB0cnlvdXQgPT0gMzoKICAgICAgICAgICAgcmFpc2UgUnVudGltZUVycm9yKAogICAgICAgICAgICAgICAgZiJDb3VsZCBub3QgZ2VuZXJhdGUgYSBwcm9wZXIganNvbiBmb3JtYXQgZm9yIHRoZSBnaXZlbiBmaWVsZHMsIHVzaW5nIGdpdmVuIG1vZGVsOiB7bW9kZWxfbmFtZX0uIgogICAgICAgICAgICAgICAgZiIgSGludDogR3B0LTQgd29ya3MgYmVzdCBmb3IgbW9zdCBzY2VuYXJpb3MuIgogICAgICAgICAgICApCiAgICByZXR1cm4gZGF0YQo= base_image: mlrun/mlrun + filename: structured_data_generator.py entry_points: generate_data: - has_varargs: false - name: generate_data - has_kwargs: false - doc: 'Structured data of elements according to the given parameters. - - The data can be later logged as a structured file with MLRun''s `returns` - parameter.' + outputs: + - type: list parameters: - name: fields type: list @@ -38,19 +44,14 @@ spec: type: int doc: Number of samples generated at each GPT query. default: 50 - outputs: - - type: list + name: generate_data + doc: 'Structured data of elements according to the given parameters. + + The data can be later logged as a structured file with MLRun''s `returns` + parameter.' + has_kwargs: false + has_varargs: false lineno: 59 command: '' description: GenAI approach of generating structured data according to a given schema default_handler: generate_data - disable_auto_mount: false - image: '' -metadata: - name: structured-data-generator - tag: '' - categories: - - data-generation - - genai -verbose: false -kind: job diff --git a/functions/src/structured_data_generator/structured_data_generator.py b/functions/src/structured_data_generator/structured_data_generator.py index 34fa36d49..d817ef274 100644 --- a/functions/src/structured_data_generator/structured_data_generator.py +++ b/functions/src/structured_data_generator/structured_data_generator.py @@ -28,7 +28,7 @@ def _set_openai_secrets() -> bool: try: import mlrun except ModuleNotFoundError: - raise EnvironmentError( + raise OSError( f"One or more of the OpenAI required environment variables ('{key}', '{base}') are missing." f"Please set them as environment variables or install mlrun (`pip install mlrun`)" f"and set them as project secrets using `projecy.set_secrets`." @@ -41,12 +41,12 @@ def _set_openai_secrets() -> bool: # If the key is not in the secrets, return False: if not openai_key: - raise EnvironmentError( + raise OSError( f"Could not find OpenAI API key in the environment variables or secrets," f" please set it as: {key}." ) if not openai_base: - raise EnvironmentError( + raise OSError( f"Could not find OpenAI API base in the environment variables or secrets," f" please set it as: {base}." ) diff --git a/functions/src/structured_data_generator/test_structured_data_generator.py b/functions/src/structured_data_generator/test_structured_data_generator.py index 3a7a7aa57..b1ddaba8a 100644 --- a/functions/src/structured_data_generator/test_structured_data_generator.py +++ b/functions/src/structured_data_generator/test_structured_data_generator.py @@ -1,4 +1,5 @@ import os + import mlrun import pytest @@ -8,11 +9,13 @@ def test_structured_data_generator(): # Create mlrun project project = mlrun.get_or_create_project("structured-data-generator-test") - #Set secrets + # Set secrets # project.set_secrets({"OPENAI_API_KEY": "", "OPENAI_API_BASE": ""}) # Import the function from the yaml file, once it's in the hub we can import from there - data_generation = project.set_function(func="structured_data_generator.py", name="structured_data_generator") + data_generation = project.set_function( + func="structured_data_generator.py", name="structured_data_generator" + ) # Run the imported function with desired file/s and params data_generation_run = data_generation.run( @@ -26,7 +29,7 @@ def test_structured_data_generator(): "last_name", "phone_number: at least 9 digits long", "email", - "client_id: at least 8 digits long, only numbers" + "client_id: at least 8 digits long, only numbers", ], }, returns=[ @@ -34,4 +37,4 @@ def test_structured_data_generator(): ], local=True, ) - assert data_generation_run.outputs["clients"] \ No newline at end of file + assert data_generation_run.outputs["clients"] diff --git a/functions/src/test_classifier/function.yaml b/functions/src/test_classifier/function.yaml index f35446b51..33b625c80 100644 --- a/functions/src/test_classifier/function.yaml +++ b/functions/src/test_classifier/function.yaml @@ -1,49 +1,35 @@ -kind: job metadata: - name: test-classifier tag: '' - hash: b4d447a2328975e90a0dbc7a28f82009924cc157 - project: '' - labels: - author: Iguazio - framework: sklearn + name: test-classifier categories: - machine-learning - model-testing +verbose: false +kind: job spec: - command: '' - args: [] image: mlrun/mlrun - env: [] - default_handler: test_classifier + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgp3YXJuaW5ncy5maWx0ZXJ3YXJuaW5ncygiaWdub3JlIikKCmltcG9ydCBwYW5kYXMgYXMgcGQKZnJvbSBjbG91ZHBpY2tsZSBpbXBvcnQgbG9hZApmcm9tIG1scnVuLmFydGlmYWN0cyBpbXBvcnQgZ2V0X21vZGVsLCB1cGRhdGVfbW9kZWwKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4ubWx1dGlscy5tb2RlbHMgaW1wb3J0IGV2YWxfbW9kZWxfdjIKCgpkZWYgdGVzdF9jbGFzc2lmaWVyKAogICAgY29udGV4dCwKICAgIG1vZGVsc19wYXRoOiBEYXRhSXRlbSwKICAgIHRlc3Rfc2V0OiBEYXRhSXRlbSwKICAgIGxhYmVsX2NvbHVtbjogc3RyLAogICAgc2NvcmVfbWV0aG9kOiBzdHIgPSAibWljcm8iLAogICAgcGxvdHNfZGVzdDogc3RyID0gIiIsCiAgICBtb2RlbF9ldmFsdWF0b3I9Tm9uZSwKICAgIGRlZmF1bHRfbW9kZWw6IHN0ciA9ICJtb2RlbC5wa2wiLAogICAgcHJlZGljdGlvbnNfY29sdW1uOiBzdHIgPSAieXNjb3JlIiwKICAgIG1vZGVsX3VwZGF0ZT1UcnVlLAopIC0+IE5vbmU6CiAgICAiIiJUZXN0IG9uZSBvciBtb3JlIGNsYXNzaWZpZXIgbW9kZWxzIGFnYWluc3QgaGVsZC1vdXQgZGF0YXNldAoKICAgIFVzaW5nIGhlbGQtb3V0IHRlc3QgZmVhdHVyZXMsIGV2YWx1YXRlcyB0aGUgcGVmb3JtYW5jZSBvZiB0aGUgZXN0aW1hdGVkIG1vZGVsCgogICAgQ2FuIGJlIHBhcnQgb2YgYSBrdWJlZmxvdyBwaXBlbGluZSBhcyBhIHRlc3Qgc3RlcCB0aGF0IGlzIHJ1biBwb3N0IEVEQSBhbmQKICAgIHRyYWluaW5nL3ZhbGlkYXRpb24gY3ljbGVzCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgdGhlIGZ1bmN0aW9uIGNvbnRleHQKICAgIDpwYXJhbSBtb2RlbHNfcGF0aDogICAgICAgIGFydGlmYWN0IG1vZGVscyByZXByZXNlbnRpbmcgYSBmaWxlIG9yIGEgZm9sZGVyCiAgICA6cGFyYW0gdGVzdF9zZXQ6ICAgICAgICAgICB0ZXN0IGZlYXR1cmVzIGFuZCBsYWJlbHMKICAgIDpwYXJhbSBsYWJlbF9jb2x1bW46ICAgICAgIGNvbHVtbiBuYW1lIGZvciBncm91bmQgdHJ1dGggbGFiZWxzCiAgICA6cGFyYW0gc2NvcmVfbWV0aG9kOiAgICAgICBmb3IgbXVsdGljbGFzcyBjbGFzc2lmaWNhdGlvbgogICAgOnBhcmFtIHBsb3RzX2Rlc3Q6ICAgICAgICAgZGlyIGZvciB0ZXN0IHBsb3RzCiAgICA6cGFyYW0gbW9kZWxfZXZhbHVhdG9yOiAgICBOT1QgSU1QTEVNRU5URUQ6IHNwZWNpZmljIG1ldGhvZCB0byBnZW5lcmF0ZSBldmFsLCBwYXNzZWQgaW4gYXMgc3RyaW5nCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvciBhdmFpbGFibGUgaW4gdGhpcyBmb2xkZXIKICAgIDpwYXJhbSBwcmVkaWN0aW9uc19jb2x1bW46IGNvbHVtbiBuYW1lIGZvciB0aGUgcHJlZGljdGlvbnMgY29sdW1uIG9uIHRoZSByZXN1bHRlZCBhcnRpZmFjdAogICAgOnBhcmFtIG1vZGVsX3VwZGF0ZTogICAgICAgKFRydWUpIHVwZGF0ZSBtb2RlbCwgd2hlbiBydW5uaW5nIGFzIHN0YW5kIGFsb25lIG5vIG5lZWQgaW4gdXBkYXRlCiAgICAiIiIKICAgIHh0ZXN0ID0gdGVzdF9zZXQuYXNfZGYoKQogICAgeXRlc3QgPSB4dGVzdC5wb3AobGFiZWxfY29sdW1uKQoKICAgIHRyeToKICAgICAgICBtb2RlbF9maWxlLCBtb2RlbF9vYmosIF8gPSBnZXRfbW9kZWwobW9kZWxzX3BhdGgsIHN1ZmZpeD0iLnBrbCIpCiAgICAgICAgbW9kZWxfb2JqID0gbG9hZChvcGVuKG1vZGVsX2ZpbGUsICJyYiIpKQogICAgZXhjZXB0IEV4Y2VwdGlvbjoKICAgICAgICByYWlzZSBFeGNlcHRpb24oIm1vZGVsIGxvY2F0aW9uIGxpa2VseSBzcGVjaWZpZWQiKQoKICAgIGV4dHJhX2RhdGEgPSBldmFsX21vZGVsX3YyKGNvbnRleHQsIHh0ZXN0LCB5dGVzdC52YWx1ZXMsIG1vZGVsX29iaikKICAgIGlmIG1vZGVsX29iaiBhbmQgbW9kZWxfdXBkYXRlID09IFRydWU6CiAgICAgICAgdXBkYXRlX21vZGVsKAogICAgICAgICAgICBtb2RlbHNfcGF0aCwKICAgICAgICAgICAgZXh0cmFfZGF0YT1leHRyYV9kYXRhLAogICAgICAgICAgICBtZXRyaWNzPWNvbnRleHQucmVzdWx0cywKICAgICAgICAgICAga2V5X3ByZWZpeD0idmFsaWRhdGlvbi0iLAogICAgICAgICkKCiAgICB5X2hhdCA9IG1vZGVsX29iai5wcmVkaWN0KHh0ZXN0KQogICAgaWYgeV9oYXQubmRpbSA9PSAxIG9yIHlfaGF0LnNoYXBlWzFdID09IDE6CiAgICAgICAgc2NvcmVfbmFtZXMgPSBbcHJlZGljdGlvbnNfY29sdW1uXQogICAgZWxzZToKICAgICAgICBzY29yZV9uYW1lcyA9IFtmIntwcmVkaWN0aW9uc19jb2x1bW59XyIgKyBzdHIoeCkgZm9yIHggaW4gcmFuZ2UoeV9oYXQuc2hhcGVbMV0pXQoKICAgIGRmID0gcGQuY29uY2F0KFt4dGVzdCwgeXRlc3QsIHBkLkRhdGFGcmFtZSh5X2hhdCwgY29sdW1ucz1zY29yZV9uYW1lcyldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KCJ0ZXN0X3NldF9wcmVkcyIsIGRmPWRmLCBmb3JtYXQ9InBhcnF1ZXQiLCBpbmRleD1GYWxzZSkK + code_origin: '' + filename: test_classifier.py entry_points: test_classifier: - name: test_classifier - doc: 'Test one or more classifier models against held-out dataset - - - Using held-out test features, evaluates the peformance of the estimated model - - - Can be part of a kubeflow pipeline as a test step that is run post EDA and - - training/validation cycles' + outputs: + - type: None parameters: - name: context doc: the function context - default: '' - name: models_path type: DataItem doc: artifact models representing a file or a folder - default: '' - name: test_set type: DataItem doc: test features and labels - default: '' - name: label_column type: str doc: column name for ground truth labels - default: '' - name: score_method type: str doc: for multiclass classification @@ -66,13 +52,19 @@ spec: - name: model_update doc: (True) update model, when running as stand alone no need in update default: true - outputs: - - default: '' - lineno: 17 + name: test_classifier + doc: 'Test one or more classifier models against held-out dataset + + + Using held-out test features, evaluates the peformance of the estimated model + + + Can be part of a kubeflow pipeline as a test step that is run post EDA and + + training/validation cycles' + has_kwargs: false + has_varargs: false + lineno: 28 + command: '' description: test a classifier using held-out or new data - build: - functionSourceCode: IyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgp3YXJuaW5ncy5maWx0ZXJ3YXJuaW5ncygiaWdub3JlIikKCmltcG9ydCBvcwppbXBvcnQgcGFuZGFzIGFzIHBkCgpmcm9tIG1scnVuLmRhdGFzdG9yZSBpbXBvcnQgRGF0YUl0ZW0KZnJvbSBtbHJ1bi5hcnRpZmFjdHMgaW1wb3J0IGdldF9tb2RlbCwgdXBkYXRlX21vZGVsCmZyb20gbWxydW4ubWx1dGlscy5tb2RlbHMgaW1wb3J0IGV2YWxfbW9kZWxfdjIKZnJvbSBjbG91ZHBpY2tsZSBpbXBvcnQgbG9hZApmcm9tIHVybGxpYi5yZXF1ZXN0IGltcG9ydCB1cmxvcGVuCgoKZGVmIHRlc3RfY2xhc3NpZmllcigKICAgIGNvbnRleHQsCiAgICBtb2RlbHNfcGF0aDogRGF0YUl0ZW0sCiAgICB0ZXN0X3NldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHNjb3JlX21ldGhvZDogc3RyID0gIm1pY3JvIiwKICAgIHBsb3RzX2Rlc3Q6IHN0ciA9ICIiLAogICAgbW9kZWxfZXZhbHVhdG9yPU5vbmUsCiAgICBkZWZhdWx0X21vZGVsOiBzdHIgPSAibW9kZWwucGtsIiwKICAgIHByZWRpY3Rpb25zX2NvbHVtbjogc3RyID0gInlzY29yZSIsCiAgICBtb2RlbF91cGRhdGU9VHJ1ZSwKKSAtPiBOb25lOgogICAgIiIiVGVzdCBvbmUgb3IgbW9yZSBjbGFzc2lmaWVyIG1vZGVscyBhZ2FpbnN0IGhlbGQtb3V0IGRhdGFzZXQKCiAgICBVc2luZyBoZWxkLW91dCB0ZXN0IGZlYXR1cmVzLCBldmFsdWF0ZXMgdGhlIHBlZm9ybWFuY2Ugb2YgdGhlIGVzdGltYXRlZCBtb2RlbAoKICAgIENhbiBiZSBwYXJ0IG9mIGEga3ViZWZsb3cgcGlwZWxpbmUgYXMgYSB0ZXN0IHN0ZXAgdGhhdCBpcyBydW4gcG9zdCBFREEgYW5kCiAgICB0cmFpbmluZy92YWxpZGF0aW9uIGN5Y2xlcwoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgIHRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxzX3BhdGg6ICAgICAgICBhcnRpZmFjdCBtb2RlbHMgcmVwcmVzZW50aW5nIGEgZmlsZSBvciBhIGZvbGRlcgogICAgOnBhcmFtIHRlc3Rfc2V0OiAgICAgICAgICAgdGVzdCBmZWF0dXJlcyBhbmQgbGFiZWxzCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uOiAgICAgICBjb2x1bW4gbmFtZSBmb3IgZ3JvdW5kIHRydXRoIGxhYmVscwogICAgOnBhcmFtIHNjb3JlX21ldGhvZDogICAgICAgZm9yIG11bHRpY2xhc3MgY2xhc3NpZmljYXRpb24KICAgIDpwYXJhbSBwbG90c19kZXN0OiAgICAgICAgIGRpciBmb3IgdGVzdCBwbG90cwogICAgOnBhcmFtIG1vZGVsX2V2YWx1YXRvcjogICAgTk9UIElNUExFTUVOVEVEOiBzcGVjaWZpYyBtZXRob2QgdG8gZ2VuZXJhdGUgZXZhbCwgcGFzc2VkIGluIGFzIHN0cmluZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb3IgYXZhaWxhYmxlIGluIHRoaXMgZm9sZGVyCiAgICA6cGFyYW0gcHJlZGljdGlvbnNfY29sdW1uOiBjb2x1bW4gbmFtZSBmb3IgdGhlIHByZWRpY3Rpb25zIGNvbHVtbiBvbiB0aGUgcmVzdWx0ZWQgYXJ0aWZhY3QKICAgIDpwYXJhbSBtb2RlbF91cGRhdGU6ICAgICAgIChUcnVlKSB1cGRhdGUgbW9kZWwsIHdoZW4gcnVubmluZyBhcyBzdGFuZCBhbG9uZSBubyBuZWVkIGluIHVwZGF0ZQogICAgIiIiCiAgICB4dGVzdCA9IHRlc3Rfc2V0LmFzX2RmKCkKICAgIHl0ZXN0ID0geHRlc3QucG9wKGxhYmVsX2NvbHVtbikKCiAgICB0cnk6CiAgICAgICAgbW9kZWxfZmlsZSwgbW9kZWxfb2JqLCBfID0gZ2V0X21vZGVsKG1vZGVsc19wYXRoLCBzdWZmaXg9Ii5wa2wiKQogICAgICAgIG1vZGVsX29iaiA9IGxvYWQob3Blbihtb2RlbF9maWxlLCAicmIiKSkKICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgYToKICAgICAgICByYWlzZSBFeGNlcHRpb24oIm1vZGVsIGxvY2F0aW9uIGxpa2VseSBzcGVjaWZpZWQiKQoKICAgIGV4dHJhX2RhdGEgPSBldmFsX21vZGVsX3YyKGNvbnRleHQsIHh0ZXN0LCB5dGVzdC52YWx1ZXMsIG1vZGVsX29iaikKICAgIGlmIG1vZGVsX29iaiBhbmQgbW9kZWxfdXBkYXRlID09IFRydWU6CiAgICAgICAgdXBkYXRlX21vZGVsKAogICAgICAgICAgICBtb2RlbHNfcGF0aCwKICAgICAgICAgICAgZXh0cmFfZGF0YT1leHRyYV9kYXRhLAogICAgICAgICAgICBtZXRyaWNzPWNvbnRleHQucmVzdWx0cywKICAgICAgICAgICAga2V5X3ByZWZpeD0idmFsaWRhdGlvbi0iLAogICAgICAgICkKCiAgICB5X2hhdCA9IG1vZGVsX29iai5wcmVkaWN0KHh0ZXN0KQogICAgaWYgeV9oYXQubmRpbSA9PSAxIG9yIHlfaGF0LnNoYXBlWzFdID09IDE6CiAgICAgICAgc2NvcmVfbmFtZXMgPSBbcHJlZGljdGlvbnNfY29sdW1uXQogICAgZWxzZToKICAgICAgICBzY29yZV9uYW1lcyA9IFtmIntwcmVkaWN0aW9uc19jb2x1bW59XyIgKyBzdHIoeCkgZm9yIHggaW4gcmFuZ2UoeV9oYXQuc2hhcGVbMV0pXQoKICAgIGRmID0gcGQuY29uY2F0KFt4dGVzdCwgeXRlc3QsIHBkLkRhdGFGcmFtZSh5X2hhdCwgY29sdW1ucz1zY29yZV9uYW1lcyldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KCJ0ZXN0X3NldF9wcmVkcyIsIGRmPWRmLCBmb3JtYXQ9InBhcnF1ZXQiLCBpbmRleD1GYWxzZSkK - commands: [] - code_origin: https://github.com/daniels290813/functions.git#55a79c32be5d233cc11efcf40cd3edbe309bfdef:/home/kali/functions/test_classifier/test_classifier.py - affinity: null -verbose: false + default_handler: test_classifier diff --git a/functions/src/test_classifier/test_classifier.py b/functions/src/test_classifier/test_classifier.py index 322ecefc5..c11a6d99e 100644 --- a/functions/src/test_classifier/test_classifier.py +++ b/functions/src/test_classifier/test_classifier.py @@ -18,14 +18,11 @@ warnings.filterwarnings("ignore") -import os import pandas as pd - -from mlrun.datastore import DataItem +from cloudpickle import load from mlrun.artifacts import get_model, update_model +from mlrun.datastore import DataItem from mlrun.mlutils.models import eval_model_v2 -from cloudpickle import load -from urllib.request import urlopen def test_classifier( @@ -64,7 +61,7 @@ def test_classifier( try: model_file, model_obj, _ = get_model(models_path, suffix=".pkl") model_obj = load(open(model_file, "rb")) - except Exception as a: + except Exception: raise Exception("model location likely specified") extra_data = eval_model_v2(context, xtest, ytest.values, model_obj) diff --git a/functions/src/text_to_audio_generator/function.yaml b/functions/src/text_to_audio_generator/function.yaml index 8edbde74f..1a4c2fc72 100644 --- a/functions/src/text_to_audio_generator/function.yaml +++ b/functions/src/text_to_audio_generator/function.yaml @@ -1,21 +1,40 @@ +metadata: + tag: '' + name: text-to-audio-generator + categories: + - data-generation + - audio +verbose: false +kind: job spec: - default_handler: generate_multi_speakers_audio + image: '' disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgaW1wb3J0bGliCmltcG9ydCBpbwppbXBvcnQgbG9nZ2luZwppbXBvcnQgb3MKaW1wb3J0IHBhdGhsaWIKaW1wb3J0IHJhbmRvbQppbXBvcnQgdGVtcGZpbGUKZnJvbSBhYmMgaW1wb3J0IEFCQywgYWJzdHJhY3RtZXRob2QKCmltcG9ydCBudW1weSBhcyBucAppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCB0b3JjaAppbXBvcnQgdG9yY2hhdWRpbwppbXBvcnQgdHFkbQoKIyBHZXQgdGhlIGdsb2JhbCBsb2dnZXI6Cl9MT0dHRVIgPSBsb2dnaW5nLmdldExvZ2dlcigpCgpPUEVOQUlfQVBJX0tFWSA9ICJPUEVOQUlfQVBJX0tFWSIKT1BFTkFJX0JBU0VfVVJMID0gIk9QRU5BSV9BUElfQkFTRSIKU0FNUExFX1JBVEUgPSAyNDAwMAoKCmRlZiBnZW5lcmF0ZV9tdWx0aV9zcGVha2Vyc19hdWRpbygKICAgIGRhdGFfcGF0aDogc3RyLAogICAgc3BlYWtlcnM6IGxpc3Rbc3RyXSB8IGRpY3Rbc3RyLCBpbnRdLAogICAgYXZhaWxhYmxlX3ZvaWNlczogbGlzdFtzdHJdLAogICAgZW5naW5lOiBzdHIgPSAib3BlbmFpIiwKICAgIG91dHB1dF9kaXJlY3Rvcnk6IHN0ciA9IE5vbmUsCiAgICB1c2VfZ3B1OiBib29sIHwgTm9uZSA9IE5vbmUsCiAgICB1c2Vfc21hbGxfbW9kZWxzOiBib29sIHwgTm9uZSA9IE5vbmUsCiAgICBvZmZsb2FkX2NwdTogYm9vbCB8IE5vbmUgPSBOb25lLAogICAgbW9kZWw6IHN0ciB8IE5vbmUgPSBOb25lLAogICAgc3BlZWQ6IGZsb2F0IHwgTm9uZSA9IE5vbmUsCiAgICBzYW1wbGVfcmF0ZTogaW50ID0gMTYwMDAsCiAgICBmaWxlX2Zvcm1hdDogc3RyID0gIndhdiIsCiAgICB2ZXJib3NlOiBib29sID0gVHJ1ZSwKICAgIGJpdHNfcGVyX3NhbXBsZTogaW50IHwgTm9uZSA9IE5vbmUsCikgLT4gdHVwbGVbc3RyLCBwZC5EYXRhRnJhbWUsIGRpY3RdOgogICAgIiIiCiAgICBHZW5lcmF0ZSBhdWRpbyBmaWxlcyBmcm9tIHRleHQgZmlsZXMuCgogICAgOnBhcmFtIGRhdGFfcGF0aDogICAgICAgICAgIFBhdGggdG8gdGhlIHRleHQgZmlsZSBvciBkaXJlY3RvcnkgY29udGFpbmluZyB0aGUgdGV4dCBmaWxlcyB0byBnZW5lcmF0ZSBhdWRpbyBmcm9tLgogICAgOnBhcmFtIHNwZWFrZXJzOiAgICAgICAgICAgIExpc3QgLyBEaWN0IG9mIHNwZWFrZXJzIHRvIGdlbmVyYXRlIGF1ZGlvIGZvci4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBhIGxpc3QgaXMgZ2l2ZW4sIHRoZSBzcGVha2VycyB3aWxsIGJlIGFzc2lnbmVkIHRvIGNoYW5uZWxzIGluIHRoZSBvcmRlciBnaXZlbi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBkaWN0aW9uYXJ5LCB0aGUga2V5cyB3aWxsIGJlIHRoZSBzcGVha2VycyBhbmQgdGhlIHZhbHVlcyB3aWxsIGJlIHRoZSBjaGFubmVscy4KICAgIDpwYXJhbSBhdmFpbGFibGVfdm9pY2VzOiAgICBMaXN0IG9mIGF2YWlsYWJsZSB2b2ljZXMgdG8gdXNlIGZvciB0aGUgZ2VuZXJhdGlvbi4KICAgICAgICAgICAgICAgICAgICAgICAgU2VlIGhlcmUgZm9yIHRoZSBhdmFpbGFibGUgdm9pY2VzIGZvciBiYXJrIGVuZ2luZToKICAgICAgICAgICAgICAgICAgICAgICAgaHR0cHM6Ly9zdW5vLWFpLm5vdGlvbi5zaXRlLzhiOGU4NzQ5ZWQ1MTRiMGNiZjNmNjk5MDEzNTQ4NjgzP3Y9YmM2N2NmZjc4NmIwNGI1MGIzY2ViNzU2ZmQwNWY2OGMKICAgICAgICAgICAgICAgICAgICAgICAgU2VlIGhlcmUgZm9yIHRoZSBhdmFpbGFibGUgdm9pY2VzIGZvciBvcGVuYWkgZW5naW5lOgogICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL2JldGEub3BlbmFpLmNvbS9kb2NzL2FwaS1yZWZlcmVuY2Uvc3BlZWNoCiAgICA6cGFyYW0gZW5naW5lOiAgICAgICAgICAgICAgVGhlIGVuZ2luZSB0byB1c2UgZm9yIHRoZSBnZW5lcmF0aW9uLiBTZWxlY3QgZWl0aGVyICJiYXJrIiBvciAib3BlbmFpIi4gRGVmYXVsdCBpcyAib3BlbmFpIi4KICAgIDpwYXJhbSBvdXRwdXRfZGlyZWN0b3J5OiAgICBQYXRoIHRvIHRoZSBkaXJlY3RvcnkgdG8gc2F2ZSB0aGUgZ2VuZXJhdGVkIGF1ZGlvIGZpbGVzIHRvLgogICAgOnBhcmFtIHVzZV9ncHU6ICAgICAgICAgICAgIFdoZXRoZXIgdG8gdXNlIHRoZSBHUFUgZm9yIHRoZSBnZW5lcmF0aW9uLiBTdXBwb3J0ZWQgb25seSBpbiAiYmFyayIgZW5naW5lLgogICAgOnBhcmFtIHVzZV9zbWFsbF9tb2RlbHM6ICAgIFdoZXRoZXIgdG8gdXNlIHRoZSBzbWFsbCBtb2RlbHMgZm9yIHRoZSBnZW5lcmF0aW9uLiBTdXBwb3J0ZWQgb25seSBpbiAiYmFyayIgZW5naW5lLgogICAgOnBhcmFtIG9mZmxvYWRfY3B1OiAgICAgICAgIFRvIHJlZHVjZSB0aGUgbWVtb3J5IGZvb3RwcmludCwgdGhlIG1vZGVscyBjYW4gYmUgb2ZmbG9hZGVkIHRvIHRoZSBDUFUgYWZ0ZXIgbG9hZGluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTdXBwb3J0ZWQgb25seSBpbiAiYmFyayIgZW5naW5lLgogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgIFdoaWNoIG1vZGVsIHRvIHVzZSBmb3IgdGhlIGdlbmVyYXRpb24uIFN1cHBvcnRlZCBvbmx5IGluICJvcGVuYWkiIGVuZ2luZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0IGlzICJ0dHMtMSIuCiAgICA6cGFyYW0gc3BlZWQ6ICAgICAgICAgICAgICAgVGhlIHNwZWVkIG9mIHRoZSBnZW5lcmF0ZWQgYXVkaW8uIFNlbGVjdCBhIHZhbHVlIGZyb20gYDAuMjVgIHRvIGA0LjBgLiBgMS4wYCBpcyB0aGUgZGVmYXVsdC4KICAgIDpwYXJhbSBzYW1wbGVfcmF0ZTogICAgICAgICBUaGUgc2FtcGxpbmcgcmF0ZSBvZiB0aGUgZ2VuZXJhdGVkIGF1ZGlvLgogICAgOnBhcmFtIGZpbGVfZm9ybWF0OiAgICAgICAgIFRoZSBmb3JtYXQgb2YgdGhlIGdlbmVyYXRlZCBhdWRpbyBmaWxlcy4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICBXaGV0aGVyIHRvIHByaW50IHRoZSBwcm9ncmVzcyBvZiB0aGUgZ2VuZXJhdGlvbi4KICAgIDpwYXJhbSBiaXRzX3Blcl9zYW1wbGU6ICAgICBDaGFuZ2VzIHRoZSBiaXQgZGVwdGggZm9yIHRoZSBzdXBwb3J0ZWQgZm9ybWF0cy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTdXBwb3J0ZWQgb25seSBpbiAid2F2IiBvciAiZmxhYyIgZm9ybWF0cy4KCiAgICA6cmV0dXJuczogICAgICAgICAgICAgICAgICAgQSB0dXBsZSBvZjoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAtIFRoZSBvdXRwdXQgZGlyZWN0b3J5IHBhdGguCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBUaGUgZ2VuZXJhdGVkIGF1ZGlvIGZpbGVzIGRhdGFmcmFtZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAtIFRoZSBlcnJvcnMnIGRpY3Rpb25hcnkuCiAgICAiIiIKCiAgICBnbG9iYWwgX0xPR0dFUgogICAgX0xPR0dFUiA9IF9nZXRfbG9nZ2VyKCkKICAgICMgR2V0IHRoZSBpbnB1dCB0ZXh0IGZpbGVzIHRvIHR1cm4gdG8gYXVkaW86CiAgICBkYXRhX3BhdGggPSBwYXRobGliLlBhdGgoZGF0YV9wYXRoKS5hYnNvbHV0ZSgpCiAgICB0ZXh0X2ZpbGVzID0gX2dldF90ZXh0X2ZpbGVzKGRhdGFfcGF0aD1kYXRhX3BhdGgpCgogICAgIyBQcmVwYXJlIHRoZSBzcGVlY2ggZW5naW5lOgogICAgZW5naW5lID0gX2dldF9lbmdpbmUoCiAgICAgICAgZW5naW5lPWVuZ2luZSwKICAgICAgICB1c2VfZ3B1PXVzZV9ncHUsCiAgICAgICAgdXNlX3NtYWxsX21vZGVscz11c2Vfc21hbGxfbW9kZWxzLAogICAgICAgIG9mZmxvYWRfY3B1PW9mZmxvYWRfY3B1LAogICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgIGZpbGVfZm9ybWF0PWZpbGVfZm9ybWF0LAogICAgICAgIHNwZWVkPXNwZWVkLAogICAgKQoKICAgICMgQ2hlY2sgZm9yIHBlciBjaGFubmVsIGdlbmVyYXRpb246CiAgICBpZiBpc2luc3RhbmNlKHNwZWFrZXJzLCBkaWN0KToKICAgICAgICBzcGVha2VyX3Blcl9jaGFubmVsID0gVHJ1ZQogICAgICAgICMgU29ydCB0aGUgZ2l2ZW4gc3BlYWtlcnMgYnkgY2hhbm5lbHM6CiAgICAgICAgc3BlYWtlcnMgPSB7CiAgICAgICAgICAgIHNwZWFrZXI6IGNoYW5uZWwKICAgICAgICAgICAgZm9yIHNwZWFrZXIsIGNoYW5uZWwgaW4gc29ydGVkKHNwZWFrZXJzLml0ZW1zKCksIGtleT1sYW1iZGEgaXRlbTogaXRlbVsxXSkKICAgICAgICB9CiAgICBlbHNlOgogICAgICAgIHNwZWFrZXJfcGVyX2NoYW5uZWwgPSBGYWxzZQoKICAgICMgUHJlcGFyZSB0aGUgcmVzYW1wbGluZyBtb2R1bGU6CiAgICByZXNhbXBsZXIgPSB0b3JjaGF1ZGlvLnRyYW5zZm9ybXMuUmVzYW1wbGUoCiAgICAgICAgb3JpZ19mcmVxPVNBTVBMRV9SQVRFLCBuZXdfZnJlcT1zYW1wbGVfcmF0ZSwgZHR5cGU9dG9yY2guZmxvYXQzMgogICAgKQoKICAgICMgUHJlcGFyZSB0aGUgZ2FwIGJldHdlZW4gZWFjaCBzcGVha2VyOgogICAgZ2FwX2JldHdlZW5fc3BlYWtlcnMgPSBucC56ZXJvcyhpbnQoMC41ICogU0FNUExFX1JBVEUpKQoKICAgICMgUHJlcGFyZSB0aGUgc3VjY2Vzc2VzIGRhdGFmcmFtZSBhbmQgZXJyb3JzIGRpY3Rpb25hcnkgdG8gYmUgcmV0dXJuZWQ6CiAgICBzdWNjZXNzZXMgPSBbXQogICAgZXJyb3JzID0ge30KCiAgICAjIENyZWF0ZSB0aGUgb3V0cHV0IGRpcmVjdG9yeToKICAgIGlmIG91dHB1dF9kaXJlY3RvcnkgaXMgTm9uZToKICAgICAgICBvdXRwdXRfZGlyZWN0b3J5ID0gdGVtcGZpbGUubWtkdGVtcCgpCiAgICBvdXRwdXRfZGlyZWN0b3J5ID0gcGF0aGxpYi5QYXRoKG91dHB1dF9kaXJlY3RvcnkpCiAgICBpZiBub3Qgb3V0cHV0X2RpcmVjdG9yeS5leGlzdHMoKToKICAgICAgICBvdXRwdXRfZGlyZWN0b3J5Lm1rZGlyKGV4aXN0X29rPVRydWUsIHBhcmVudHM9VHJ1ZSkKCiAgICAjIFN0YXJ0IGdlbmVyYXRpbmcgYXVkaW86CiAgICAjIEdvIG92ZXIgdGhlIGF1ZGlvIGZpbGVzIGFuZCB0cmFuc2NyaWJlOgogICAgZm9yIHRleHRfZmlsZSBpbiB0cWRtLnRxZG0oCiAgICAgICAgdGV4dF9maWxlcywgZGVzYz0iR2VuZXJhdGluZyIsIHVuaXQ9ImZpbGUiLCBkaXNhYmxlPW5vdCB2ZXJib3NlCiAgICApOgogICAgICAgIHRyeToKICAgICAgICAgICAgIyBSYW5kb21pemUgdm9pY2VzIGZvciBlYWNoIHNwZWFrZXI6CiAgICAgICAgICAgIGNob3Nlbl92b2ljZXMgPSB7fQogICAgICAgICAgICBhdmFpbGFibGVfdm9pY2VzX2NvcHkgPSBhdmFpbGFibGVfdm9pY2VzLmNvcHkoKQogICAgICAgICAgICBmb3Igc3BlYWtlciBpbiBzcGVha2VyczoKICAgICAgICAgICAgICAgIHZvaWNlID0gcmFuZG9tLmNob2ljZShhdmFpbGFibGVfdm9pY2VzX2NvcHkpCiAgICAgICAgICAgICAgICBjaG9zZW5fdm9pY2VzW3NwZWFrZXJdID0gdm9pY2UKICAgICAgICAgICAgICAgIGF2YWlsYWJsZV92b2ljZXNfY29weS5yZW1vdmUodm9pY2UpCiAgICAgICAgICAgICMgUmVhZCB0ZXh0OgogICAgICAgICAgICB3aXRoIG9wZW4odGV4dF9maWxlKSBhcyBmcDoKICAgICAgICAgICAgICAgIHRleHQgPSBmcC5yZWFkKCkKICAgICAgICAgICAgIyBQcmVwYXJlIGEgaG9sZGVyIGZvciBhbGwgdGhlIGdlbmVyYXRlZCBwaWVjZXMgKGlmIHBlciBjaGFubmVsIGVhY2ggc3BlYWtlciB3aWxsIGhhdmUgaXRzIG93bik6CiAgICAgICAgICAgIGF1ZGlvX3BpZWNlcyA9ICgKICAgICAgICAgICAgICAgIHtzcGVha2VyOiBbXSBmb3Igc3BlYWtlciBpbiBzcGVha2Vyc30KICAgICAgICAgICAgICAgIGlmIHNwZWFrZXJfcGVyX2NoYW5uZWwKICAgICAgICAgICAgICAgIGVsc2UgeyJhbGwiOiBbXX0KICAgICAgICAgICAgKQoKICAgICAgICAgICAgIyBHZW5lcmF0ZSBhdWRpbyBwZXIgbGluZToKICAgICAgICAgICAgZm9yIGxpbmUgaW4gdGV4dC5zcGxpdGxpbmVzKCk6CiAgICAgICAgICAgICAgICAjIFZhbGlkYXRlIGxpbmUgaXMgaW4gY29ycmVjdCBzcGVha2VyIGZvcm1hdDoKCiAgICAgICAgICAgICAgICBpZiAiOiAiIG5vdCBpbiBsaW5lOgogICAgICAgICAgICAgICAgICAgIGlmIHZlcmJvc2U6CiAgICAgICAgICAgICAgICAgICAgICAgIF9MT0dHRVIud2FybmluZyhmIlNraXBwaW5nIGxpbmU6IHtsaW5lfSIpCiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgICMgU3BsaXQgbGluZSB0byBzcGVha2VyIGFuZCBoaXMgd29yZHM6CiAgICAgICAgICAgICAgICBjdXJyZW50X3NwZWFrZXIsIHNlbnRlbmNlcyA9IGxpbmUuc3BsaXQoIjogIiwgMSkKICAgICAgICAgICAgICAgICMgVmFsaWRhdGUgc3BlYWtlciBpcyBrbm93bjoKICAgICAgICAgICAgICAgIGlmIGN1cnJlbnRfc3BlYWtlciBub3QgaW4gc3BlYWtlcnM6CiAgICAgICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAgICAgZiJVbmtub3duIHNwZWFrZXI6IHtjdXJyZW50X3NwZWFrZXJ9LiBHaXZlbiBzcGVha2VycyBhcmU6IHtzcGVha2Vyc30iCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZm9yIHNlbnRlbmNlIGluIF9zcGxpdF9saW5lKGxpbmU9c2VudGVuY2VzKToKICAgICAgICAgICAgICAgICAgICAjIEdlbmVyYXRlIHdvcmRzIGF1ZGlvOgogICAgICAgICAgICAgICAgICAgIGF1ZGlvID0gZW5naW5lLl9nZW5lcmF0ZV9hdWRpbygKICAgICAgICAgICAgICAgICAgICAgICAgdGV4dD1zZW50ZW5jZSwKICAgICAgICAgICAgICAgICAgICAgICAgdm9pY2U9Y2hvc2VuX3ZvaWNlc1tjdXJyZW50X3NwZWFrZXJdLAogICAgICAgICAgICAgICAgICAgICkKCiAgICAgICAgICAgICAgICAgICAgaWYgc3BlYWtlcl9wZXJfY2hhbm5lbDoKICAgICAgICAgICAgICAgICAgICAgICAgc2lsZW5jZSA9IG5wLnplcm9zX2xpa2UoYXVkaW8pCiAgICAgICAgICAgICAgICAgICAgICAgIGZvciBzcGVha2VyIGluIGF1ZGlvX3BpZWNlcy5rZXlzKCk6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICBpZiBzcGVha2VyID09IGN1cnJlbnRfc3BlYWtlcjoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhdWRpb19waWVjZXNbc3BlYWtlcl0gKz0gW2F1ZGlvLCBnYXBfYmV0d2Vlbl9zcGVha2Vyc10KICAgICAgICAgICAgICAgICAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYXVkaW9fcGllY2VzW3NwZWFrZXJdICs9IFtzaWxlbmNlLCBnYXBfYmV0d2Vlbl9zcGVha2Vyc10KICAgICAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgICAgICBhdWRpb19waWVjZXNbImFsbCJdICs9IFthdWRpbywgZ2FwX2JldHdlZW5fc3BlYWtlcnNdCiAgICAgICAgICAgICMgQ29uc3RydWN0IGEgc2luZ2xlIGF1ZGlvIGFycmF5IGZyb20gYWxsIHRoZSBwaWVjZXMgYW5kIGNoYW5uZWxzOgoKICAgICAgICAgICAgYXVkaW8gPSBucC52c3RhY2soCiAgICAgICAgICAgICAgICBbbnAuY29uY2F0ZW5hdGUoYXVkaW9fcGllY2VzW3NwZWFrZXJdKSBmb3Igc3BlYWtlciBpbiBzcGVha2Vyc10KICAgICAgICAgICAgKS5hc3R5cGUoZHR5cGU9bnAuZmxvYXQzMikKICAgICAgICAgICAgIyBSZXNhbXBsZToKICAgICAgICAgICAgYXVkaW8gPSB0b3JjaC5mcm9tX251bXB5KGF1ZGlvKQogICAgICAgICAgICBhdWRpbyA9IHJlc2FtcGxlcihhdWRpbykKICAgICAgICAgICAgIyBTYXZlIHRvIGF1ZGlvIGZpbGU6CiAgICAgICAgICAgIGF1ZGlvX2ZpbGUgPSBvdXRwdXRfZGlyZWN0b3J5IC8gZiJ7dGV4dF9maWxlLnN0ZW19LntmaWxlX2Zvcm1hdH0iCgogICAgICAgICAgICB0b3JjaGF1ZGlvLnNhdmUoCiAgICAgICAgICAgICAgICB1cmk9c3RyKGF1ZGlvX2ZpbGUpLAogICAgICAgICAgICAgICAgc3JjPWF1ZGlvLAogICAgICAgICAgICAgICAgc2FtcGxlX3JhdGU9c2FtcGxlX3JhdGUsCiAgICAgICAgICAgICAgICBmb3JtYXQ9ZmlsZV9mb3JtYXQsCiAgICAgICAgICAgICAgICBiaXRzX3Blcl9zYW1wbGU9Yml0c19wZXJfc2FtcGxlLAogICAgICAgICAgICApCgogICAgICAgICAgICAjIENvbGxlY3QgdG8gdGhlIHN1Y2Nlc3NlczoKICAgICAgICAgICAgc3VjY2Vzc2VzLmFwcGVuZChbdGV4dF9maWxlLm5hbWUsIGF1ZGlvX2ZpbGUubmFtZV0pCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBleGNlcHRpb246CiAgICAgICAgICAgICMgTm90ZSB0aGUgZXhjZXB0aW9uIGFzIGVycm9yIGluIHRoZSBkaWN0aW9uYXJ5OgogICAgICAgICAgICBpZiB2ZXJib3NlOgogICAgICAgICAgICAgICAgX0xPR0dFUi53YXJuaW5nKGYiRXJyb3IgaW4gZmlsZTogJ3t0ZXh0X2ZpbGUubmFtZX0nIikKICAgICAgICAgICAgcHJpbnQoZXhjZXB0aW9uKQogICAgICAgICAgICBlcnJvcnNbdGV4dF9maWxlLm5hbWVdID0gc3RyKGV4Y2VwdGlvbikKCiAgICAjIENvbnN0cnVjdCB0aGUgdHJhbnNsYXRpb25zIGRhdGFmcmFtZToKICAgIHN1Y2Nlc3NlcyA9IHBkLkRhdGFGcmFtZSgKICAgICAgICBzdWNjZXNzZXMsCiAgICAgICAgY29sdW1ucz1bInRleHRfZmlsZSIsICJhdWRpb19maWxlIl0sCiAgICApCgogICAgIyBQcmludCB0aGUgaGVhZCBvZiB0aGUgcHJvZHVjZWQgZGF0YWZyYW1lIGFuZCByZXR1cm46CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygKICAgICAgICAgICAgZiJEb25lICh7c3VjY2Vzc2VzLnNoYXBlWzBdfS97bGVuKHRleHRfZmlsZXMpfSlcbiIKICAgICAgICAgICAgZiJUcmFuc2xhdGlvbnMgc3VtbWFyeTpcbiIKICAgICAgICAgICAgZiJ7c3VjY2Vzc2VzLmhlYWQoKX0iCiAgICAgICAgKQogICAgcmV0dXJuIHN0cihvdXRwdXRfZGlyZWN0b3J5KSwgc3VjY2Vzc2VzLCBlcnJvcnMKCgpjbGFzcyBTcGVlY2hFbmdpbmUoQUJDKToKICAgIEBhYnN0cmFjdG1ldGhvZAogICAgZGVmIF9nZW5lcmF0ZV9hdWRpbyhzZWxmLCB0ZXh0OiBzdHIsIHZvaWNlOiBzdHIpIC0+IG5wLm5kYXJyYXk6CiAgICAgICAgcGFzcwoKCmNsYXNzIEJhcmtFbmdpbmUoU3BlZWNoRW5naW5lKToKICAgIGRlZiBfX2luaXRfXygKICAgICAgICBzZWxmLAogICAgICAgIHVzZV9ncHU6IGJvb2wgPSBUcnVlLAogICAgICAgIHVzZV9zbWFsbF9tb2RlbHM6IGJvb2wgPSBGYWxzZSwKICAgICAgICBvZmZsb2FkX2NwdTogYm9vbCA9IEZhbHNlLAogICAgKToKICAgICAgICB0cnk6CiAgICAgICAgICAgIHNlbGYuYmFyayA9IGltcG9ydGxpYi5pbXBvcnRfbW9kdWxlKCJiYXJrIikKICAgICAgICBleGNlcHQgSW1wb3J0RXJyb3I6CiAgICAgICAgICAgIHJhaXNlIEltcG9ydEVycm9yKAogICAgICAgICAgICAgICAgIlRoZSAnYmFyaycgbGlicmFyeSBpcyByZXF1aXJlZCBmb3IgdGhlIEJhcmtFbmdpbmUuIFBsZWFzZSBpbnN0YWxsIGl0IHVzaW5nICdwaXAgaW5zdGFsbCBiYXJrLWFpJy4iCiAgICAgICAgICAgICkKCiAgICAgICAgc2VsZi5iYXJrLnByZWxvYWRfbW9kZWxzKAogICAgICAgICAgICB0ZXh0X3VzZV9ncHU9dXNlX2dwdSwKICAgICAgICAgICAgdGV4dF91c2Vfc21hbGw9dXNlX3NtYWxsX21vZGVscywKICAgICAgICAgICAgY29hcnNlX3VzZV9ncHU9dXNlX2dwdSwKICAgICAgICAgICAgY29hcnNlX3VzZV9zbWFsbD11c2Vfc21hbGxfbW9kZWxzLAogICAgICAgICAgICBmaW5lX3VzZV9ncHU9dXNlX2dwdSwKICAgICAgICAgICAgZmluZV91c2Vfc21hbGw9dXNlX3NtYWxsX21vZGVscywKICAgICAgICAgICAgY29kZWNfdXNlX2dwdT11c2VfZ3B1LAogICAgICAgICAgICBmb3JjZV9yZWxvYWQ9b2ZmbG9hZF9jcHUsCiAgICAgICAgKQoKICAgIGRlZiBfZ2VuZXJhdGVfYXVkaW8oc2VsZiwgdGV4dDogc3RyLCB2b2ljZTogc3RyKSAtPiBucC5uZGFycmF5OgogICAgICAgICMgR2VuZXJhdGUgd29yZHMgYXVkaW86CiAgICAgICAgYXVkaW8gPSBzZWxmLmJhcmsuZ2VuZXJhdGVfYXVkaW8oCiAgICAgICAgICAgIHRleHQsCiAgICAgICAgICAgIGhpc3RvcnlfcHJvbXB0PXZvaWNlLAogICAgICAgICAgICBzaWxlbnQ9VHJ1ZSwKICAgICAgICApCiAgICAgICAgcmV0dXJuIGF1ZGlvCgoKY2xhc3MgT3BlbkFJRW5naW5lKFNwZWVjaEVuZ2luZSk6CiAgICBkZWYgX19pbml0X18oCiAgICAgICAgc2VsZiwgbW9kZWw6IHN0ciA9ICJ0dHMtMSIsIGZpbGVfZm9ybWF0OiBzdHIgPSAid2F2Iiwgc3BlZWQ6IGZsb2F0ID0gMS4wCiAgICApOgogICAgICAgIHRyeToKICAgICAgICAgICAgc2VsZi5vcGVuYWkgPSBpbXBvcnRsaWIuaW1wb3J0X21vZHVsZSgib3BlbmFpIikKICAgICAgICAgICAgc2VsZi5weWR1YiA9IGltcG9ydGxpYi5pbXBvcnRfbW9kdWxlKCJweWR1YiIpCiAgICAgICAgZXhjZXB0IEltcG9ydEVycm9yOgogICAgICAgICAgICByYWlzZSBJbXBvcnRFcnJvcigKICAgICAgICAgICAgICAgICJUaGUgJ29wZW5haScgYW5kICdweWR1YicgbGlicmFyaWVzIGFyZSByZXF1aXJlZCBmb3IgdGhlIE9wZW5BSUVuZ2luZS4gUGxlYXNlIGluc3RhbGwgdGhlbSB1c2luZyAncGlwIGluc3RhbGwgb3BlbmFpIHB5ZHViJy4iCiAgICAgICAgICAgICkKCiAgICAgICAgYXBpX2tleSA9IG9zLmdldGVudihPUEVOQUlfQVBJX0tFWSkKICAgICAgICBiYXNlX3VybCA9IG9zLmdldGVudihPUEVOQUlfQkFTRV9VUkwpCiAgICAgICAgIyBDaGVjayBpZiB0aGUga2V5IGlzIGFscmVhZHkgaW4gdGhlIGVudmlyb25tZW50IHZhcmlhYmxlczoKICAgICAgICBpZiBub3QgYXBpX2tleSBvciBub3QgYmFzZV91cmw6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIGltcG9ydCBtbHJ1bgoKICAgICAgICAgICAgICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJjb250ZXh0IikKICAgICAgICAgICAgICAgICMgQ2hlY2sgaWYgdGhlIGtleSBpcyBpbiB0aGUgc2VjcmV0czoKICAgICAgICAgICAgICAgIGFwaV9rZXkgPSBjb250ZXh0LmdldF9zZWNyZXQoT1BFTkFJX0FQSV9LRVkpCiAgICAgICAgICAgICAgICBiYXNlX3VybCA9IGNvbnRleHQuZ2V0X3NlY3JldChPUEVOQUlfQkFTRV9VUkwpCiAgICAgICAgICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgICAgICAgICAgICAgcmFpc2UgT1NFcnJvcigKICAgICAgICAgICAgICAgICAgICBmIk9uZSBvciBtb3JlIG9mIHRoZSBPcGVuQUkgcmVxdWlyZWQgZW52aXJvbm1lbnQgdmFyaWFibGVzICgne09QRU5BSV9BUElfS0VZfScsICd7T1BFTkFJX0JBU0VfVVJMfScpIGFyZSBtaXNzaW5nLiIKICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSBzZXQgdGhlbSBhcyBlbnZpcm9ubWVudCB2YXJpYWJsZXMgb3IgaW5zdGFsbCBtbHJ1biAoYHBpcCBpbnN0YWxsIG1scnVuYCkiCiAgICAgICAgICAgICAgICAgICAgZiJhbmQgc2V0IHRoZW0gYXMgcHJvamVjdCBzZWNyZXRzIHVzaW5nIGBwcm9qZWN0LnNldF9zZWNyZXRzYC4iCiAgICAgICAgICAgICAgICApCgogICAgICAgIHNlbGYuY2xpZW50ID0gc2VsZi5vcGVuYWkuT3BlbkFJKGFwaV9rZXk9YXBpX2tleSwgYmFzZV91cmw9YmFzZV91cmwpCiAgICAgICAgc2VsZi5tb2RlbCA9IG1vZGVsCiAgICAgICAgc2VsZi5maWxlX2Zvcm1hdCA9IGZpbGVfZm9ybWF0CiAgICAgICAgc2VsZi5zcGVlZCA9IHNwZWVkCgogICAgZGVmIF9nZW5lcmF0ZV9hdWRpbyhzZWxmLCB0ZXh0OiBzdHIsIHZvaWNlOiBzdHIpIC0+IG5wLm5kYXJyYXk6CiAgICAgICAgIyBHZW5lcmF0ZSB3b3JkcyBhdWRpbzoKICAgICAgICBhdWRpbyA9IHNlbGYuY2xpZW50LmF1ZGlvLnNwZWVjaC5jcmVhdGUoCiAgICAgICAgICAgIG1vZGVsPXNlbGYubW9kZWwsCiAgICAgICAgICAgIGlucHV0PXRleHQsCiAgICAgICAgICAgIHZvaWNlPXZvaWNlLAogICAgICAgICAgICByZXNwb25zZV9mb3JtYXQ9c2VsZi5maWxlX2Zvcm1hdCwKICAgICAgICAgICAgc3BlZWQ9c2VsZi5zcGVlZCwKICAgICAgICApCiAgICAgICAgYXVkaW8gPSBhdWRpby5jb250ZW50CiAgICAgICAgYXVkaW8gPSBzZWxmLl9ieXRlc190b19ucF9hcnJheShhdWRpbz1hdWRpbykKICAgICAgICByZXR1cm4gYXVkaW8KCiAgICBkZWYgX2J5dGVzX3RvX25wX2FycmF5KHNlbGYsIGF1ZGlvOiBieXRlcyk6CiAgICAgICAgaWYgc2VsZi5maWxlX2Zvcm1hdCA9PSAibXAzIjoKICAgICAgICAgICAgYXVkaW9fc2VnbWVudCA9IHNlbGYucHlkdWIuQXVkaW9TZWdtZW50LmZyb21fbXAzKGlvLkJ5dGVzSU8oYXVkaW8pKQoKICAgICAgICAgICAgIyBDb252ZXJ0IHRvIHJhdyBQQ00gYXVkaW8gZGF0YQogICAgICAgICAgICBzYW1wbGVzID0gYXVkaW9fc2VnbWVudC5nZXRfYXJyYXlfb2Zfc2FtcGxlcygpCgogICAgICAgICAgICAjIENvbnZlcnQgdG8gbnVtcHkgYXJyYXkKICAgICAgICAgICAgYXVkaW9fYXJyYXkgPSBucC5hcnJheShzYW1wbGVzKQoKICAgICAgICAgICAgIyBOb3JtYWxpemUgdG8gZmxvYXQgYmV0d2VlbiAtMSBhbmQgMQogICAgICAgICAgICByZXR1cm4gYXVkaW9fYXJyYXkuYXN0eXBlKG5wLmZsb2F0MzIpIC8gbnAuaWluZm8oc2FtcGxlcy50eXBlY29kZSkubWF4CiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmV0dXJuIG5wLmZyb21idWZmZXIoYXVkaW8sIGR0eXBlPW5wLmludDE2KSAvIDMyNzY4LjAKCgpkZWYgX2dldF9lbmdpbmUoZW5naW5lOiBzdHIsIGZpbGVfZm9ybWF0OiBzdHIsICoqa3dhcmdzKSAtPiBTcGVlY2hFbmdpbmU6CiAgICAjIGVsaW1pbmF0ZSB0aGUgTm9uZSB2YWx1ZXM6CiAgICBrd2FyZ3MgPSB7a2V5OiB2YWx1ZSBmb3Iga2V5LCB2YWx1ZSBpbiBrd2FyZ3MuaXRlbXMoKSBpZiB2YWx1ZSBpcyBub3QgTm9uZX0KCiAgICBpZiBlbmdpbmUgPT0gImJhcmsiOgogICAgICAgIHJldHVybiBCYXJrRW5naW5lKCoqa3dhcmdzKQogICAgZWxpZiBlbmdpbmUgPT0gIm9wZW5haSI6CiAgICAgICAgcmV0dXJuIE9wZW5BSUVuZ2luZShmaWxlX2Zvcm1hdD1maWxlX2Zvcm1hdCwgKiprd2FyZ3MpCiAgICBlbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgIGYiVW5yZWNvZ25pemVkIGVuZ2luZS4gVGhlIHBhcmFtZXRlciBgZW5naW5lYCBtdXN0IGJlIGVpdGhlciAnYmFyaycgb3IgJ29wZW5haScuIEdpdmVuOiB7ZW5naW5lfSIKICAgICAgICApCgoKZGVmIF9nZXRfdGV4dF9maWxlcygKICAgIGRhdGFfcGF0aDogcGF0aGxpYi5QYXRoLAopIC0+IGxpc3RbcGF0aGxpYi5QYXRoXToKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICB0ZXh0X2ZpbGVzID0gbGlzdChkYXRhX3BhdGguZ2xvYigiKi4qIikpCiAgICBlbGlmIGRhdGFfcGF0aC5pc19maWxlKCk6CiAgICAgICAgdGV4dF9maWxlcyA9IFtkYXRhX3BhdGhdCiAgICBlbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgIGYiVW5yZWNvZ25pemVkIGRhdGEgcGF0aC4gVGhlIHBhcmFtZXRlciBgZGF0YV9wYXRoYCBtdXN0IGJlIGVpdGhlciBhIGRpcmVjdG9yeSBwYXRoIG9yIGEgZmlsZSBwYXRoLiAiCiAgICAgICAgICAgIGYiR2l2ZW46IHtzdHIoZGF0YV9wYXRoKX0gIgogICAgICAgICkKCiAgICByZXR1cm4gdGV4dF9maWxlcwoKCmRlZiBfc3BsaXRfbGluZShsaW5lOiBzdHIsIG1heF9sZW5ndGg6IGludCA9IDI1MCkgLT4gbGlzdFtzdHJdOgogICAgaWYgbGVuKGxpbmUpIDwgbWF4X2xlbmd0aDoKICAgICAgICByZXR1cm4gW2xpbmVdCgogICAgc2VudGVuY2VzID0gWwogICAgICAgIGYie3NlbnRlbmNlLnN0cmlwKCl9LiIgZm9yIHNlbnRlbmNlIGluIGxpbmUuc3BsaXQoIi4iKSBpZiBzZW50ZW5jZS5zdHJpcCgpCiAgICBdCgogICAgc3BsaXRzID0gW10KICAgIGN1cnJlbnRfbGVuZ3RoID0gbGVuKHNlbnRlbmNlc1swXSkKICAgIHNwbGl0ID0gc2VudGVuY2VzWzBdCiAgICBmb3Igc2VudGVuY2UgaW4gc2VudGVuY2VzWzE6XToKICAgICAgICBpZiBjdXJyZW50X2xlbmd0aCArIGxlbihzZW50ZW5jZSkgPiBtYXhfbGVuZ3RoOgogICAgICAgICAgICBzcGxpdHMuYXBwZW5kKHNwbGl0KQogICAgICAgICAgICBzcGxpdCA9IHNlbnRlbmNlCiAgICAgICAgICAgIGN1cnJlbnRfbGVuZ3RoID0gbGVuKHNlbnRlbmNlKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGN1cnJlbnRfbGVuZ3RoICs9IGxlbihzZW50ZW5jZSkKICAgICAgICAgICAgc3BsaXQgKz0gIiAiICsgc2VudGVuY2UKICAgIGlmIHNwbGl0OgogICAgICAgIHNwbGl0cy5hcHBlbmQoc3BsaXQpCgogICAgcmV0dXJuIHNwbGl0cwoKCmRlZiBfZ2V0X2xvZ2dlcigpOgogICAgZ2xvYmFsIF9MT0dHRVIKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KCiAgICAgICAgIyBDaGVjayBpZiBNTFJ1biBpcyBhdmFpbGFibGU6CiAgICAgICAgY29udGV4dCA9IG1scnVuLmdldF9vcl9jcmVhdGVfY3R4KG5hbWU9Im1scnVuIikKICAgICAgICByZXR1cm4gY29udGV4dC5sb2dnZXIKICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgICAgIHJldHVybiBfTE9HR0VSCg== + requirements: + - torchaudio + - pydub + code_origin: '' + base_image: mlrun/mlrun + filename: text_to_audio_generator.py entry_points: generate_multi_speakers_audio: - lineno: 38 + outputs: + - doc: 'A tuple of: - The output directory path. - The generated audio files + dataframe. - The errors'' dictionary.' + type: tuple[str, pd.DataFrame, dict] parameters: - name: data_path type: str doc: Path to the text file or directory containing the text files to generate audio from. - name: speakers - type: Union[List[str], Dict[str, int]] doc: List / Dict of speakers to generate audio for. If a list is given, the speakers will be assigned to channels in the order given. If dictionary, the keys will be the speakers and the values will be the channels. - name: available_voices - type: List[str] + type: list[str] doc: 'List of available voices to use for the generation. See here for the available voices for bark engine: https://suno-ai.notion.site/8b8e8749ed514b0cbf3f699013548683?v=bc67cff786b04b50b3ceb756fd05f68c See here for the available voices for openai engine: https://beta.openai.com/docs/api-reference/speech' @@ -29,26 +48,21 @@ spec: doc: Path to the directory to save the generated audio files to. default: null - name: use_gpu - type: Optional[bool] doc: Whether to use the GPU for the generation. Supported only in "bark" engine. default: null - name: use_small_models - type: Optional[bool] doc: Whether to use the small models for the generation. Supported only in "bark" engine. default: null - name: offload_cpu - type: Optional[bool] doc: To reduce the memory footprint, the models can be offloaded to the CPU after loading. Supported only in "bark" engine. default: null - name: model - type: Optional[str] doc: Which model to use for the generation. Supported only in "openai" engine. Default is "tts-1". default: null - name: speed - type: Optional[float] doc: The speed of the generated audio. Select a value from `0.25` to `4.0`. `1.0` is the default. default: null @@ -65,34 +79,14 @@ spec: doc: Whether to print the progress of the generation. default: true - name: bits_per_sample - type: Optional[int] doc: Changes the bit depth for the supported formats. Supported only in "wav" or "flac" formats. default: null name: generate_multi_speakers_audio + doc: Generate audio files from text files. has_kwargs: false has_varargs: false - outputs: - - doc: 'A tuple of: - The output directory path. - The generated audio files - dataframe. - The errors'' dictionary.' - type: Tuple[str, pd.DataFrame, dict] - doc: Generate audio files from text files. + lineno: 37 command: '' - image: '' description: Generate audio file from text using different speakers - build: - requirements: - - torchaudio - - pydub - base_image: mlrun/mlrun - code_origin: '' - origin_filename: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgppbXBvcnQgaW1wb3J0bGliCmltcG9ydCBpbwppbXBvcnQgbG9nZ2luZwppbXBvcnQgb3MKaW1wb3J0IHBhdGhsaWIKaW1wb3J0IHJhbmRvbQppbXBvcnQgdGVtcGZpbGUKZnJvbSBhYmMgaW1wb3J0IEFCQywgYWJzdHJhY3RtZXRob2QKZnJvbSB0eXBpbmcgaW1wb3J0IERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBudW1weSBhcyBucAppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCB0b3JjaAppbXBvcnQgdG9yY2hhdWRpbwppbXBvcnQgdHFkbQoKIyBHZXQgdGhlIGdsb2JhbCBsb2dnZXI6Cl9MT0dHRVIgPSBsb2dnaW5nLmdldExvZ2dlcigpCgpPUEVOQUlfQVBJX0tFWSA9ICJPUEVOQUlfQVBJX0tFWSIKT1BFTkFJX0JBU0VfVVJMID0gIk9QRU5BSV9BUElfQkFTRSIKU0FNUExFX1JBVEUgPSAyNDAwMAoKCmRlZiBnZW5lcmF0ZV9tdWx0aV9zcGVha2Vyc19hdWRpbygKICAgIGRhdGFfcGF0aDogc3RyLAogICAgc3BlYWtlcnM6IFVuaW9uW0xpc3Rbc3RyXSwgRGljdFtzdHIsIGludF1dLAogICAgYXZhaWxhYmxlX3ZvaWNlczogTGlzdFtzdHJdLAogICAgZW5naW5lOiBzdHIgPSAib3BlbmFpIiwKICAgIG91dHB1dF9kaXJlY3Rvcnk6IHN0ciA9IE5vbmUsCiAgICB1c2VfZ3B1OiBPcHRpb25hbFtib29sXSA9IE5vbmUsCiAgICB1c2Vfc21hbGxfbW9kZWxzOiBPcHRpb25hbFtib29sXSA9IE5vbmUsCiAgICBvZmZsb2FkX2NwdTogT3B0aW9uYWxbYm9vbF0gPSBOb25lLAogICAgbW9kZWw6IE9wdGlvbmFsW3N0cl0gPSBOb25lLAogICAgc3BlZWQ6IE9wdGlvbmFsW2Zsb2F0XSA9IE5vbmUsCiAgICBzYW1wbGVfcmF0ZTogaW50ID0gMTYwMDAsCiAgICBmaWxlX2Zvcm1hdDogc3RyID0gIndhdiIsCiAgICB2ZXJib3NlOiBib29sID0gVHJ1ZSwKICAgIGJpdHNfcGVyX3NhbXBsZTogT3B0aW9uYWxbaW50XSA9IE5vbmUsCikgLT4gVHVwbGVbc3RyLCBwZC5EYXRhRnJhbWUsIGRpY3RdOgogICAgIiIiCiAgICBHZW5lcmF0ZSBhdWRpbyBmaWxlcyBmcm9tIHRleHQgZmlsZXMuCgogICAgOnBhcmFtIGRhdGFfcGF0aDogICAgICAgICAgIFBhdGggdG8gdGhlIHRleHQgZmlsZSBvciBkaXJlY3RvcnkgY29udGFpbmluZyB0aGUgdGV4dCBmaWxlcyB0byBnZW5lcmF0ZSBhdWRpbyBmcm9tLgogICAgOnBhcmFtIHNwZWFrZXJzOiAgICAgICAgICAgIExpc3QgLyBEaWN0IG9mIHNwZWFrZXJzIHRvIGdlbmVyYXRlIGF1ZGlvIGZvci4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBhIGxpc3QgaXMgZ2l2ZW4sIHRoZSBzcGVha2VycyB3aWxsIGJlIGFzc2lnbmVkIHRvIGNoYW5uZWxzIGluIHRoZSBvcmRlciBnaXZlbi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBkaWN0aW9uYXJ5LCB0aGUga2V5cyB3aWxsIGJlIHRoZSBzcGVha2VycyBhbmQgdGhlIHZhbHVlcyB3aWxsIGJlIHRoZSBjaGFubmVscy4KICAgIDpwYXJhbSBhdmFpbGFibGVfdm9pY2VzOiAgICBMaXN0IG9mIGF2YWlsYWJsZSB2b2ljZXMgdG8gdXNlIGZvciB0aGUgZ2VuZXJhdGlvbi4KICAgICAgICAgICAgICAgICAgICAgICAgU2VlIGhlcmUgZm9yIHRoZSBhdmFpbGFibGUgdm9pY2VzIGZvciBiYXJrIGVuZ2luZToKICAgICAgICAgICAgICAgICAgICAgICAgaHR0cHM6Ly9zdW5vLWFpLm5vdGlvbi5zaXRlLzhiOGU4NzQ5ZWQ1MTRiMGNiZjNmNjk5MDEzNTQ4NjgzP3Y9YmM2N2NmZjc4NmIwNGI1MGIzY2ViNzU2ZmQwNWY2OGMKICAgICAgICAgICAgICAgICAgICAgICAgU2VlIGhlcmUgZm9yIHRoZSBhdmFpbGFibGUgdm9pY2VzIGZvciBvcGVuYWkgZW5naW5lOgogICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL2JldGEub3BlbmFpLmNvbS9kb2NzL2FwaS1yZWZlcmVuY2Uvc3BlZWNoCiAgICA6cGFyYW0gZW5naW5lOiAgICAgICAgICAgICAgVGhlIGVuZ2luZSB0byB1c2UgZm9yIHRoZSBnZW5lcmF0aW9uLiBTZWxlY3QgZWl0aGVyICJiYXJrIiBvciAib3BlbmFpIi4gRGVmYXVsdCBpcyAib3BlbmFpIi4KICAgIDpwYXJhbSBvdXRwdXRfZGlyZWN0b3J5OiAgICBQYXRoIHRvIHRoZSBkaXJlY3RvcnkgdG8gc2F2ZSB0aGUgZ2VuZXJhdGVkIGF1ZGlvIGZpbGVzIHRvLgogICAgOnBhcmFtIHVzZV9ncHU6ICAgICAgICAgICAgIFdoZXRoZXIgdG8gdXNlIHRoZSBHUFUgZm9yIHRoZSBnZW5lcmF0aW9uLiBTdXBwb3J0ZWQgb25seSBpbiAiYmFyayIgZW5naW5lLgogICAgOnBhcmFtIHVzZV9zbWFsbF9tb2RlbHM6ICAgIFdoZXRoZXIgdG8gdXNlIHRoZSBzbWFsbCBtb2RlbHMgZm9yIHRoZSBnZW5lcmF0aW9uLiBTdXBwb3J0ZWQgb25seSBpbiAiYmFyayIgZW5naW5lLgogICAgOnBhcmFtIG9mZmxvYWRfY3B1OiAgICAgICAgIFRvIHJlZHVjZSB0aGUgbWVtb3J5IGZvb3RwcmludCwgdGhlIG1vZGVscyBjYW4gYmUgb2ZmbG9hZGVkIHRvIHRoZSBDUFUgYWZ0ZXIgbG9hZGluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTdXBwb3J0ZWQgb25seSBpbiAiYmFyayIgZW5naW5lLgogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgIFdoaWNoIG1vZGVsIHRvIHVzZSBmb3IgdGhlIGdlbmVyYXRpb24uIFN1cHBvcnRlZCBvbmx5IGluICJvcGVuYWkiIGVuZ2luZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0IGlzICJ0dHMtMSIuCiAgICA6cGFyYW0gc3BlZWQ6ICAgICAgICAgICAgICAgVGhlIHNwZWVkIG9mIHRoZSBnZW5lcmF0ZWQgYXVkaW8uIFNlbGVjdCBhIHZhbHVlIGZyb20gYDAuMjVgIHRvIGA0LjBgLiBgMS4wYCBpcyB0aGUgZGVmYXVsdC4KICAgIDpwYXJhbSBzYW1wbGVfcmF0ZTogICAgICAgICBUaGUgc2FtcGxpbmcgcmF0ZSBvZiB0aGUgZ2VuZXJhdGVkIGF1ZGlvLgogICAgOnBhcmFtIGZpbGVfZm9ybWF0OiAgICAgICAgIFRoZSBmb3JtYXQgb2YgdGhlIGdlbmVyYXRlZCBhdWRpbyBmaWxlcy4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgICBXaGV0aGVyIHRvIHByaW50IHRoZSBwcm9ncmVzcyBvZiB0aGUgZ2VuZXJhdGlvbi4KICAgIDpwYXJhbSBiaXRzX3Blcl9zYW1wbGU6ICAgICBDaGFuZ2VzIHRoZSBiaXQgZGVwdGggZm9yIHRoZSBzdXBwb3J0ZWQgZm9ybWF0cy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTdXBwb3J0ZWQgb25seSBpbiAid2F2IiBvciAiZmxhYyIgZm9ybWF0cy4KCiAgICA6cmV0dXJuczogICAgICAgICAgICAgICAgICAgQSB0dXBsZSBvZjoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAtIFRoZSBvdXRwdXQgZGlyZWN0b3J5IHBhdGguCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBUaGUgZ2VuZXJhdGVkIGF1ZGlvIGZpbGVzIGRhdGFmcmFtZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAtIFRoZSBlcnJvcnMnIGRpY3Rpb25hcnkuCiAgICAiIiIKCiAgICBnbG9iYWwgX0xPR0dFUgogICAgX0xPR0dFUiA9IF9nZXRfbG9nZ2VyKCkKICAgICMgR2V0IHRoZSBpbnB1dCB0ZXh0IGZpbGVzIHRvIHR1cm4gdG8gYXVkaW86CiAgICBkYXRhX3BhdGggPSBwYXRobGliLlBhdGgoZGF0YV9wYXRoKS5hYnNvbHV0ZSgpCiAgICB0ZXh0X2ZpbGVzID0gX2dldF90ZXh0X2ZpbGVzKGRhdGFfcGF0aD1kYXRhX3BhdGgpCgoKICAgICMgUHJlcGFyZSB0aGUgc3BlZWNoIGVuZ2luZToKICAgIGVuZ2luZSA9IF9nZXRfZW5naW5lKAogICAgICAgIGVuZ2luZT1lbmdpbmUsCiAgICAgICAgdXNlX2dwdT11c2VfZ3B1LAogICAgICAgIHVzZV9zbWFsbF9tb2RlbHM9dXNlX3NtYWxsX21vZGVscywKICAgICAgICBvZmZsb2FkX2NwdT1vZmZsb2FkX2NwdSwKICAgICAgICBtb2RlbD1tb2RlbCwKICAgICAgICBmaWxlX2Zvcm1hdD1maWxlX2Zvcm1hdCwKICAgICAgICBzcGVlZD1zcGVlZAogICAgKQoKICAgICMgQ2hlY2sgZm9yIHBlciBjaGFubmVsIGdlbmVyYXRpb246CiAgICBpZiBpc2luc3RhbmNlKHNwZWFrZXJzLCBkaWN0KToKICAgICAgICBzcGVha2VyX3Blcl9jaGFubmVsID0gVHJ1ZQogICAgICAgICMgU29ydCB0aGUgZ2l2ZW4gc3BlYWtlcnMgYnkgY2hhbm5lbHM6CiAgICAgICAgc3BlYWtlcnMgPSB7CiAgICAgICAgICAgIHNwZWFrZXI6IGNoYW5uZWwKICAgICAgICAgICAgZm9yIHNwZWFrZXIsIGNoYW5uZWwgaW4gc29ydGVkKHNwZWFrZXJzLml0ZW1zKCksIGtleT1sYW1iZGEgaXRlbTogaXRlbVsxXSkKICAgICAgICB9CiAgICBlbHNlOgogICAgICAgIHNwZWFrZXJfcGVyX2NoYW5uZWwgPSBGYWxzZQoKICAgICMgUHJlcGFyZSB0aGUgcmVzYW1wbGluZyBtb2R1bGU6CiAgICByZXNhbXBsZXIgPSB0b3JjaGF1ZGlvLnRyYW5zZm9ybXMuUmVzYW1wbGUoCiAgICAgICAgb3JpZ19mcmVxPVNBTVBMRV9SQVRFLCBuZXdfZnJlcT1zYW1wbGVfcmF0ZSwgZHR5cGU9dG9yY2guZmxvYXQzMgogICAgKQoKICAgICMgUHJlcGFyZSB0aGUgZ2FwIGJldHdlZW4gZWFjaCBzcGVha2VyOgogICAgZ2FwX2JldHdlZW5fc3BlYWtlcnMgPSBucC56ZXJvcyhpbnQoMC41ICogU0FNUExFX1JBVEUpKQoKICAgICMgUHJlcGFyZSB0aGUgc3VjY2Vzc2VzIGRhdGFmcmFtZSBhbmQgZXJyb3JzIGRpY3Rpb25hcnkgdG8gYmUgcmV0dXJuZWQ6CiAgICBzdWNjZXNzZXMgPSBbXQogICAgZXJyb3JzID0ge30KCiAgICAjIENyZWF0ZSB0aGUgb3V0cHV0IGRpcmVjdG9yeToKICAgIGlmIG91dHB1dF9kaXJlY3RvcnkgaXMgTm9uZToKICAgICAgICBvdXRwdXRfZGlyZWN0b3J5ID0gdGVtcGZpbGUubWtkdGVtcCgpCiAgICBvdXRwdXRfZGlyZWN0b3J5ID0gcGF0aGxpYi5QYXRoKG91dHB1dF9kaXJlY3RvcnkpCiAgICBpZiBub3Qgb3V0cHV0X2RpcmVjdG9yeS5leGlzdHMoKToKICAgICAgICBvdXRwdXRfZGlyZWN0b3J5Lm1rZGlyKGV4aXN0X29rPVRydWUsIHBhcmVudHM9VHJ1ZSkKCiAgICAjIFN0YXJ0IGdlbmVyYXRpbmcgYXVkaW86CiAgICAjIEdvIG92ZXIgdGhlIGF1ZGlvIGZpbGVzIGFuZCB0cmFuc2NyaWJlOgogICAgZm9yIHRleHRfZmlsZSBpbiB0cWRtLnRxZG0oCiAgICAgICAgdGV4dF9maWxlcywgZGVzYz0iR2VuZXJhdGluZyIsIHVuaXQ9ImZpbGUiLCBkaXNhYmxlPW5vdCB2ZXJib3NlCiAgICApOgoKICAgICAgICB0cnk6CiAgICAgICAgICAgICMgUmFuZG9taXplIHZvaWNlcyBmb3IgZWFjaCBzcGVha2VyOgogICAgICAgICAgICBjaG9zZW5fdm9pY2VzID0ge30KICAgICAgICAgICAgYXZhaWxhYmxlX3ZvaWNlc19jb3B5ID0gYXZhaWxhYmxlX3ZvaWNlcy5jb3B5KCkKICAgICAgICAgICAgZm9yIHNwZWFrZXIgaW4gc3BlYWtlcnM6CiAgICAgICAgICAgICAgICB2b2ljZSA9IHJhbmRvbS5jaG9pY2UoYXZhaWxhYmxlX3ZvaWNlc19jb3B5KQogICAgICAgICAgICAgICAgY2hvc2VuX3ZvaWNlc1tzcGVha2VyXSA9IHZvaWNlCiAgICAgICAgICAgICAgICBhdmFpbGFibGVfdm9pY2VzX2NvcHkucmVtb3ZlKHZvaWNlKQogICAgICAgICAgICAjIFJlYWQgdGV4dDoKICAgICAgICAgICAgd2l0aCBvcGVuKHRleHRfZmlsZSwgInIiKSBhcyBmcDoKICAgICAgICAgICAgICAgIHRleHQgPSBmcC5yZWFkKCkKICAgICAgICAgICAgIyBQcmVwYXJlIGEgaG9sZGVyIGZvciBhbGwgdGhlIGdlbmVyYXRlZCBwaWVjZXMgKGlmIHBlciBjaGFubmVsIGVhY2ggc3BlYWtlciB3aWxsIGhhdmUgaXRzIG93bik6CiAgICAgICAgICAgIGF1ZGlvX3BpZWNlcyA9ICgKICAgICAgICAgICAgICAgIHtzcGVha2VyOiBbXSBmb3Igc3BlYWtlciBpbiBzcGVha2Vyc30KICAgICAgICAgICAgICAgIGlmIHNwZWFrZXJfcGVyX2NoYW5uZWwKICAgICAgICAgICAgICAgIGVsc2UgeyJhbGwiOiBbXX0KICAgICAgICAgICAgKQoKICAgICAgICAgICAgIyBHZW5lcmF0ZSBhdWRpbyBwZXIgbGluZToKICAgICAgICAgICAgZm9yIGxpbmUgaW4gdGV4dC5zcGxpdGxpbmVzKCk6CiAgICAgICAgICAgICAgICAjIFZhbGlkYXRlIGxpbmUgaXMgaW4gY29ycmVjdCBzcGVha2VyIGZvcm1hdDoKCiAgICAgICAgICAgICAgICBpZiAiOiAiIG5vdCBpbiBsaW5lOgogICAgICAgICAgICAgICAgICAgIGlmIHZlcmJvc2U6CiAgICAgICAgICAgICAgICAgICAgICAgIF9MT0dHRVIud2FybmluZyhmIlNraXBwaW5nIGxpbmU6IHtsaW5lfSIpCiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgICMgU3BsaXQgbGluZSB0byBzcGVha2VyIGFuZCBoaXMgd29yZHM6CiAgICAgICAgICAgICAgICBjdXJyZW50X3NwZWFrZXIsIHNlbnRlbmNlcyA9IGxpbmUuc3BsaXQoIjogIiwgMSkKICAgICAgICAgICAgICAgICMgVmFsaWRhdGUgc3BlYWtlciBpcyBrbm93bjoKICAgICAgICAgICAgICAgIGlmIGN1cnJlbnRfc3BlYWtlciBub3QgaW4gc3BlYWtlcnM6CiAgICAgICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAgICAgZiJVbmtub3duIHNwZWFrZXI6IHtjdXJyZW50X3NwZWFrZXJ9LiBHaXZlbiBzcGVha2VycyBhcmU6IHtzcGVha2Vyc30iCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZm9yIHNlbnRlbmNlIGluIF9zcGxpdF9saW5lKGxpbmU9c2VudGVuY2VzKToKICAgICAgICAgICAgICAgICAgICAjIEdlbmVyYXRlIHdvcmRzIGF1ZGlvOgogICAgICAgICAgICAgICAgICAgIGF1ZGlvID0gZW5naW5lLl9nZW5lcmF0ZV9hdWRpbygKICAgICAgICAgICAgICAgICAgICAgICAgdGV4dD1zZW50ZW5jZSwKICAgICAgICAgICAgICAgICAgICAgICAgdm9pY2U9Y2hvc2VuX3ZvaWNlc1tjdXJyZW50X3NwZWFrZXJdLAogICAgICAgICAgICAgICAgICAgICkKCiAgICAgICAgICAgICAgICAgICAgaWYgc3BlYWtlcl9wZXJfY2hhbm5lbDoKICAgICAgICAgICAgICAgICAgICAgICAgc2lsZW5jZSA9IG5wLnplcm9zX2xpa2UoYXVkaW8pCiAgICAgICAgICAgICAgICAgICAgICAgIGZvciBzcGVha2VyIGluIGF1ZGlvX3BpZWNlcy5rZXlzKCk6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICBpZiBzcGVha2VyID09IGN1cnJlbnRfc3BlYWtlcjoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhdWRpb19waWVjZXNbc3BlYWtlcl0gKz0gW2F1ZGlvLCBnYXBfYmV0d2Vlbl9zcGVha2Vyc10KICAgICAgICAgICAgICAgICAgICAgICAgICAgIGVsc2U6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgYXVkaW9fcGllY2VzW3NwZWFrZXJdICs9IFtzaWxlbmNlLCBnYXBfYmV0d2Vlbl9zcGVha2Vyc10KICAgICAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgICAgICBhdWRpb19waWVjZXNbImFsbCJdICs9IFthdWRpbywgZ2FwX2JldHdlZW5fc3BlYWtlcnNdCiAgICAgICAgICAgICMgQ29uc3RydWN0IGEgc2luZ2xlIGF1ZGlvIGFycmF5IGZyb20gYWxsIHRoZSBwaWVjZXMgYW5kIGNoYW5uZWxzOgoKICAgICAgICAgICAgYXVkaW8gPSBucC52c3RhY2soCiAgICAgICAgICAgICAgICBbbnAuY29uY2F0ZW5hdGUoYXVkaW9fcGllY2VzW3NwZWFrZXJdKSBmb3Igc3BlYWtlciBpbiBzcGVha2Vyc10KICAgICAgICAgICAgKS5hc3R5cGUoZHR5cGU9bnAuZmxvYXQzMikKICAgICAgICAgICAgIyBSZXNhbXBsZToKICAgICAgICAgICAgYXVkaW8gPSB0b3JjaC5mcm9tX251bXB5KGF1ZGlvKQogICAgICAgICAgICBhdWRpbyA9IHJlc2FtcGxlcihhdWRpbykKICAgICAgICAgICAgIyBTYXZlIHRvIGF1ZGlvIGZpbGU6CiAgICAgICAgICAgIGF1ZGlvX2ZpbGUgPSBvdXRwdXRfZGlyZWN0b3J5IC8gZiJ7dGV4dF9maWxlLnN0ZW19LntmaWxlX2Zvcm1hdH0iCgogICAgICAgICAgICB0b3JjaGF1ZGlvLnNhdmUoCiAgICAgICAgICAgICAgICB1cmk9c3RyKGF1ZGlvX2ZpbGUpLAogICAgICAgICAgICAgICAgc3JjPWF1ZGlvLAogICAgICAgICAgICAgICAgc2FtcGxlX3JhdGU9c2FtcGxlX3JhdGUsCiAgICAgICAgICAgICAgICBmb3JtYXQ9ZmlsZV9mb3JtYXQsCiAgICAgICAgICAgICAgICBiaXRzX3Blcl9zYW1wbGU9Yml0c19wZXJfc2FtcGxlLAogICAgICAgICAgICApCgogICAgICAgICAgICAjIENvbGxlY3QgdG8gdGhlIHN1Y2Nlc3NlczoKICAgICAgICAgICAgc3VjY2Vzc2VzLmFwcGVuZChbdGV4dF9maWxlLm5hbWUsIGF1ZGlvX2ZpbGUubmFtZV0pCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBleGNlcHRpb246CiAgICAgICAgICAgICMgTm90ZSB0aGUgZXhjZXB0aW9uIGFzIGVycm9yIGluIHRoZSBkaWN0aW9uYXJ5OgogICAgICAgICAgICBpZiB2ZXJib3NlOgogICAgICAgICAgICAgICAgX0xPR0dFUi53YXJuaW5nKGYiRXJyb3IgaW4gZmlsZTogJ3t0ZXh0X2ZpbGUubmFtZX0nIikKICAgICAgICAgICAgcHJpbnQoZXhjZXB0aW9uKQogICAgICAgICAgICBlcnJvcnNbdGV4dF9maWxlLm5hbWVdID0gc3RyKGV4Y2VwdGlvbikKCiAgICAjIENvbnN0cnVjdCB0aGUgdHJhbnNsYXRpb25zIGRhdGFmcmFtZToKICAgIHN1Y2Nlc3NlcyA9IHBkLkRhdGFGcmFtZSgKICAgICAgICBzdWNjZXNzZXMsCiAgICAgICAgY29sdW1ucz1bInRleHRfZmlsZSIsICJhdWRpb19maWxlIl0sCiAgICApCgogICAgIyBQcmludCB0aGUgaGVhZCBvZiB0aGUgcHJvZHVjZWQgZGF0YWZyYW1lIGFuZCByZXR1cm46CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygKICAgICAgICAgICAgZiJEb25lICh7c3VjY2Vzc2VzLnNoYXBlWzBdfS97bGVuKHRleHRfZmlsZXMpfSlcbiIKICAgICAgICAgICAgZiJUcmFuc2xhdGlvbnMgc3VtbWFyeTpcbiIKICAgICAgICAgICAgZiJ7c3VjY2Vzc2VzLmhlYWQoKX0iCiAgICAgICAgKQogICAgcmV0dXJuIHN0cihvdXRwdXRfZGlyZWN0b3J5KSwgc3VjY2Vzc2VzLCBlcnJvcnMKCgpjbGFzcyBTcGVlY2hFbmdpbmUoQUJDKToKICAgIEBhYnN0cmFjdG1ldGhvZAogICAgZGVmIF9nZW5lcmF0ZV9hdWRpbyhzZWxmLCB0ZXh0OiBzdHIsIHZvaWNlOiBzdHIpIC0+IG5wLm5kYXJyYXk6CiAgICAgICAgcGFzcwoKCmNsYXNzIEJhcmtFbmdpbmUoU3BlZWNoRW5naW5lKToKICAgIGRlZiBfX2luaXRfXyhzZWxmLCB1c2VfZ3B1OiBib29sID0gVHJ1ZSwgdXNlX3NtYWxsX21vZGVsczogYm9vbCA9IEZhbHNlLCBvZmZsb2FkX2NwdTogYm9vbCA9IEZhbHNlKToKICAgICAgICB0cnk6CiAgICAgICAgICAgIHNlbGYuYmFyayA9IGltcG9ydGxpYi5pbXBvcnRfbW9kdWxlKCJiYXJrIikKICAgICAgICBleGNlcHQgSW1wb3J0RXJyb3I6CiAgICAgICAgICAgIHJhaXNlIEltcG9ydEVycm9yKAogICAgICAgICAgICAgICAgIlRoZSAnYmFyaycgbGlicmFyeSBpcyByZXF1aXJlZCBmb3IgdGhlIEJhcmtFbmdpbmUuIFBsZWFzZSBpbnN0YWxsIGl0IHVzaW5nICdwaXAgaW5zdGFsbCBiYXJrLWFpJy4iCiAgICAgICAgICAgICkKCiAgICAgICAgc2VsZi5iYXJrLnByZWxvYWRfbW9kZWxzKAogICAgICAgICAgICB0ZXh0X3VzZV9ncHU9dXNlX2dwdSwKICAgICAgICAgICAgdGV4dF91c2Vfc21hbGw9dXNlX3NtYWxsX21vZGVscywKICAgICAgICAgICAgY29hcnNlX3VzZV9ncHU9dXNlX2dwdSwKICAgICAgICAgICAgY29hcnNlX3VzZV9zbWFsbD11c2Vfc21hbGxfbW9kZWxzLAogICAgICAgICAgICBmaW5lX3VzZV9ncHU9dXNlX2dwdSwKICAgICAgICAgICAgZmluZV91c2Vfc21hbGw9dXNlX3NtYWxsX21vZGVscywKICAgICAgICAgICAgY29kZWNfdXNlX2dwdT11c2VfZ3B1LAogICAgICAgICAgICBmb3JjZV9yZWxvYWQ9b2ZmbG9hZF9jcHUsCiAgICAgICAgKQoKICAgIGRlZiBfZ2VuZXJhdGVfYXVkaW8oc2VsZiwgdGV4dDogc3RyLCB2b2ljZTogc3RyKSAtPiBucC5uZGFycmF5OgogICAgICAgICMgR2VuZXJhdGUgd29yZHMgYXVkaW86CiAgICAgICAgYXVkaW8gPSBzZWxmLmJhcmsuZ2VuZXJhdGVfYXVkaW8oCiAgICAgICAgICAgIHRleHQsCiAgICAgICAgICAgIGhpc3RvcnlfcHJvbXB0PXZvaWNlLAogICAgICAgICAgICBzaWxlbnQ9VHJ1ZSwKICAgICAgICApCiAgICAgICAgcmV0dXJuIGF1ZGlvCgoKY2xhc3MgT3BlbkFJRW5naW5lKFNwZWVjaEVuZ2luZSk6CiAgICBkZWYgX19pbml0X18oc2VsZiwgbW9kZWw6IHN0ciA9ICJ0dHMtMSIsIGZpbGVfZm9ybWF0OiBzdHIgPSAid2F2Iiwgc3BlZWQ6IGZsb2F0ID0gMS4wKToKICAgICAgICB0cnk6CiAgICAgICAgICAgIHNlbGYub3BlbmFpID0gaW1wb3J0bGliLmltcG9ydF9tb2R1bGUoIm9wZW5haSIpCiAgICAgICAgICAgIHNlbGYucHlkdWIgPSBpbXBvcnRsaWIuaW1wb3J0X21vZHVsZSgicHlkdWIiKQogICAgICAgIGV4Y2VwdCBJbXBvcnRFcnJvcjoKICAgICAgICAgICAgcmFpc2UgSW1wb3J0RXJyb3IoCiAgICAgICAgICAgICAgICAiVGhlICdvcGVuYWknIGFuZCAncHlkdWInIGxpYnJhcmllcyBhcmUgcmVxdWlyZWQgZm9yIHRoZSBPcGVuQUlFbmdpbmUuIFBsZWFzZSBpbnN0YWxsIHRoZW0gdXNpbmcgJ3BpcCBpbnN0YWxsIG9wZW5haSBweWR1YicuIgogICAgICAgICAgICApCgogICAgICAgIGFwaV9rZXkgPSBvcy5nZXRlbnYoT1BFTkFJX0FQSV9LRVkpCiAgICAgICAgYmFzZV91cmwgPSBvcy5nZXRlbnYoT1BFTkFJX0JBU0VfVVJMKQogICAgICAgICMgQ2hlY2sgaWYgdGhlIGtleSBpcyBhbHJlYWR5IGluIHRoZSBlbnZpcm9ubWVudCB2YXJpYWJsZXM6CiAgICAgICAgaWYgbm90IGFwaV9rZXkgb3Igbm90IGJhc2VfdXJsOgogICAgICAgICAgICB0cnk6CiAgICAgICAgICAgICAgICBpbXBvcnQgbWxydW4KCiAgICAgICAgICAgICAgICBjb250ZXh0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9jdHgobmFtZT0iY29udGV4dCIpCiAgICAgICAgICAgICAgICAjIENoZWNrIGlmIHRoZSBrZXkgaXMgaW4gdGhlIHNlY3JldHM6CiAgICAgICAgICAgICAgICBhcGlfa2V5ID0gY29udGV4dC5nZXRfc2VjcmV0KE9QRU5BSV9BUElfS0VZKQogICAgICAgICAgICAgICAgYmFzZV91cmwgPSBjb250ZXh0LmdldF9zZWNyZXQoT1BFTkFJX0JBU0VfVVJMKQogICAgICAgICAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvcjoKICAgICAgICAgICAgICAgIHJhaXNlIEVudmlyb25tZW50RXJyb3IoCiAgICAgICAgICAgICAgICAgICAgZiJPbmUgb3IgbW9yZSBvZiB0aGUgT3BlbkFJIHJlcXVpcmVkIGVudmlyb25tZW50IHZhcmlhYmxlcyAoJ3tPUEVOQUlfQVBJX0tFWX0nLCAne09QRU5BSV9CQVNFX1VSTH0nKSBhcmUgbWlzc2luZy4iCiAgICAgICAgICAgICAgICAgICAgZiJQbGVhc2Ugc2V0IHRoZW0gYXMgZW52aXJvbm1lbnQgdmFyaWFibGVzIG9yIGluc3RhbGwgbWxydW4gKGBwaXAgaW5zdGFsbCBtbHJ1bmApIgogICAgICAgICAgICAgICAgICAgIGYiYW5kIHNldCB0aGVtIGFzIHByb2plY3Qgc2VjcmV0cyB1c2luZyBgcHJvamVjdC5zZXRfc2VjcmV0c2AuIgogICAgICAgICAgICAgICAgKQoKICAgICAgICBzZWxmLmNsaWVudCA9IHNlbGYub3BlbmFpLk9wZW5BSShhcGlfa2V5PWFwaV9rZXksIGJhc2VfdXJsPWJhc2VfdXJsKQogICAgICAgIHNlbGYubW9kZWwgPSBtb2RlbAogICAgICAgIHNlbGYuZmlsZV9mb3JtYXQgPSBmaWxlX2Zvcm1hdAogICAgICAgIHNlbGYuc3BlZWQgPSBzcGVlZAoKICAgIGRlZiBfZ2VuZXJhdGVfYXVkaW8oc2VsZiwgdGV4dDogc3RyLCB2b2ljZTogc3RyKSAtPiBucC5uZGFycmF5OgogICAgICAgICMgR2VuZXJhdGUgd29yZHMgYXVkaW86CiAgICAgICAgYXVkaW8gPSBzZWxmLmNsaWVudC5hdWRpby5zcGVlY2guY3JlYXRlKAogICAgICAgICAgICBtb2RlbD1zZWxmLm1vZGVsLAogICAgICAgICAgICBpbnB1dD10ZXh0LAogICAgICAgICAgICB2b2ljZT12b2ljZSwKICAgICAgICAgICAgcmVzcG9uc2VfZm9ybWF0PXNlbGYuZmlsZV9mb3JtYXQsCiAgICAgICAgICAgIHNwZWVkPXNlbGYuc3BlZWQsCiAgICAgICAgKQogICAgICAgIGF1ZGlvID0gYXVkaW8uY29udGVudAogICAgICAgIGF1ZGlvID0gc2VsZi5fYnl0ZXNfdG9fbnBfYXJyYXkoYXVkaW89YXVkaW8pCiAgICAgICAgcmV0dXJuIGF1ZGlvCgogICAgZGVmIF9ieXRlc190b19ucF9hcnJheShzZWxmLCBhdWRpbzogYnl0ZXMpOgogICAgICAgIGlmIHNlbGYuZmlsZV9mb3JtYXQgPT0gIm1wMyI6CiAgICAgICAgICAgIGF1ZGlvX3NlZ21lbnQgPSBzZWxmLnB5ZHViLkF1ZGlvU2VnbWVudC5mcm9tX21wMyhpby5CeXRlc0lPKGF1ZGlvKSkKCiAgICAgICAgICAgICMgQ29udmVydCB0byByYXcgUENNIGF1ZGlvIGRhdGEKICAgICAgICAgICAgc2FtcGxlcyA9IGF1ZGlvX3NlZ21lbnQuZ2V0X2FycmF5X29mX3NhbXBsZXMoKQoKICAgICAgICAgICAgIyBDb252ZXJ0IHRvIG51bXB5IGFycmF5CiAgICAgICAgICAgIGF1ZGlvX2FycmF5ID0gbnAuYXJyYXkoc2FtcGxlcykKCiAgICAgICAgICAgICMgTm9ybWFsaXplIHRvIGZsb2F0IGJldHdlZW4gLTEgYW5kIDEKICAgICAgICAgICAgcmV0dXJuIGF1ZGlvX2FycmF5LmFzdHlwZShucC5mbG9hdDMyKSAvIG5wLmlpbmZvKHNhbXBsZXMudHlwZWNvZGUpLm1heAogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHJldHVybiBucC5mcm9tYnVmZmVyKGF1ZGlvLCBkdHlwZT1ucC5pbnQxNikgLyAzMjc2OC4wCgoKZGVmIF9nZXRfZW5naW5lKGVuZ2luZTogc3RyLCBmaWxlX2Zvcm1hdDogc3RyLCAqKmt3YXJncykgLT4gU3BlZWNoRW5naW5lOgogICAgIyBlbGltaW5hdGUgdGhlIE5vbmUgdmFsdWVzOgogICAga3dhcmdzID0ge2tleTogdmFsdWUgZm9yIGtleSwgdmFsdWUgaW4ga3dhcmdzLml0ZW1zKCkgaWYgdmFsdWUgaXMgbm90IE5vbmV9CgogICAgaWYgZW5naW5lID09ICJiYXJrIjoKICAgICAgICByZXR1cm4gQmFya0VuZ2luZSgqKmt3YXJncykKICAgIGVsaWYgZW5naW5lID09ICJvcGVuYWkiOgogICAgICAgIHJldHVybiBPcGVuQUlFbmdpbmUoZmlsZV9mb3JtYXQ9ZmlsZV9mb3JtYXQsICoqa3dhcmdzKQogICAgZWxzZToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICBmIlVucmVjb2duaXplZCBlbmdpbmUuIFRoZSBwYXJhbWV0ZXIgYGVuZ2luZWAgbXVzdCBiZSBlaXRoZXIgJ2JhcmsnIG9yICdvcGVuYWknLiBHaXZlbjoge2VuZ2luZX0iCiAgICAgICAgKQoKZGVmIF9nZXRfdGV4dF9maWxlcygKICAgIGRhdGFfcGF0aDogcGF0aGxpYi5QYXRoLAopIC0+IExpc3RbcGF0aGxpYi5QYXRoXToKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICB0ZXh0X2ZpbGVzID0gbGlzdChkYXRhX3BhdGguZ2xvYigiKi4qIikpCiAgICBlbGlmIGRhdGFfcGF0aC5pc19maWxlKCk6CiAgICAgICAgdGV4dF9maWxlcyA9IFtkYXRhX3BhdGhdCiAgICBlbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgIGYiVW5yZWNvZ25pemVkIGRhdGEgcGF0aC4gVGhlIHBhcmFtZXRlciBgZGF0YV9wYXRoYCBtdXN0IGJlIGVpdGhlciBhIGRpcmVjdG9yeSBwYXRoIG9yIGEgZmlsZSBwYXRoLiAiCiAgICAgICAgICAgIGYiR2l2ZW46IHtzdHIoZGF0YV9wYXRoKX0gIgogICAgICAgICkKCiAgICByZXR1cm4gdGV4dF9maWxlcwoKCmRlZiBfc3BsaXRfbGluZShsaW5lOiBzdHIsIG1heF9sZW5ndGg6IGludCA9IDI1MCkgLT4gTGlzdFtzdHJdOgogICAgaWYgbGVuKGxpbmUpIDwgbWF4X2xlbmd0aDoKICAgICAgICByZXR1cm4gW2xpbmVdCgogICAgc2VudGVuY2VzID0gWwogICAgICAgIGYie3NlbnRlbmNlLnN0cmlwKCl9LiIgZm9yIHNlbnRlbmNlIGluIGxpbmUuc3BsaXQoIi4iKSBpZiBzZW50ZW5jZS5zdHJpcCgpCiAgICBdCgogICAgc3BsaXRzID0gW10KICAgIGN1cnJlbnRfbGVuZ3RoID0gbGVuKHNlbnRlbmNlc1swXSkKICAgIHNwbGl0ID0gc2VudGVuY2VzWzBdCiAgICBmb3Igc2VudGVuY2UgaW4gc2VudGVuY2VzWzE6XToKICAgICAgICBpZiBjdXJyZW50X2xlbmd0aCArIGxlbihzZW50ZW5jZSkgPiBtYXhfbGVuZ3RoOgogICAgICAgICAgICBzcGxpdHMuYXBwZW5kKHNwbGl0KQogICAgICAgICAgICBzcGxpdCA9IHNlbnRlbmNlCiAgICAgICAgICAgIGN1cnJlbnRfbGVuZ3RoID0gbGVuKHNlbnRlbmNlKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGN1cnJlbnRfbGVuZ3RoICs9IGxlbihzZW50ZW5jZSkKICAgICAgICAgICAgc3BsaXQgKz0gIiAiICsgc2VudGVuY2UKICAgIGlmIHNwbGl0OgogICAgICAgIHNwbGl0cy5hcHBlbmQoc3BsaXQpCgogICAgcmV0dXJuIHNwbGl0cwoKCmRlZiBfZ2V0X2xvZ2dlcigpOgogICAgZ2xvYmFsIF9MT0dHRVIKICAgIHRyeToKICAgICAgICBpbXBvcnQgbWxydW4KCiAgICAgICAgIyBDaGVjayBpZiBNTFJ1biBpcyBhdmFpbGFibGU6CiAgICAgICAgY29udGV4dCA9IG1scnVuLmdldF9vcl9jcmVhdGVfY3R4KG5hbWU9Im1scnVuIikKICAgICAgICByZXR1cm4gY29udGV4dC5sb2dnZXIKICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yOgogICAgICAgIHJldHVybiBfTE9HR0VSCg== -metadata: - categories: - - data-generation - - audio - tag: '' - name: text-to-audio-generator -kind: job -verbose: false + default_handler: generate_multi_speakers_audio diff --git a/functions/src/text_to_audio_generator/test_text_to_audio_generator.py b/functions/src/text_to_audio_generator/test_text_to_audio_generator.py index fb8db3198..c8695cb03 100644 --- a/functions/src/text_to_audio_generator/test_text_to_audio_generator.py +++ b/functions/src/text_to_audio_generator/test_text_to_audio_generator.py @@ -86,4 +86,4 @@ def test_generate_multi_speakers_audio_openai(file_format, bits_per_sample): ) assert function_run.error == "" for key in ["audio_files", "audio_files_dataframe", "text_to_speech_errors"]: - assert key in function_run.outputs and function_run.outputs[key] is not None \ No newline at end of file + assert key in function_run.outputs and function_run.outputs[key] is not None diff --git a/functions/src/text_to_audio_generator/text_to_audio_generator.py b/functions/src/text_to_audio_generator/text_to_audio_generator.py index e03b827ff..4c2de03e3 100644 --- a/functions/src/text_to_audio_generator/text_to_audio_generator.py +++ b/functions/src/text_to_audio_generator/text_to_audio_generator.py @@ -19,7 +19,6 @@ import random import tempfile from abc import ABC, abstractmethod -from typing import Dict, List, Optional, Tuple, Union import numpy as np import pandas as pd @@ -37,20 +36,20 @@ def generate_multi_speakers_audio( data_path: str, - speakers: Union[List[str], Dict[str, int]], - available_voices: List[str], + speakers: list[str] | dict[str, int], + available_voices: list[str], engine: str = "openai", output_directory: str = None, - use_gpu: Optional[bool] = None, - use_small_models: Optional[bool] = None, - offload_cpu: Optional[bool] = None, - model: Optional[str] = None, - speed: Optional[float] = None, + use_gpu: bool | None = None, + use_small_models: bool | None = None, + offload_cpu: bool | None = None, + model: str | None = None, + speed: float | None = None, sample_rate: int = 16000, file_format: str = "wav", verbose: bool = True, - bits_per_sample: Optional[int] = None, -) -> Tuple[str, pd.DataFrame, dict]: + bits_per_sample: int | None = None, +) -> tuple[str, pd.DataFrame, dict]: """ Generate audio files from text files. @@ -90,7 +89,6 @@ def generate_multi_speakers_audio( data_path = pathlib.Path(data_path).absolute() text_files = _get_text_files(data_path=data_path) - # Prepare the speech engine: engine = _get_engine( engine=engine, @@ -99,7 +97,7 @@ def generate_multi_speakers_audio( offload_cpu=offload_cpu, model=model, file_format=file_format, - speed=speed + speed=speed, ) # Check for per channel generation: @@ -137,7 +135,6 @@ def generate_multi_speakers_audio( for text_file in tqdm.tqdm( text_files, desc="Generating", unit="file", disable=not verbose ): - try: # Randomize voices for each speaker: chosen_voices = {} @@ -147,7 +144,7 @@ def generate_multi_speakers_audio( chosen_voices[speaker] = voice available_voices_copy.remove(voice) # Read text: - with open(text_file, "r") as fp: + with open(text_file) as fp: text = fp.read() # Prepare a holder for all the generated pieces (if per channel each speaker will have its own): audio_pieces = ( @@ -238,7 +235,12 @@ def _generate_audio(self, text: str, voice: str) -> np.ndarray: class BarkEngine(SpeechEngine): - def __init__(self, use_gpu: bool = True, use_small_models: bool = False, offload_cpu: bool = False): + def __init__( + self, + use_gpu: bool = True, + use_small_models: bool = False, + offload_cpu: bool = False, + ): try: self.bark = importlib.import_module("bark") except ImportError: @@ -268,7 +270,9 @@ def _generate_audio(self, text: str, voice: str) -> np.ndarray: class OpenAIEngine(SpeechEngine): - def __init__(self, model: str = "tts-1", file_format: str = "wav", speed: float = 1.0): + def __init__( + self, model: str = "tts-1", file_format: str = "wav", speed: float = 1.0 + ): try: self.openai = importlib.import_module("openai") self.pydub = importlib.import_module("pydub") @@ -289,7 +293,7 @@ def __init__(self, model: str = "tts-1", file_format: str = "wav", speed: float api_key = context.get_secret(OPENAI_API_KEY) base_url = context.get_secret(OPENAI_BASE_URL) except ModuleNotFoundError: - raise EnvironmentError( + raise OSError( f"One or more of the OpenAI required environment variables ('{OPENAI_API_KEY}', '{OPENAI_BASE_URL}') are missing." f"Please set them as environment variables or install mlrun (`pip install mlrun`)" f"and set them as project secrets using `project.set_secrets`." @@ -342,9 +346,10 @@ def _get_engine(engine: str, file_format: str, **kwargs) -> SpeechEngine: f"Unrecognized engine. The parameter `engine` must be either 'bark' or 'openai'. Given: {engine}" ) + def _get_text_files( data_path: pathlib.Path, -) -> List[pathlib.Path]: +) -> list[pathlib.Path]: # Check if the path is of a directory or a file: if data_path.is_dir(): # Get all files inside the directory: @@ -360,7 +365,7 @@ def _get_text_files( return text_files -def _split_line(line: str, max_length: int = 250) -> List[str]: +def _split_line(line: str, max_length: int = 250) -> list[str]: if len(line) < max_length: return [line] diff --git a/functions/src/tf2_serving/function.yaml b/functions/src/tf2_serving/function.yaml index 17cf2fbb9..bb2fb852f 100644 --- a/functions/src/tf2_serving/function.yaml +++ b/functions/src/tf2_serving/function.yaml @@ -1,52 +1,32 @@ -kind: remote metadata: + tag: '' name: tf2-serving - hash: 134293b94996e74275d90546f8d4ef96198af679 - project: '' - labels: - author: Iguazio categories: - model-serving - machine-learning +verbose: false +kind: remote spec: + image: mlrun/mlrun + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgp3YXJuaW5ncy5zaW1wbGVmaWx0ZXIoYWN0aW9uPSJpZ25vcmUiLCBjYXRlZ29yeT1GdXR1cmVXYXJuaW5nKQoKaW1wb3J0IGpzb24KZnJvbSBvcyBpbXBvcnQgZW52aXJvbgoKaW1wb3J0IG1scnVuCmltcG9ydCBudW1weSBhcyBucApmcm9tIFBJTCBpbXBvcnQgSW1hZ2UKZnJvbSB0ZW5zb3JmbG93LmtlcmFzLm1vZGVscyBpbXBvcnQgbG9hZF9tb2RlbApmcm9tIHRlbnNvcmZsb3cua2VyYXMucHJlcHJvY2Vzc2luZyBpbXBvcnQgaW1hZ2UKCgpjbGFzcyBURk1vZGVsKG1scnVuLnJ1bnRpbWVzLk1MTW9kZWxTZXJ2ZXIpOgogICAgZGVmIF9faW5pdF9fKHNlbGYsIG5hbWU6IHN0ciwgbW9kZWxfZGlyOiBzdHIpOgogICAgICAgIHN1cGVyKCkuX19pbml0X18obmFtZSwgbW9kZWxfZGlyKQoKICAgICAgICBzZWxmLklNQUdFX1dJRFRIID0gaW50KGVudmlyb24uZ2V0KCJJTUFHRV9XSURUSCIsICIxMjgiKSkKICAgICAgICBzZWxmLklNQUdFX0hFSUdIVCA9IGludChlbnZpcm9uLmdldCgiSU1BR0VfSEVJR0hUIiwgIjEyOCIpKQoKICAgICAgICB0cnk6CiAgICAgICAgICAgIHdpdGggb3BlbihlbnZpcm9uWyJjbGFzc2VzX21hcCJdKSBhcyBmOgogICAgICAgICAgICAgICAgc2VsZi5jbGFzc2VzID0ganNvbi5sb2FkKGYpCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgICAgICBzZWxmLmNsYXNzZXMgPSBOb25lCiAgICAgICAgICAgIHByaW50KGYiY291bGQgbm90IGxvYWQgY2xhc3NlcyBtYXA6IHtlfSIpCgogICAgZGVmIGxvYWQoc2VsZik6CiAgICAgICAgbW9kZWxfZmlsZSwgZXh0cmFfZGF0YSA9IHNlbGYuZ2V0X21vZGVsKCIuaDUiKQogICAgICAgIHNlbGYubW9kZWwgPSBsb2FkX21vZGVsKG1vZGVsX2ZpbGUpCgogICAgZGVmIHByZXByb2Nlc3Moc2VsZiwgYm9keSk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICBvdXRwdXQgPSB7Imluc3RhbmNlcyI6IFtdfQogICAgICAgICAgICBpbnN0YW5jZXMgPSBib2R5LmdldCgiaW5zdGFuY2VzIiwgW10pCiAgICAgICAgICAgIGZvciBieXRlX2ltYWdlIGluIGluc3RhbmNlczoKICAgICAgICAgICAgICAgIGltZyA9IEltYWdlLm9wZW4oYnl0ZV9pbWFnZSkKICAgICAgICAgICAgICAgIGltZyA9IGltZy5yZXNpemUoKHNlbGYuSU1BR0VfV0lEVEgsIHNlbGYuSU1BR0VfSEVJR0hUKSkKCiAgICAgICAgICAgICAgICB4ID0gaW1hZ2UuaW1nX3RvX2FycmF5KGltZykKICAgICAgICAgICAgICAgIHggPSBucC5leHBhbmRfZGltcyh4LCBheGlzPTApCiAgICAgICAgICAgICAgICBvdXRwdXRbImluc3RhbmNlcyJdLmFwcGVuZCh4KQoKICAgICAgICAgICAgb3V0cHV0WyJpbnN0YW5jZXMiXSA9IFtucC52c3RhY2sob3V0cHV0WyJpbnN0YW5jZXMiXSldCiAgICAgICAgICAgIHJldHVybiBvdXRwdXQKICAgICAgICBleGNlcHQ6CiAgICAgICAgICAgIHJhaXNlIEV4Y2VwdGlvbihmInJlY2VpdmVkOiB7Ym9keX0iKQoKICAgIGRlZiBwcmVkaWN0KHNlbGYsIGRhdGEpOgogICAgICAgIGltYWdlcyA9IGRhdGEuZ2V0KCJpbnN0YW5jZXMiLCBbXSkKCiAgICAgICAgcHJlZGljdGVkX3Byb2JhYmlsaXR5ID0gc2VsZi5tb2RlbC5wcmVkaWN0KGltYWdlcykKCiAgICAgICAgcmV0dXJuIHByZWRpY3RlZF9wcm9iYWJpbGl0eQoKICAgIGRlZiBwb3N0cHJvY2VzcyhzZWxmLCBwcmVkaWN0ZWRfcHJvYmFiaWxpdHkpOgogICAgICAgIGlmIHNlbGYuY2xhc3NlczoKICAgICAgICAgICAgcHJlZGljdGVkX2NsYXNzZXMgPSBucC5hcm91bmQocHJlZGljdGVkX3Byb2JhYmlsaXR5LCAxKS50b2xpc3QoKVswXQogICAgICAgICAgICBwcmVkaWN0ZWRfcHJvYmFiaWxpdGllcyA9IHByZWRpY3RlZF9wcm9iYWJpbGl0eS50b2xpc3QoKVswXQogICAgICAgICAgICByZXR1cm4gewogICAgICAgICAgICAgICAgInByZWRpY3Rpb24iOiBbCiAgICAgICAgICAgICAgICAgICAgc2VsZi5jbGFzc2VzW3N0cihpbnQoY2xzKSldIGZvciBjbHMgaW4gcHJlZGljdGVkX2NsYXNzZXMKICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICBmIntzZWxmLmNsYXNzZXNbJzEnXX0tcHJvYmFiaWxpdHkiOiBwcmVkaWN0ZWRfcHJvYmFiaWxpdGllcywKICAgICAgICAgICAgfQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHJldHVybiBwcmVkaWN0ZWRfcHJvYmFiaWxpdHkudG9saXN0KClbMF0KCmZyb20gbWxydW4ucnVudGltZXMgaW1wb3J0IG51Y2xpb19pbml0X2hvb2sKZGVmIGluaXRfY29udGV4dChjb250ZXh0KToKICAgIG51Y2xpb19pbml0X2hvb2soY29udGV4dCwgZ2xvYmFscygpLCAnc2VydmluZycpCgpkZWYgaGFuZGxlcihjb250ZXh0LCBldmVudCk6CiAgICByZXR1cm4gY29udGV4dC5tbHJ1bl9oYW5kbGVyKGNvbnRleHQsIGV2ZW50KQo= + requirements: + - requests + - pillow + - tensorflow>=2.1 + code_origin: '' + filename: tf2_serving.py + min_replicas: 1 command: '' - args: [] - image: '' - description: tf2 image classification server + default_handler: '' + source: '' max_replicas: 4 + base_image_pull: false + description: tf2 image classification server + function_kind: serving + function_handler: tf2-serving-nuclio:handler env: - - name: MODEL_CLASS - value: TFModel - - name: ENABLE_EXPLAINER - value: 'False' - config: - spec.triggers.http: - kind: http - maxWorkers: 8 - attributes: - ingresses: {} - annotations: {} - base_spec: - apiVersion: nuclio.io/v1 - kind: nuclio:serving - metadata: - annotations: - nuclio.io/generated_by: function generated from 01-09-2020 - labels: {} - name: tf2-serving - spec: - build: - baseImage: mlrun/mlrun - commands: - - pip install tensorflow>=2.1 - - pip install requests pillow - functionSourceCode: IyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKCgppbXBvcnQgd2FybmluZ3MKd2FybmluZ3Muc2ltcGxlZmlsdGVyKGFjdGlvbj0iaWdub3JlIiwgY2F0ZWdvcnk9RnV0dXJlV2FybmluZykKCmltcG9ydCBqc29uCmltcG9ydCBudW1weSBhcyBucAppbXBvcnQgcmVxdWVzdHMKZnJvbSB0ZW5zb3JmbG93IGltcG9ydCBrZXJhcwpmcm9tIHRlbnNvcmZsb3cua2VyYXMubW9kZWxzIGltcG9ydCBsb2FkX21vZGVsCmZyb20gdGVuc29yZmxvdy5rZXJhcy5wcmVwcm9jZXNzaW5nIGltcG9ydCBpbWFnZQpmcm9tIHRlbnNvcmZsb3cua2VyYXMucHJlcHJvY2Vzc2luZy5pbWFnZSBpbXBvcnQgbG9hZF9pbWcKZnJvbSBvcyBpbXBvcnQgZW52aXJvbiwgcGF0aApmcm9tIFBJTCBpbXBvcnQgSW1hZ2UKZnJvbSBpbyBpbXBvcnQgQnl0ZXNJTwpmcm9tIHVybGxpYi5yZXF1ZXN0IGltcG9ydCB1cmxvcGVuCmltcG9ydCBtbHJ1bgoKY2xhc3MgVEZNb2RlbChtbHJ1bi5ydW50aW1lcy5NTE1vZGVsU2VydmVyKToKICAgIGRlZiBfX2luaXRfXyhzZWxmLCBuYW1lOiBzdHIsIG1vZGVsX2Rpcjogc3RyKToKICAgICAgICBzdXBlcigpLl9faW5pdF9fKG5hbWUsIG1vZGVsX2RpcikKCiAgICAgICAgc2VsZi5JTUFHRV9XSURUSCA9IGludChlbnZpcm9uLmdldCgnSU1BR0VfV0lEVEgnLCAnMTI4JykpCiAgICAgICAgc2VsZi5JTUFHRV9IRUlHSFQgPSBpbnQoZW52aXJvbi5nZXQoJ0lNQUdFX0hFSUdIVCcsICcxMjgnKSkKICAgICAgICAKICAgICAgICB0cnk6CiAgICAgICAgICAgIHdpdGggb3BlbihlbnZpcm9uWydjbGFzc2VzX21hcCddLCAncicpIGFzIGY6CiAgICAgICAgICAgICAgICBzZWxmLmNsYXNzZXMgPSBqc29uLmxvYWQoZikKICAgICAgICBleGNlcHQ6CiAgICAgICAgICAgIHNlbGYuY2xhc3NlcyA9IE5vbmUKICAgICAgICAKICAgIGRlZiBsb2FkKHNlbGYpOgogICAgICAgIG1vZGVsX2ZpbGUsIGV4dHJhX2RhdGEgPSBzZWxmLmdldF9tb2RlbCgnLmg1JykKICAgICAgICBzZWxmLm1vZGVsID0gbG9hZF9tb2RlbChtb2RlbF9maWxlKQogICAgICAgIAogICAgZGVmIHByZXByb2Nlc3Moc2VsZiwgYm9keSk6CiAgICAgICAgdHJ5OgogICAgICAgICAgICBvdXRwdXQgPSB7J2luc3RhbmNlcyc6IFtdfQogICAgICAgICAgICBpbnN0YW5jZXMgPSBib2R5LmdldCgnaW5zdGFuY2VzJywgW10pCiAgICAgICAgICAgIGZvciBieXRlX2ltYWdlIGluIGluc3RhbmNlczoKICAgICAgICAgICAgICAgIGltZyA9IEltYWdlLm9wZW4oYnl0ZV9pbWFnZSkKICAgICAgICAgICAgICAgIGltZyA9IGltZy5yZXNpemUoKHNlbGYuSU1BR0VfV0lEVEgsIHNlbGYuSU1BR0VfSEVJR0hUKSkKCiAgICAgICAgICAgICAgICB4ID0gaW1hZ2UuaW1nX3RvX2FycmF5KGltZykKICAgICAgICAgICAgICAgIHggPSBucC5leHBhbmRfZGltcyh4LCBheGlzPTApCiAgICAgICAgICAgICAgICBvdXRwdXRbJ2luc3RhbmNlcyddLmFwcGVuZCh4KQogICAgICAgICAgICAKICAgICAgICAgICAgb3V0cHV0WydpbnN0YW5jZXMnXSA9IFtucC52c3RhY2sob3V0cHV0WydpbnN0YW5jZXMnXSldCiAgICAgICAgICAgIHJldHVybiBvdXRwdXQKICAgICAgICBleGNlcHQ6CiAgICAgICAgICAgIHJhaXNlIEV4Y2VwdGlvbihmJ3JlY2VpdmVkOiB7Ym9keX0nKQogICAgICAgICAgICAKCiAgICBkZWYgcHJlZGljdChzZWxmLCBkYXRhKToKICAgICAgICBpbWFnZXMgPSBkYXRhLmdldCgnaW5zdGFuY2VzJywgW10pCgogICAgICAgIHByZWRpY3RlZF9wcm9iYWJpbGl0eSA9IHNlbGYubW9kZWwucHJlZGljdChpbWFnZXMpCgogICAgICAgIHJldHVybiBwcmVkaWN0ZWRfcHJvYmFiaWxpdHkKICAgICAgICAKICAgIGRlZiBwb3N0cHJvY2VzcyhzZWxmLCBwcmVkaWN0ZWRfcHJvYmFiaWxpdHkpOgogICAgICAgIGlmIHNlbGYuY2xhc3NlczoKICAgICAgICAgICAgcHJlZGljdGVkX2NsYXNzZXMgPSBucC5hcm91bmQocHJlZGljdGVkX3Byb2JhYmlsaXR5LCAxKS50b2xpc3QoKVswXQogICAgICAgICAgICBwcmVkaWN0ZWRfcHJvYmFiaWxpdGllcyA9IHByZWRpY3RlZF9wcm9iYWJpbGl0eS50b2xpc3QoKVswXQogICAgICAgICAgICByZXR1cm4gewogICAgICAgICAgICAgICAgJ3ByZWRpY3Rpb24nOiBbc2VsZi5jbGFzc2VzW3N0cihpbnQoY2xzKSldIGZvciBjbHMgaW4gcHJlZGljdGVkX2NsYXNzZXNdLCAKICAgICAgICAgICAgICAgIGYne3NlbGYuY2xhc3Nlc1siMSJdfS1wcm9iYWJpbGl0eSc6IHByZWRpY3RlZF9wcm9iYWJpbGl0aWVzCiAgICAgICAgICAgIH0KICAgICAgICBlbHNlOgogICAgICAgICAgICByZXR1cm4gcHJlZGljdGVkX3Byb2JhYmlsaXR5LnRvbGlzdCgpWzBdCgoKZnJvbSBtbHJ1bi5ydW50aW1lcyBpbXBvcnQgbnVjbGlvX2luaXRfaG9vawpkZWYgaW5pdF9jb250ZXh0KGNvbnRleHQpOgogICAgbnVjbGlvX2luaXRfaG9vayhjb250ZXh0LCBnbG9iYWxzKCksICdzZXJ2aW5nJykKCmRlZiBoYW5kbGVyKGNvbnRleHQsIGV2ZW50KToKICAgIHJldHVybiBjb250ZXh0Lm1scnVuX2hhbmRsZXIoY29udGV4dCwgZXZlbnQpCg== - noBaseImagesPull: true - env: - - name: MODEL_CLASS - value: TF2Model - handler: tf2_serving:handler - runtime: python:3.9 - volumes: [] - source: '' - function_kind: serving \ No newline at end of file + - name: MLRUN_HTTPDB__NUCLIO__EXPLICIT_ACK + value: enabled diff --git a/functions/src/tf2_serving/tf2_serving.py b/functions/src/tf2_serving/tf2_serving.py index 57380fbfa..820c4fae9 100644 --- a/functions/src/tf2_serving/tf2_serving.py +++ b/functions/src/tf2_serving/tf2_serving.py @@ -19,17 +19,13 @@ warnings.simplefilter(action="ignore", category=FutureWarning) import json +from os import environ + +import mlrun import numpy as np -import requests -from tensorflow import keras +from PIL import Image from tensorflow.keras.models import load_model from tensorflow.keras.preprocessing import image -from tensorflow.keras.preprocessing.image import load_img -from os import environ, path -from PIL import Image -from io import BytesIO -from urllib.request import urlopen -import mlrun class TFModel(mlrun.runtimes.MLModelServer): @@ -40,10 +36,11 @@ def __init__(self, name: str, model_dir: str): self.IMAGE_HEIGHT = int(environ.get("IMAGE_HEIGHT", "128")) try: - with open(environ["classes_map"], "r") as f: + with open(environ["classes_map"]) as f: self.classes = json.load(f) - except: + except Exception as e: self.classes = None + print(f"could not load classes map: {e}") def load(self): model_file, extra_data = self.get_model(".h5") @@ -81,7 +78,7 @@ def postprocess(self, predicted_probability): "prediction": [ self.classes[str(int(cls))] for cls in predicted_classes ], - f'{self.classes["1"]}-probability': predicted_probabilities, + f"{self.classes['1']}-probability": predicted_probabilities, } else: return predicted_probability.tolist()[0] diff --git a/functions/src/transcribe/test_transcribe.py b/functions/src/transcribe/test_transcribe.py index f70b3856d..4e80580df 100644 --- a/functions/src/transcribe/test_transcribe.py +++ b/functions/src/transcribe/test_transcribe.py @@ -20,7 +20,6 @@ import mlrun import pytest - expected_outputs = [ "This is a speech to text test.", "In the heart of the stadium, " @@ -30,7 +29,6 @@ "as the game writes its unpredictable story on the field of destiny.", ] models = [ - "openai/whisper-tiny", ] @@ -42,7 +40,9 @@ def test_transcribe(model_name: str, audio_path: str): # Setting variables and importing function: artifact_path = tempfile.mkdtemp() project = mlrun.get_or_create_project("test") - transcribe_function = project.set_function("transcribe.py", "transcribe", kind="job", image="mlrun/mlrun") + transcribe_function = project.set_function( + "transcribe.py", "transcribe", kind="job", image="mlrun/mlrun" + ) # transcribe_function = mlrun.import_function("function.yaml") temp_dir = tempfile.mkdtemp() @@ -80,7 +80,7 @@ def test_transcribe(model_name: str, audio_path: str): # Check that the transcribed text was approximately (90%) generated from audio: for text_file, expected in zip(text_files, expected_outputs): - with open(os.path.join(temp_dir, text_file), "r") as f: + with open(os.path.join(temp_dir, text_file)) as f: output = f.readlines()[0] ratio = SequenceMatcher(None, expected, output).ratio() assert ratio >= 0.9 diff --git a/functions/src/transcribe/transcribe.py b/functions/src/transcribe/transcribe.py index 9cabcb1e8..7f30563cb 100644 --- a/functions/src/transcribe/transcribe.py +++ b/functions/src/transcribe/transcribe.py @@ -15,10 +15,11 @@ import operator import os import tempfile +from collections.abc import Generator from functools import reduce, wraps from multiprocessing import Process, Queue from pathlib import Path -from typing import Any, Dict, Generator, List, Literal, NamedTuple, Tuple, Union +from typing import Any, Literal, NamedTuple import pandas as pd import torch @@ -38,7 +39,7 @@ class BaseTask: """ def __init__( - self, audio_file: Path, transcription_output: Union[dict, str], text_file: Path + self, audio_file: Path, transcription_output: dict | str, text_file: Path ): """ Initialize the task. @@ -75,7 +76,7 @@ def is_failed(self) -> bool: """ return self._error is not None - def get_result(self) -> Tuple[str, str]: + def get_result(self) -> tuple[str, str]: """ Get the result of the task. If the task failed, the error will be returned, otherwise, the result will be the text file name. @@ -86,7 +87,7 @@ def get_result(self) -> Tuple[str, str]: return self._audio_file.name, self._error return self._audio_file.name, self._text_file.name - def to_tuple(self) -> Tuple[str, dict]: + def to_tuple(self) -> tuple[str, dict]: """ Convert the task to a tuple to reconstruct it later (used for multiprocessing to pass in queue). @@ -147,7 +148,7 @@ def __init__( audio_file: Path, transcription_output: dict, text_file: Path, - speech_diarization: List[Tuple[float, float, str]], + speech_diarization: list[tuple[float, float, str]], ): """ Initialize the task. @@ -163,10 +164,10 @@ def __init__( text_file=text_file, ) self._speech_diarization = speech_diarization - self._segments: List[SpeechDiarizationTask._DiarizationSegment] = None + self._segments: list[SpeechDiarizationTask._DiarizationSegment] = None self._last_chosen_index = 0 - def to_tuple(self) -> Tuple[str, dict]: + def to_tuple(self) -> tuple[str, dict]: """ Convert the task to a tuple to reconstruct it later (used for multiprocessing to pass in queue). @@ -334,10 +335,10 @@ def __init__(self, audio_file: Path, text_file: Path): super().__init__( audio_file=audio_file, transcription_output={}, text_file=text_file ) - self._transcription_output_channels: List[Tuple[str, dict]] = [] + self._transcription_output_channels: list[tuple[str, dict]] = [] @property - def transcription_output_channels(self) -> List[Tuple[str, dict]]: + def transcription_output_channels(self) -> list[tuple[str, dict]]: """ Get the transcription output channels. @@ -355,7 +356,7 @@ def do_task(self): return super().do_task() - def to_tuple(self) -> Tuple[str, dict]: + def to_tuple(self) -> tuple[str, dict]: """ Convert the task to a tuple to reconstruct it later (used for multiprocessing to pass in queue). @@ -412,7 +413,7 @@ class BatchProcessor: associated methods. """ - def __init__(self, audio_files: List[Path], output_directory: Path): + def __init__(self, audio_files: list[Path], output_directory: Path): """ Initialize the batch processor. @@ -425,10 +426,10 @@ def __init__(self, audio_files: List[Path], output_directory: Path): # Prepare the batching variables: self._current_file_index = 0 - self._tasks: List[BaseTask] = [] - self._results: List[Tuple[bool, Tuple[str, str]]] = [] + self._tasks: list[BaseTask] = [] + self._results: list[tuple[bool, tuple[str, str]]] = [] - def process_batch(self, batch: List[Union[dict, str]]): + def process_batch(self, batch: list[dict | str]): """ Process a batch of transcriptions. Tasks related to the given batch will be created and stored in the batch processor. @@ -450,7 +451,7 @@ def process_batch(self, batch: List[Union[dict, str]]): ] ) - def get_tasks(self) -> List[BaseTask]: + def get_tasks(self) -> list[BaseTask]: """ Get the tasks to perform. @@ -468,7 +469,7 @@ def do_tasks(self): task.do_task() self._results.append((task.is_failed(), task.get_result())) - def get_results(self) -> List[Tuple[bool, Tuple[str, str]]]: + def get_results(self) -> list[tuple[bool, tuple[str, str]]]: """ Get the results of the tasks. The stored results are then cleared. @@ -478,7 +479,7 @@ def get_results(self) -> List[Tuple[bool, Tuple[str, str]]]: self._results = [] return results - def _get_current_files(self, batch_size: int) -> List[Path]: + def _get_current_files(self, batch_size: int) -> list[Path]: """ Get the current files to process. @@ -504,7 +505,7 @@ class SpeechDiarizationBatchProcessor(BatchProcessor): """ def __init__( - self, audio_files: List[Path], output_directory: Path, speech_diarization: dict + self, audio_files: list[Path], output_directory: Path, speech_diarization: dict ): """ Initialize the batch processor. @@ -517,7 +518,7 @@ def __init__( self._speech_diarization = speech_diarization self._audio_files = audio_files - def process_batch(self, batch: List[dict]): + def process_batch(self, batch: list[dict]): """ Process a batch of transcriptions. Tasks related to the given batch will be created and stored in the batch processor. @@ -550,10 +551,10 @@ class PerChannelSpeechDiarizationBatchProcessor(BatchProcessor): def __init__( self, - audio_files: List[Path], + audio_files: list[Path], output_directory: Path, n_channels: int, - speakers: List[str], + speakers: list[str], ): """ Initialize the batch processor. @@ -572,7 +573,7 @@ def __init__( # Prepare a channel buffer to store the channels until the current task created is fully covered: self._task_in_process: SpeechDiarizationPerChannelTask = None - def process_batch(self, batch: List[dict]): + def process_batch(self, batch: list[dict]): """ Process a batch of transcriptions. Tasks related to the given batch will be created and stored in the batch processor. @@ -627,50 +628,50 @@ def __init__( batch_size: int = 2, spoken_language: str = None, translate_to_english: bool = False, - return_timestamps: Union[bool, Literal["word"]] = False, + return_timestamps: bool | Literal["word"] = False, per_channel_transcription: int = 0, ): """ - Initialize the transcriber. - - :param model_name: The model name to use. Should be a model from the OpenAI's Whisper models for - best results (for example "tiny", "base", "large", etc.). - :param device: The device to use for inference. If not given, will use GPU if available. - :param use_flash_attention_2: Whether to use the Flash Attention 2 implementation. It can be used only with - one of the following GPUs: Nvidia H series and Nvidia A series. T4 support - will be available soon. - - Note: If both `use_flash_attention_2` and - `use_better_transformers` are `None`, the optimization will be chosen - automatically according to the available resources. - - :param use_better_transformers: Whether to use the Better Transformers library to further optimize the model. - Should be used for all use cases that do not support flash attention 2. - - Note: If both `use_flash_attention_2` and `use_better_transformers` are - `None`, the optimization will be chosen automatically according to the - available resources. - :param assistant_model: The assistant model name to use for inference. Notice that the optimizations - (flash attention 2 and better transformers) will be applied for the assistant - as well. Should be a model from Huggingface's distil-whisper (see here for - more information: https://github.com/huggingface/distil-whisper). - :param max_new_tokens: The maximum number of new tokens to generate. This is used to limit the - generation length. Default is 128 tokens. - :param chunk_length_s: The audio chunk to split the audio to (in seconds). Default is 30 seconds. - :param batch_size: The batch size to use for inference. Default is 2. - :param spoken_language: Aim whisper to know what language is spoken. If None, it will try to detect it - for each chunk. - :param translate_to_english: Whether to translate the transcriptions to English. Default is False. - :param return_timestamps: Whether to return the timestamps of the words. If "word", will return the - timestamps of each word. If True will return the timestamps of each chunk. - Default is False. Aimed to be used for speech diarization. - :param per_channel_transcription: Whether to do per channel transcription. If needed to run per channel - transcription, pass the number of channels expected for each audio file here. - 0 means regular transcription (merge channels). - - Note: If `per_channel_transcription` is not 0, `batch_size` must be treated to - be the number of channels and not audio files. Aimed to be used for per - channel speech diarization. + Initialize the transcriber. + + :param model_name: The model name to use. Should be a model from the OpenAI's Whisper models for + best results (for example "tiny", "base", "large", etc.). + :param device: The device to use for inference. If not given, will use GPU if available. + :param use_flash_attention_2: Whether to use the Flash Attention 2 implementation. It can be used only with + one of the following GPUs: Nvidia H series and Nvidia A series. T4 support + will be available soon. + + Note: If both `use_flash_attention_2` and + `use_better_transformers` are `None`, the optimization will be chosen + automatically according to the available resources. + + :param use_better_transformers: Whether to use the Better Transformers library to further optimize the model. + Should be used for all use cases that do not support flash attention 2. + + Note: If both `use_flash_attention_2` and `use_better_transformers` are + `None`, the optimization will be chosen automatically according to the + available resources. + :param assistant_model: The assistant model name to use for inference. Notice that the optimizations + (flash attention 2 and better transformers) will be applied for the assistant + as well. Should be a model from Huggingface's distil-whisper (see here for + more information: https://github.com/huggingface/distil-whisper). + :param max_new_tokens: The maximum number of new tokens to generate. This is used to limit the + generation length. Default is 128 tokens. + :param chunk_length_s: The audio chunk to split the audio to (in seconds). Default is 30 seconds. + :param batch_size: The batch size to use for inference. Default is 2. + :param spoken_language: Aim whisper to know what language is spoken. If None, it will try to detect it + for each chunk. + :param translate_to_english: Whether to translate the transcriptions to English. Default is False. + :param return_timestamps: Whether to return the timestamps of the words. If "word", will return the + timestamps of each word. If True will return the timestamps of each chunk. + Default is False. Aimed to be used for speech diarization. + :param per_channel_transcription: Whether to do per channel transcription. If needed to run per channel + transcription, pass the number of channels expected for each audio file here. + 0 means regular transcription (merge channels). + + Note: If `per_channel_transcription` is not 0, `batch_size` must be treated to + be the number of channels and not audio files. Aimed to be used for per + channel speech diarization. """ # Store loading parameters: self._model_name = model_name @@ -781,11 +782,11 @@ def load(self): def transcribe( self, - audio_files: List[Path], + audio_files: list[Path], batch_processor: BatchProcessor = None, batches_queue: Queue = None, verbose: bool = False, - ) -> Union[List[List[dict]], None]: + ) -> list[list[dict]] | None: """ Transcribe the given audio files. The transcriptions will be sent to a queue or a batch processor for further processing like writing to text files. If no queue or batch processor is given, the transcriptions outputs from @@ -799,9 +800,10 @@ def transcribe( :returns: The transcriptions outputs from the pipeline if no queue or batch processor is given, otherwise, `None`. """ + # Wrap the audio files with a function to iterate over them via a generator (save memory and runtime with # Huggingface's pipelines as they preload each input while inference is running): - def audio_iterator() -> Generator[Union[dict, str], None, None]: + def audio_iterator() -> Generator[dict | str, None, None]: if self._per_channel_transcription: for audio_file in audio_files: audio, sampling_rate = torchaudio.load(str(audio_file)) @@ -813,7 +815,7 @@ def audio_iterator() -> Generator[Union[dict, str], None, None]: yield str(audio_file) # Create a batch iterator: - def batch_iterator() -> Generator[List[Union[dict, str]], None, None]: + def batch_iterator() -> Generator[list[dict | str], None, None]: batch = [] for audio in audio_iterator(): batch.append(audio) @@ -899,7 +901,7 @@ def _multiprocessing_process_batches( """ while True: # Get the batch: - batch: List[dict] = batches_queue.get() + batch: list[dict] = batches_queue.get() if batch == _MULTIPROCESSING_STOP_MARK: break @@ -955,7 +957,7 @@ def _multiprocessing_complete_tasks(tasks_queue: Queue, results_queue: Queue): def open_mpi_handler( - worker_inputs: List[str], root_worker_inputs: Dict[str, Any] = None + worker_inputs: list[str], root_worker_inputs: dict[str, Any] = None ): global _LOGGER @@ -1056,7 +1058,7 @@ def wrapper(**kwargs): if comm.recv(source=0): files = [] for file in os.listdir(output_directory): - with open(output_directory / file, "r") as f: + with open(output_directory / file) as f: files.append((file, f.read())) comm.send(files, dest=0) return None @@ -1066,7 +1068,7 @@ def wrapper(**kwargs): return decorator -def _check_mlrun_and_open_mpi() -> Tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intracomm"]: +def _check_mlrun_and_open_mpi() -> tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intracomm"]: is_mpi = False try: import mlrun @@ -1096,7 +1098,7 @@ def _check_mlrun_and_open_mpi() -> Tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intrac @open_mpi_handler(worker_inputs=["data_path"], root_worker_inputs={"verbose": True}) def transcribe( # Input / Output kwargs: - data_path: Union[str, Path, List[Union[str, Path]]], + data_path: str | Path | list[str | Path], output_directory: str = None, # Model loading kwargs: model_name: str = "openai/whisper-tiny", @@ -1111,11 +1113,11 @@ def transcribe( spoken_language: str = None, translate_to_english: bool = False, # Diarization kwargs: - speech_diarization: Dict[str, List[Tuple[float, float, str]]] = None, + speech_diarization: dict[str, list[tuple[float, float, str]]] = None, speech_diarize_per_channel: int = None, - speaker_labels: List[str] = None, + speaker_labels: list[str] = None, # Other kwargs: - use_multiprocessing: Union[bool, int] = False, + use_multiprocessing: bool | int = False, verbose: bool = False, ): """ @@ -1314,8 +1316,8 @@ def transcribe( def _get_audio_files( - data_path: Union[Path, str, list], -) -> List[Path]: + data_path: Path | str | list, +) -> list[Path]: """ Get the audio files to transcribe. If a path to a directory is given, all files in the directory will be collected. @@ -1350,11 +1352,11 @@ def _get_audio_files( def _run( - audio_files: List[Path], + audio_files: list[Path], batch_processor: BatchProcessor, transcriber: Transcriber, verbose: bool, -) -> List[Tuple[bool, Tuple[str, str]]]: +) -> list[tuple[bool, tuple[str, str]]]: """ Run the transcription without multiprocessing. @@ -1367,7 +1369,7 @@ def _run( """ # Load the transcription pipeline: if verbose: - _LOGGER.info(f"Loading the transcription pipeline.") + _LOGGER.info("Loading the transcription pipeline.") transcriber.load() if verbose: _LOGGER.info("Transcription pipeline loaded.") @@ -1385,7 +1387,7 @@ def _run( def _parallel_run( n_workers: int, - audio_files: List[Path], + audio_files: list[Path], batch_processor: BatchProcessor, transcriber: Transcriber, verbose: bool, @@ -1431,7 +1433,7 @@ def _parallel_run( # Load the transcription pipeline: if verbose: - _LOGGER.info(f"Loading the transcription pipeline.") + _LOGGER.info("Loading the transcription pipeline.") transcriber.load() if verbose: _LOGGER.info("Transcription pipeline loaded.") @@ -1446,7 +1448,7 @@ def _parallel_run( stop_marks_counter = 0 while True: # Get a result from the queue: - result: Tuple[bool, Tuple[str, str]] = results_queue.get() + result: tuple[bool, tuple[str, str]] = results_queue.get() if result == _MULTIPROCESSING_STOP_MARK: stop_marks_counter += 1 if stop_marks_counter == n_workers: @@ -1461,4 +1463,4 @@ def _parallel_run( for p in task_completion_processes: p.join() - return results \ No newline at end of file + return results diff --git a/functions/src/translate/function.yaml b/functions/src/translate/function.yaml index eb1ffd345..bda404af3 100644 --- a/functions/src/translate/function.yaml +++ b/functions/src/translate/function.yaml @@ -1,43 +1,58 @@ +metadata: + tag: '' + name: translate + categories: + - genai + - NLP verbose: false +kind: job spec: - description: Translate text files from one language to another - filename: /Users/Daniel_Perez/PycharmProjects/functions/functions/src/translate/translate.py - command: '' + image: '' + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgoKaW1wb3J0IGxvZ2dpbmcKaW1wb3J0IG9wZXJhdG9yCmltcG9ydCBwYXRobGliCmZyb20gZnVuY3Rvb2xzIGltcG9ydCByZWR1Y2UsIHdyYXBzCmZyb20gdHlwaW5nIGltcG9ydCBBbnkKCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IHRyYW5zZm9ybWVycwpmcm9tIHRxZG0gaW1wb3J0IHRxZG0KCiMgR2V0IHRoZSBnbG9iYWwgbG9nZ2VyOgpfTE9HR0VSID0gbG9nZ2luZy5nZXRMb2dnZXIoKQoKCmRlZiBfY2hlY2tfbWxydW5fYW5kX29wZW5fbXBpKCkgLT4gdHVwbGVbIm1scnVuLk1MQ2xpZW50Q3R4IiwgIm1waTRweS5NUEkuSW50cmFjb21tIl06CiAgICBpc19tcGkgPSBGYWxzZQogICAgdHJ5OgogICAgICAgIGltcG9ydCBtbHJ1bgoKICAgICAgICBjb250ZXh0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9jdHgobmFtZT0ibWxydW4iKQogICAgICAgIGlzX21waSA9IGNvbnRleHQubGFiZWxzLmdldCgia2luZCIsICJqb2IiKSA9PSAibXBpam9iIgoKICAgICAgICBpZiBpc19tcGk6CiAgICAgICAgICAgIHRyeToKICAgICAgICAgICAgICAgIGZyb20gbXBpNHB5IGltcG9ydCBNUEkKCiAgICAgICAgICAgICAgICByZXR1cm4gY29udGV4dCwgTVBJLkNPTU1fV09STEQKICAgICAgICAgICAgZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3IgYXMgbXBpNHB5X25vdF9mb3VuZDoKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICAgICAgICAgICJUbyBkaXN0cmlidXRlIHRoZSBmdW5jdGlvbiB1c2luZyBNTFJ1bidzICdtcGlqb2InIHlvdSBuZWVkIHRvIGhhdmUgYG1waTRweWAgcGFja2FnZSBpbiB5b3VyICIKICAgICAgICAgICAgICAgICAgICAiaW50ZXJwcmV0ZXIuIFBsZWFzZSBydW4gYHBpcCBpbnN0YWxsIG1waTRweWAgYW5kIG1ha2Ugc3VyZSB5b3UgaGF2ZSBvcGVuLW1waS4iCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBtcGk0cHlfbm90X2ZvdW5kCiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmV0dXJuIGNvbnRleHQsIE5vbmUKICAgIGV4Y2VwdCBNb2R1bGVOb3RGb3VuZEVycm9yIGFzIG1vZHVsZV9ub3RfZm91bmQ6CiAgICAgICAgaWYgaXNfbXBpOgogICAgICAgICAgICByYWlzZSBtb2R1bGVfbm90X2ZvdW5kCiAgICByZXR1cm4gTm9uZSwgTm9uZQoKCmRlZiBvcGVuX21waV9oYW5kbGVyKAogICAgd29ya2VyX2lucHV0czogbGlzdFtzdHJdLCByb290X3dvcmtlcl9pbnB1dHM6IGRpY3Rbc3RyLCBBbnldID0gTm9uZQopOgogICAgZ2xvYmFsIF9MT0dHRVIKCiAgICAjIENoZWNrIGZvciBNTFJ1biBhbmQgT3Blbk1QSSBhdmFpbGFiaWxpdHk6CiAgICBjb250ZXh0LCBjb21tID0gX2NoZWNrX21scnVuX2FuZF9vcGVuX21waSgpCgogICAgIyBDaGVjayBpZiBNTFJ1biBpcyBhdmFpbGFibGUsIHNldCB0aGUgZ2xvYmFsIGxvZ2dlciB0byBNTFJ1bidzOgogICAgaWYgY29udGV4dDoKICAgICAgICBfTE9HR0VSID0gY29udGV4dC5sb2dnZXIKCiAgICBkZWYgZGVjb3JhdG9yKGhhbmRsZXIpOgogICAgICAgIGlmIGNvbW0gaXMgTm9uZSBvciBjb21tLkdldF9zaXplKCkgPT0gMToKICAgICAgICAgICAgcmV0dXJuIGhhbmRsZXIKCiAgICAgICAgQHdyYXBzKGhhbmRsZXIpCiAgICAgICAgZGVmIHdyYXBwZXIoKiprd2FyZ3MpOgogICAgICAgICAgICAjIEdldCB0aGUgb3BlbiBtcGkgZW52aXJvbm1lbnQgcHJvcGVydGllczoKICAgICAgICAgICAgc2l6ZSA9IGNvbW0uR2V0X3NpemUoKQogICAgICAgICAgICByYW5rID0gY29tbS5HZXRfcmFuaygpCgogICAgICAgICAgICAjIEdpdmUgdGhlIGNvcnJlY3QgY2h1bmsgb2YgdGhlIHdvcmtlcnMgaW5wdXRzOgogICAgICAgICAgICBmb3Igd29ya2VyX2lucHV0IGluIHdvcmtlcl9pbnB1dHM6CiAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IGt3YXJnc1t3b3JrZXJfaW5wdXRdCiAgICAgICAgICAgICAgICBpZiBpbnB1dF9hcmd1bWVudCBpcyBOb25lOgogICAgICAgICAgICAgICAgICAgIGNvbnRpbnVlCiAgICAgICAgICAgICAgICBpZiBpc2luc3RhbmNlKGlucHV0X2FyZ3VtZW50LCAoc3RyLCBwYXRobGliLlBhdGgpKToKICAgICAgICAgICAgICAgICAgICBpbnB1dF9hcmd1bWVudCA9IF9nZXRfdGV4dF9maWxlcygKICAgICAgICAgICAgICAgICAgICAgICAgZGF0YV9wYXRoPXBhdGhsaWIuUGF0aChpbnB1dF9hcmd1bWVudCkuYWJzb2x1dGUoKQogICAgICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGlmIGxlbihpbnB1dF9hcmd1bWVudCkgPCBzaXplOgogICAgICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICAgICAgICAgICAgIGYiQ2Fubm90IHNwbGl0IHRoZSBpbnB1dCAne3dvcmtlcl9pbnB1dH0nIG9mIGxlbmd0aCB7bGVuKGlucHV0X2FyZ3VtZW50KX0gdG8ge3NpemV9IHdvcmtlcnMuICIKICAgICAgICAgICAgICAgICAgICAgICAgZiJQbGVhc2UgcmVkdWNlIHRoZSBhbW91bnQgb2Ygd29ya2VycyBmb3IgdGhpcyBpbnB1dC4iCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZXZlbl9jaHVua19zaXplID0gbGVuKGlucHV0X2FyZ3VtZW50KSAvLyBzaXplCiAgICAgICAgICAgICAgICBjaHVua19zdGFydCA9IHJhbmsgKiBldmVuX2NodW5rX3NpemUKICAgICAgICAgICAgICAgIGNodW5rX2VuZCA9ICgKICAgICAgICAgICAgICAgICAgICAocmFuayArIDEpICogZXZlbl9jaHVua19zaXplCiAgICAgICAgICAgICAgICAgICAgaWYgcmFuayArIDEgPCBzaXplCiAgICAgICAgICAgICAgICAgICAgZWxzZSBsZW4oaW5wdXRfYXJndW1lbnQpCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgICAgIGYiUmFuayAje3Jhbmt9OiBQcm9jZXNzaW5nIGlucHV0IGNodW5rIG9mICd7d29ya2VyX2lucHV0fScgIgogICAgICAgICAgICAgICAgICAgIGYiZnJvbSBpbmRleCB7Y2h1bmtfc3RhcnR9IHRvIHtjaHVua19lbmR9LiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIGxpc3QpOgogICAgICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0gaW5wdXRfYXJndW1lbnRbY2h1bmtfc3RhcnQ6Y2h1bmtfZW5kXQogICAgICAgICAgICAgICAgZWxpZiBpc2luc3RhbmNlKGlucHV0X2FyZ3VtZW50LCBwZC5EYXRhRnJhbWUpOgogICAgICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0gaW5wdXRfYXJndW1lbnQuaWxvY1tjaHVua19zdGFydDpjaHVua19lbmQ6LCA6XQogICAgICAgICAgICAgICAga3dhcmdzW3dvcmtlcl9pbnB1dF0gPSBpbnB1dF9hcmd1bWVudAoKICAgICAgICAgICAgIyBTZXQgdGhlIHJvb3Qgd29ya2VyIG9ubHkgYXJndW1lbnRzOgogICAgICAgICAgICBpZiByYW5rID09IDAgYW5kIHJvb3Rfd29ya2VyX2lucHV0czoKICAgICAgICAgICAgICAgIGt3YXJncy51cGRhdGUocm9vdF93b3JrZXJfaW5wdXRzKQoKICAgICAgICAgICAgIyBSdW4gdGhlIHdvcmtlcjoKICAgICAgICAgICAgb3V0cHV0ID0gaGFuZGxlcigqKmt3YXJncykKCiAgICAgICAgICAgICMgU2VuZCB0aGUgb3V0cHV0IHRvIHRoZSByb290IHJhbmsgKHJhbmsgIzApOgogICAgICAgICAgICBvdXRwdXQgPSBjb21tLmdhdGhlcihvdXRwdXQsIHJvb3Q9MCkKICAgICAgICAgICAgaWYgcmFuayA9PSAwOgogICAgICAgICAgICAgICAgIyBKb2luIHRoZSBvdXRwdXRzOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygiQ29sbGVjdGluZyBkYXRhIGZyb20gd29ya2VycyB0byByb290IHdvcmtlci4iKQogICAgICAgICAgICAgICAgb3V0cHV0X2RpcmVjdG9yeSA9IG91dHB1dFswXVswXQogICAgICAgICAgICAgICAgZGF0YWZyYW1lID0gcGQuY29uY2F0KG9ianM9W2RmIGZvciBfLCBkZiwgXyBpbiBvdXRwdXRdLCBheGlzPTApCiAgICAgICAgICAgICAgICBlcnJvcnNfZGljdGlvbmFyeSA9IHJlZHVjZSgKICAgICAgICAgICAgICAgICAgICBvcGVyYXRvci5pb3IsIFtlcnIgZm9yIF8sIF8sIGVyciBpbiBvdXRwdXRdLCB7fQogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgcmV0dXJuIG91dHB1dF9kaXJlY3RvcnksIGRhdGFmcmFtZSwgZXJyb3JzX2RpY3Rpb25hcnkKICAgICAgICAgICAgcmV0dXJuIE5vbmUKCiAgICAgICAgcmV0dXJuIHdyYXBwZXIKCiAgICByZXR1cm4gZGVjb3JhdG9yCgoKQG9wZW5fbXBpX2hhbmRsZXIod29ya2VyX2lucHV0cz1bImRhdGFfcGF0aCJdLCByb290X3dvcmtlcl9pbnB1dHM9eyJ2ZXJib3NlIjogVHJ1ZX0pCmRlZiB0cmFuc2xhdGUoCiAgICBkYXRhX3BhdGg6IHN0ciB8IGxpc3Rbc3RyXSB8IHBhdGhsaWIuUGF0aCwKICAgIG91dHB1dF9kaXJlY3Rvcnk6IHN0ciwKICAgIG1vZGVsX25hbWU6IHN0ciA9IE5vbmUsCiAgICBzb3VyY2VfbGFuZ3VhZ2U6IHN0ciA9IE5vbmUsCiAgICB0YXJnZXRfbGFuZ3VhZ2U6IHN0ciA9IE5vbmUsCiAgICBkZXZpY2U6IHN0ciA9IE5vbmUsCiAgICBtb2RlbF9rd2FyZ3M6IGRpY3QgPSBOb25lLAogICAgYmF0Y2hfc2l6ZTogaW50ID0gMSwKICAgIHRyYW5zbGF0aW9uX2t3YXJnczogZGljdCA9IE5vbmUsCiAgICB2ZXJib3NlOiBib29sID0gRmFsc2UsCikgLT4gdHVwbGVbc3RyLCBwZC5EYXRhRnJhbWUsIGRpY3RdOgogICAgIiIiCiAgICBUcmFuc2xhdGUgdGV4dCBmaWxlcyB1c2luZyBhIHRyYW5zZm9ybWVyIG1vZGVsIGZyb20gSHVnZ2luZ2ZhY2UncyBodWIgYWNjb3JkaW5nIHRvIHRoZSBzb3VyY2UgYW5kIHRhcmdldCBsYW5ndWFnZXMKICAgIGdpdmVuIChvciB1c2luZyB0aGUgZGlyZWN0bHkgcHJvdmlkZWQgbW9kZWwgbmFtZSkuIFRoZSBlbmQgcmVzdWx0IGlzIGEgZGlyZWN0b3J5IG9mIHRyYW5zbGF0ZWQgdGV4dCBmaWxlcyBhbmQgYQogICAgZGF0YWZyYW1lIGNvbnRhaW5pbmcgdGhlIGZvbGxvd2luZyBjb2x1bW5zOgoKICAgICogdGV4dF9maWxlIC0gVGhlIHRleHQgZmlsZSBwYXRoLgogICAgKiB0cmFuc2xhdGlvbl9maWxlIC0gVGhlIHRyYW5zbGF0aW9uIHRleHQgZmlsZSBuYW1lIGluIHRoZSBvdXRwdXQgZGlyZWN0b3J5LgoKICAgIDpwYXJhbSBkYXRhX3BhdGg6ICAgICAgICAgIEEgZGlyZWN0b3J5IG9mIHRleHQgZmlsZXMgb3IgYSBzaW5nbGUgZmlsZSBvciBhIGxpc3Qgb2YgZmlsZXMgdG8gdHJhbnNsYXRlLgogICAgOnBhcmFtIG91dHB1dF9kaXJlY3Rvcnk6ICAgRGlyZWN0b3J5IHdoZXJlIHRoZSB0cmFuc2xhdGVkIGZpbGVzIHdpbGwgYmUgc2F2ZWQuCiAgICA6cGFyYW0gbW9kZWxfbmFtZTogICAgICAgICBUaGUgbmFtZSBvZiBhIG1vZGVsIHRvIGxvYWQuIElmIE5vbmUsIHRoZSBtb2RlbCBuYW1lIGlzIGNvbnN0cnVjdGVkIHVzaW5nIHRoZSBzb3VyY2UgYW5kCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0YXJnZXQgbGFuZ3VhZ2VzIHBhcmFtZXRlcnMuCiAgICA6cGFyYW0gc291cmNlX2xhbmd1YWdlOiAgICBUaGUgc291cmNlIGxhbmd1YWdlIGNvZGUgKGUuZy4sICdlbicgZm9yIEVuZ2xpc2gpLgogICAgOnBhcmFtIHRhcmdldF9sYW5ndWFnZTogICAgVGhlIHRhcmdldCBsYW5ndWFnZSBjb2RlIChlLmcuLCAnZW4nIGZvciBFbmdsaXNoKS4KICAgIDpwYXJhbSBtb2RlbF9rd2FyZ3M6ICAgICAgIEtleXdvcmQgYXJndW1lbnRzIHRvIHBhc3MgcmVnYXJkaW5nIHRoZSBsb2FkaW5nIG9mIHRoZSBtb2RlbCBpbiBIdWdnaW5nRmFjZSdzIGBwaXBlbGluZWAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGZ1bmN0aW9uLgogICAgOnBhcmFtIGRldmljZTogICAgICAgICAgICAgVGhlIGRldmljZSBpbmRleCBmb3IgdHJhbnNmb3JtZXJzLiBEZWZhdWx0IHdpbGwgcHJlZmVyIGN1ZGEgaWYgYXZhaWxhYmxlLgogICAgOnBhcmFtIGJhdGNoX3NpemU6ICAgICAgICAgVGhlIG51bWJlciBvZiBiYXRjaGVzIHRvIHVzZSBpbiB0cmFuc2xhdGlvbi4gVGhlIGZpbGVzIGFyZSB0cmFuc2xhdGVkIG9uZSBieSBvbmUsIGJ1dCB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNlbnRlbmNlcyBjYW4gYmUgYmF0Y2hlZC4KICAgIDpwYXJhbSB0cmFuc2xhdGlvbl9rd2FyZ3M6IEFkZGl0aW9uYWwga2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byBhIGB0cmFuc2Zvcm1lcnMuVHJhbnNsYXRpb25QaXBlbGluZWAgd2hlbiBkb2luZwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhlIHRyYW5zbGF0aW9uIGluZmVyZW5jZS4gTm90aWNlIHRoZSBiYXRjaCBzaXplIGhlcmUgaXMgYmVpbmcgYWRkZWQgYXV0b21hdGljYWxseS4KICAgIDpwYXJhbSB2ZXJib3NlOiAgICAgICAgICAgIFdoZXRoZXIgdG8gcHJlc2VudCBsb2dzIG9mIGEgcHJvZ3Jlc3MgYmFyIGFuZCBlcnJvcnMuIERlZmF1bHQ6IFRydWUuCgogICAgOnJldHVybnM6IEEgdHVwbGUgb2Y6CgogICAgICAgICAgICAgICogUGF0aCB0byB0aGUgb3V0cHV0IGRpcmVjdG9yeS4KICAgICAgICAgICAgICAqIEEgZGF0YWZyYW1lIGRhdGFzZXQgb2YgdGhlIHRyYW5zbGF0ZWQgZmlsZSBuYW1lcy4KICAgICAgICAgICAgICAqIEEgZGljdGlvbmFyeSBvZiBlcnJvcmVkIGZpbGVzIHRoYXQgd2VyZSBub3QgdHJhbnNsYXRlZC4KICAgICIiIgogICAgZ2xvYmFsIF9MT0dHRVIKCiAgICAjIEdldCB0aGUgaW5wdXQgdGV4dCBmaWxlcyB0byB0cmFuc2xhdGU6CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygiQ29sbGVjdGluZyB0ZXh0IGZpbGVzLiIpCiAgICBpZiBpc2luc3RhbmNlKGRhdGFfcGF0aCwgc3RyKToKICAgICAgICBkYXRhX3BhdGggPSBwYXRobGliLlBhdGgoZGF0YV9wYXRoKS5hYnNvbHV0ZSgpCiAgICAgICAgdGV4dF9maWxlcyA9IF9nZXRfdGV4dF9maWxlcyhkYXRhX3BhdGg9ZGF0YV9wYXRoKQogICAgZWxzZToKICAgICAgICB0ZXh0X2ZpbGVzID0gZGF0YV9wYXRoCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIkNvbGxlY3RlZCB7bGVuKHRleHRfZmlsZXMpfSB0ZXh0IGZpbGVzLiIpCgogICAgIyBHZXQgdGhlIHRyYW5zbGF0aW9uIHBpcGVsaW5lOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJMb2FkaW5nIG1vZGVsIC0gdXNpbmcgZGV2aWNlICd7ZGV2aWNlfScuIikKICAgIHRyYW5zbGF0aW9uX3BpcGVsaW5lLCBtb2RlbF9uYW1lID0gX2dldF90cmFuc2xhdGlvbl9waXBlbGluZSgKICAgICAgICBtb2RlbF9uYW1lPW1vZGVsX25hbWUsCiAgICAgICAgc291cmNlX2xhbmd1YWdlPXNvdXJjZV9sYW5ndWFnZSwKICAgICAgICB0YXJnZXRfbGFuZ3VhZ2U9dGFyZ2V0X2xhbmd1YWdlLAogICAgICAgIGRldmljZT1kZXZpY2UsCiAgICAgICAgbW9kZWxfa3dhcmdzPW1vZGVsX2t3YXJncywKICAgICAgICBiYXRjaF9zaXplPWJhdGNoX3NpemUgaWYgYmF0Y2hfc2l6ZSAhPSAxIGVsc2UgTm9uZSwKICAgICkKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiTW9kZWwgJ3ttb2RlbF9uYW1lfScgd2FzIGxvYWRlZCBzdWNjZXNzZnVsbHkuIikKCiAgICAjIFByZXBhcmUgdGhlIHN1Y2Nlc3NlcyBkYXRhZnJhbWUgYW5kIGVycm9ycyBkaWN0aW9uYXJ5IHRvIGJlIHJldHVybmVkOgogICAgc3VjY2Vzc2VzID0gW10KICAgIGVycm9ycyA9IHt9CgogICAgIyBDcmVhdGUgdGhlIG91dHB1dCBkaXJlY3Rvcnk6CiAgICBvdXRwdXRfZGlyZWN0b3J5ID0gcGF0aGxpYi5QYXRoKG91dHB1dF9kaXJlY3RvcnkpCiAgICBvdXRwdXRfZGlyZWN0b3J5Lm1rZGlyKHBhcmVudHM9VHJ1ZSwgZXhpc3Rfb2s9VHJ1ZSkKCiAgICAjIFByZXBhcmUgdGhlIHRyYW5zbGF0aW9uIGtleXdvcmQgYXJndW1lbnRzOgogICAgdHJhbnNsYXRpb25fa3dhcmdzID0gdHJhbnNsYXRpb25fa3dhcmdzIG9yIHt9CgogICAgIyBHbyBvdmVyIHRoZSBhdWRpbyBmaWxlcyBhbmQgdHJhbnNjcmliZToKICAgIGZvciB0ZXh0X2ZpbGUgaW4gdHFkbSgKICAgICAgICB0ZXh0X2ZpbGVzLCBkZXNjPSJUcmFuc2xhdGluZyIsIHVuaXQ9ImZpbGUiLCBkaXNhYmxlPW5vdCB2ZXJib3NlCiAgICApOgogICAgICAgIHRyeToKICAgICAgICAgICAgIyBUcmFuc2xhdGU6CiAgICAgICAgICAgIHRyYW5zbGF0aW9uID0gX3RyYW5zbGF0ZSgKICAgICAgICAgICAgICAgIHRleHRfZmlsZT10ZXh0X2ZpbGUsCiAgICAgICAgICAgICAgICB0cmFuc2xhdGlvbl9waXBlbGluZT10cmFuc2xhdGlvbl9waXBlbGluZSwKICAgICAgICAgICAgICAgIHRyYW5zbGF0aW9uX2t3YXJncz10cmFuc2xhdGlvbl9rd2FyZ3MsCiAgICAgICAgICAgICkKICAgICAgICAgICAgIyBXcml0ZSB0aGUgdHJhbnNjcmlwdGlvbiB0byBmaWxlOgogICAgICAgICAgICB0cmFuc2xhdGlvbl9maWxlID0gX3NhdmVfdG9fZmlsZSgKICAgICAgICAgICAgICAgIHRyYW5zbGF0aW9uPXRyYW5zbGF0aW9uLAogICAgICAgICAgICAgICAgZmlsZV9uYW1lPXRleHRfZmlsZS5zdGVtLAogICAgICAgICAgICAgICAgb3V0cHV0X2RpcmVjdG9yeT1vdXRwdXRfZGlyZWN0b3J5LAogICAgICAgICAgICApCiAgICAgICAgICAgICMgTm90ZSBhcyBhIHN1Y2Nlc3MgaW4gdGhlIGxpc3Q6CiAgICAgICAgICAgIHN1Y2Nlc3Nlcy5hcHBlbmQoCiAgICAgICAgICAgICAgICBbCiAgICAgICAgICAgICAgICAgICAgdGV4dF9maWxlLm5hbWUsCiAgICAgICAgICAgICAgICAgICAgdHJhbnNsYXRpb25fZmlsZS5uYW1lLAogICAgICAgICAgICAgICAgXQogICAgICAgICAgICApCiAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBleGNlcHRpb246CiAgICAgICAgICAgICMgTm90ZSB0aGUgZXhjZXB0aW9uIGFzIGVycm9yIGluIHRoZSBkaWN0aW9uYXJ5OgogICAgICAgICAgICBpZiB2ZXJib3NlOgogICAgICAgICAgICAgICAgX0xPR0dFUi53YXJuaW5nKGYiRXJyb3IgaW4gZmlsZTogJ3t0ZXh0X2ZpbGUubmFtZX0nIikKICAgICAgICAgICAgZXJyb3JzW3N0cih0ZXh0X2ZpbGUubmFtZSldID0gc3RyKGV4Y2VwdGlvbikKICAgICAgICAgICAgY29udGludWUKCiAgICAjIENvbnN0cnVjdCB0aGUgdHJhbnNsYXRpb25zIGRhdGFmcmFtZToKICAgIGNvbHVtbnMgPSBbCiAgICAgICAgInRleHRfZmlsZSIsCiAgICAgICAgInRyYW5zbGF0aW9uX2ZpbGUiLAogICAgXQogICAgc3VjY2Vzc2VzID0gcGQuRGF0YUZyYW1lKAogICAgICAgIHN1Y2Nlc3NlcywKICAgICAgICBjb2x1bW5zPWNvbHVtbnMsCiAgICApCgogICAgIyBQcmludCB0aGUgaGVhZCBvZiB0aGUgcHJvZHVjZWQgZGF0YWZyYW1lIGFuZCByZXR1cm46CiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbygKICAgICAgICAgICAgZiJEb25lICh7c3VjY2Vzc2VzLnNoYXBlWzBdfS97bGVuKHRleHRfZmlsZXMpfSlcbiIKICAgICAgICAgICAgZiJUcmFuc2xhdGlvbnMgc3VtbWFyeTpcbiIKICAgICAgICAgICAgZiJ7c3VjY2Vzc2VzLmhlYWQoKX0iCiAgICAgICAgKQogICAgcmV0dXJuIHN0cihvdXRwdXRfZGlyZWN0b3J5KSwgc3VjY2Vzc2VzLCBlcnJvcnMKCgpkZWYgX2dldF90ZXh0X2ZpbGVzKAogICAgZGF0YV9wYXRoOiBwYXRobGliLlBhdGgsCikgLT4gbGlzdFtwYXRobGliLlBhdGhdOgogICAgIyBDaGVjayBpZiB0aGUgcGF0aCBpcyBvZiBhIGRpcmVjdG9yeSBvciBhIGZpbGU6CiAgICBpZiBkYXRhX3BhdGguaXNfZGlyKCk6CiAgICAgICAgIyBHZXQgYWxsIGZpbGVzIGluc2lkZSB0aGUgZGlyZWN0b3J5OgogICAgICAgIHRleHRfZmlsZXMgPSBsaXN0KGRhdGFfcGF0aC5nbG9iKCIqLioiKSkKICAgIGVsaWYgZGF0YV9wYXRoLmlzX2ZpbGUoKToKICAgICAgICB0ZXh0X2ZpbGVzID0gW2RhdGFfcGF0aF0KICAgIGVsc2U6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgZiJVbnJlY29nbml6ZWQgZGF0YSBwYXRoLiBUaGUgcGFyYW1ldGVyIGBkYXRhX3BhdGhgIG11c3QgYmUgZWl0aGVyIGEgZGlyZWN0b3J5IHBhdGggb3IgYSBmaWxlIHBhdGguICIKICAgICAgICAgICAgZiJHaXZlbjoge3N0cihkYXRhX3BhdGgpfSAiCiAgICAgICAgKQoKICAgIHJldHVybiB0ZXh0X2ZpbGVzCgoKZGVmIF9nZXRfdHJhbnNsYXRpb25fcGlwZWxpbmUoCiAgICBtb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgc291cmNlX2xhbmd1YWdlOiBzdHIgPSBOb25lLAogICAgdGFyZ2V0X2xhbmd1YWdlOiBzdHIgPSBOb25lLAogICAgZGV2aWNlOiBzdHIgPSBOb25lLAogICAgbW9kZWxfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIGJhdGNoX3NpemU6IGludCA9IE5vbmUsCikgLT4gdHVwbGVbdHJhbnNmb3JtZXJzLlBpcGVsaW5lLCBzdHJdOgogICAgIyBDb25zdHJ1Y3QgdGhlIG1vZGVsIG5hbWUgLSBpZiBtb2RlbCBuYW1lIGlzIHByb3ZpZGVkIChub3QgTm9uZSkgdGhlbiB3ZSB0YWtlIGl0LCBvdGhlcndpc2Ugd2UgY2hlY2sgYm90aCBzb3VyY2UKICAgICMgYW5kIHRhcmdldCB3ZXJlIHByb3ZpZGVkIHRvIGNvbnN0cnVjdCB0aGUgbW9kZWwgbmFtZToKICAgIGlmIG1vZGVsX25hbWUgaXMgTm9uZSBhbmQgKHNvdXJjZV9sYW5ndWFnZSBpcyBOb25lIG9yIHRhcmdldF9sYW5ndWFnZSBpcyBOb25lKToKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAiTm8gbW9kZWwgbmFtZSB3ZXJlIGdpdmVuIGFuZCBtaXNzaW5nIHNvdXJjZSBhbmQgLyBvciB0YXJnZXQgbGFuZ3VhZ2VzLiBJbiBvcmRlciB0byB0cmFuc2xhdGUgeW91IG11c3QgIgogICAgICAgICAgICAicGFzcyBhIGBtb2RlbF9uYW1lYCBvciBib3RoIGBzb3VyY2VfbGFuZ3VhZ2VgIGFuZCBgdGFyZ2V0X2xhbmd1YWdlYC4iCiAgICAgICAgKQogICAgZWxpZiBtb2RlbF9uYW1lIGlzIE5vbmU6CiAgICAgICAgbW9kZWxfbmFtZSA9IGYiSGVsc2lua2ktTkxQL29wdXMtbXQte3NvdXJjZV9sYW5ndWFnZX0te3RhcmdldF9sYW5ndWFnZX0iCgogICAgIyBJbml0aWFsaXplIHRoZSB0cmFuc2xhdGlvbiBwaXBlbGluZToKICAgIHRyeToKICAgICAgICB0cmFuc2xhdGlvbl9waXBlbGluZSA9IHRyYW5zZm9ybWVycy5waXBlbGluZSgKICAgICAgICAgICAgdGFzaz0idHJhbnNsYXRpb24iLAogICAgICAgICAgICBtb2RlbD1tb2RlbF9uYW1lLAogICAgICAgICAgICB0b2tlbml6ZXI9bW9kZWxfbmFtZSwKICAgICAgICAgICAgZGV2aWNlPWRldmljZSwKICAgICAgICAgICAgbW9kZWxfa3dhcmdzPW1vZGVsX2t3YXJncywKICAgICAgICAgICAgYmF0Y2hfc2l6ZT1iYXRjaF9zaXplLAogICAgICAgICkKICAgIGV4Y2VwdCBPU0Vycm9yIGFzIGxvYWRfZXhjZXB0aW9uOgogICAgICAgIGlmICgKICAgICAgICAgICAgImlzIG5vdCBhIHZhbGlkIG1vZGVsIGlkZW50aWZpZXIgbGlzdGVkIG9uICdodHRwczovL2h1Z2dpbmdmYWNlLmNvL21vZGVscyciCiAgICAgICAgICAgIGluIHN0cihsb2FkX2V4Y2VwdGlvbikKICAgICAgICAgICAgYW5kIHNvdXJjZV9sYW5ndWFnZQogICAgICAgICk6CiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgICAgICBmIlRoZSBtb2RlbCAne21vZGVsX25hbWV9JyBpcyBub3QgYSB2YWxpZCBtb2RlbCBpZGVudGlmaWVyLiAiCiAgICAgICAgICAgICAgICBmIlRoZSBwYXJhbWV0ZXJzIGBzb3VyY2VfbGFuZ3VhZ2VgIGFuZCBgdGFyZ2V0X2xhbmd1YWdlYCBhcmUgdXNlZCB0byBjb25zdHJ1Y3QgYSBIZWxzaW5raSBtb2RlbCBmb3IgIgogICAgICAgICAgICAgICAgZiJ0ZXh0IHRvIHRleHQgZ2VuZXJhdGlvbiwgYnV0IHRoZSBtb2RlbCBjcmVhdGVkIGZyb20gdGhlIGdpdmVuIGxhbmd1YWdlcyBkb2VzIG5vdCBleGlzdC4gIgogICAgICAgICAgICAgICAgZiJZb3UgbWF5IGNoZWNrIGxhbmd1YWdlIGlkZW50aWZpZXJzIGF0ICIKICAgICAgICAgICAgICAgIGYiaHR0cHM6Ly9kZXZlbG9wZXJzLmdvb2dsZS5jb20vYWRtaW4tc2RrL2RpcmVjdG9yeS92MS9sYW5ndWFnZXMsIGFuZCBpZiB0aGUgZXJyb3Igd2FzIG5vdCBmaXhlZCwgb25lICIKICAgICAgICAgICAgICAgIGYib3IgbW9yZSBsYW5ndWFnZSBjb2RlIG1pZ2h0IGJlIHdpdGggMyBsZXR0ZXJzIGFuZCBuZWVkcyB0byBiZSBmb3VuZCBvbmxpbmUuICIKICAgICAgICAgICAgICAgIGYiUmVtZW1iZXIsIHlvdSBjYW4gYWx3YXlzIGNob29zZSBhIG1vZGVsIGRpcmVjdGx5IGZyb20gdGhlIEh1Z2dpbmdmYWNlIGh1YiBieSB1c2luZyB0aGUgYG1vZGVsX25hbWVgICIKICAgICAgICAgICAgICAgIGYicGFyYW1ldGVyLiIKICAgICAgICAgICAgKSBmcm9tIGxvYWRfZXhjZXB0aW9uCiAgICAgICAgcmFpc2UgbG9hZF9leGNlcHRpb24KCiAgICByZXR1cm4gdHJhbnNsYXRpb25fcGlwZWxpbmUsIG1vZGVsX25hbWUKCgpkZWYgX3RyYW5zbGF0ZSgKICAgIHRleHRfZmlsZTogcGF0aGxpYi5QYXRoLAogICAgdHJhbnNsYXRpb25fcGlwZWxpbmU6IHRyYW5zZm9ybWVycy5QaXBlbGluZSwKICAgIHRyYW5zbGF0aW9uX2t3YXJnczogZGljdCwKKSAtPiBzdHI6CiAgICAjIFJlYWQgdGhlIHRleHQgZnJvbSBmaWxlOgogICAgd2l0aCBvcGVuKHRleHRfZmlsZSkgYXMgZnA6CiAgICAgICAgdGV4dCA9IGZwLnJlYWQoKQoKICAgICMgU3BsaXQgdG8gcGFyYWdyYXBocyBhbmQgZWFjaCBwYXJhZ3JhcGggdG8gc2VudGVuY2VzOgogICAgcGFyYWdyYXBocyA9IFtwYXJhZ3JhcGguc3BsaXQoIi4iKSBmb3IgcGFyYWdyYXBoIGluIHRleHQuc3BsaXQoIlxuIildCgogICAgIyBEaXNjb3ZlciB0aGUgbmV3bGluZSBpbmRleGVzIHRvIHJlc3RvcmUgdGhlIGZpbGUgdG8gaXRzIHN0cnVjdHVyZSBwb3N0IHRyYW5zbGF0aW9uOgogICAgbmV3bGluZXNfaW5kZXhlcyA9IFtdCiAgICBmb3IgcGFyYWdyYXBoIGluIHBhcmFncmFwaHNbOi0xXToKICAgICAgICBpZiBsZW4obmV3bGluZXNfaW5kZXhlcykgPT0gMDoKICAgICAgICAgICAgbmV3bGluZXNfaW5kZXhlcy5hcHBlbmQobGVuKHBhcmFncmFwaCkgLSAxKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIG5ld2xpbmVzX2luZGV4ZXMuYXBwZW5kKG5ld2xpbmVzX2luZGV4ZXNbLTFdICsgbGVuKHBhcmFncmFwaCkpCgogICAgIyBQcmVwYXJlIHRoZSBiYXRjaGVzIChlYWNoIHNlbnRlbmNlIGZyb20gdGhlIHBhcmFncmFwaHMpLiBOb3RpY2Ugd2UgYWRkIGEgZG90IG5vdCBvbmx5IHRvIHJlc3RvcmUgdGhlIHNlbnRlbmNlCiAgICAjIHN0cnVjdHVyZSBidXQgdG8gaWdub3JlIGVtcHR5IHN0cmluZ3MgYXMgaXQgd2lsbCBydWluIHRoZSB0cmFuc2xhdGlvbjoKICAgIHNlbnRlbmNlcyA9IFtmIntsaW5lfS4iIGZvciBwYXJhZ3JhcGggaW4gcGFyYWdyYXBocyBmb3IgbGluZSBpbiBwYXJhZ3JhcGhdCgogICAgIyBUcmFuc2xhdGUgdGhlIHNlbnRlbmNlczoKICAgIHRyYW5zbGF0aW9ucyA9IHRyYW5zbGF0aW9uX3BpcGVsaW5lKHNlbnRlbmNlcywgKip0cmFuc2xhdGlvbl9rd2FyZ3MpCgogICAgIyBSZXN0cnVjdHVyZSB0aGUgZnVsbCB0ZXh0IGZyb20gdGhlIHNlbnRlbmNlczoKICAgIHRyYW5zbGF0ZWRfdGV4dCA9IFtdCiAgICBuZXdsaW5lX2luZGV4ID0gbmV3bGluZXNfaW5kZXhlcy5wb3AoMCkgaWYgbmV3bGluZXNfaW5kZXhlcyBlbHNlIE5vbmUKICAgIGZvciBpLCB0cmFuc2xhdGlvbiBpbiBlbnVtZXJhdGUodHJhbnNsYXRpb25zKToKICAgICAgICAjIEdldCB0aGUgdHJhbnNsYXRpb246CiAgICAgICAgdGV4dCA9IHRyYW5zbGF0aW9uWyJ0cmFuc2xhdGlvbl90ZXh0Il0KICAgICAgICAjIFZhbGlkYXRlIGlmIGl0IHdhcyBhbiBlbXB0eSBzZW50ZW5jZSBiZWZvcmU6CiAgICAgICAgaWYgdGV4dCA9PSAiLiI6CiAgICAgICAgICAgIHRleHQgPSAiIgogICAgICAgICMgQ2hlY2sgaWYgbmVlZGVkIHRvIGluc2VydCBhIG5ld2xpbmU6CiAgICAgICAgaWYgbmV3bGluZV9pbmRleCBhbmQgbmV3bGluZV9pbmRleCA9PSBpOgogICAgICAgICAgICB0ZXh0ICs9ICJcbiIKICAgICAgICAgICAgbmV3bGluZV9pbmRleCA9IG5ld2xpbmVzX2luZGV4ZXMucG9wKDApIGlmIG5ld2xpbmVzX2luZGV4ZXMgZWxzZSBOb25lCiAgICAgICAgIyBDb2xsZWN0IGl0OgogICAgICAgIHRyYW5zbGF0ZWRfdGV4dC5hcHBlbmQodGV4dCkKICAgIHRyYW5zbGF0ZWRfdGV4dCA9ICIiLmpvaW4odHJhbnNsYXRlZF90ZXh0KQoKICAgIHJldHVybiB0cmFuc2xhdGVkX3RleHQKCgpkZWYgX3NhdmVfdG9fZmlsZSgKICAgIHRyYW5zbGF0aW9uOiBzdHIsIGZpbGVfbmFtZTogc3RyLCBvdXRwdXRfZGlyZWN0b3J5OiBwYXRobGliLlBhdGgKKSAtPiBwYXRobGliLlBhdGg6CiAgICAjIFByZXBhcmUgdGhlIGZpbGUgZnVsbCBwYXRoIChjaGVja2luZyBmb3Igbm8gZHVwbGljYXRpb25zKToKICAgIHRyYW5zbGF0aW9uX2ZpbGUgPSBvdXRwdXRfZGlyZWN0b3J5IC8gZiJ7ZmlsZV9uYW1lfS50eHQiCiAgICBpID0gMQogICAgd2hpbGUgdHJhbnNsYXRpb25fZmlsZS5leGlzdHMoKToKICAgICAgICBpICs9IDEKICAgICAgICB0cmFuc2xhdGlvbl9maWxlID0gb3V0cHV0X2RpcmVjdG9yeSAvIGYie2ZpbGVfbmFtZX1fe2l9LnR4dCIKCiAgICAjIE1ha2Ugc3VyZSBhbGwgZGlyZWN0b3JpZXMgYXJlIGNyZWF0ZWQ6CiAgICB0cmFuc2xhdGlvbl9maWxlLnBhcmVudC5ta2RpcihleGlzdF9vaz1UcnVlLCBwYXJlbnRzPVRydWUpCgogICAgIyBXcml0ZSB0byBmaWxlOgogICAgd2l0aCBvcGVuKHRyYW5zbGF0aW9uX2ZpbGUsICJ3IikgYXMgZnA6CiAgICAgICAgZnAud3JpdGUodHJhbnNsYXRpb24pCgogICAgcmV0dXJuIHRyYW5zbGF0aW9uX2ZpbGUK + requirements: + - transformers + - sentencepiece + - torch>=2.6 + - tqdm + code_origin: '' + base_image: mlrun/mlrun + filename: translate.py entry_points: open_mpi_handler: - lineno: 56 parameters: - name: worker_inputs - type: List[str] + type: list[str] - name: root_worker_inputs - type: Dict[str, Any] + type: dict[str, Any] default: null + name: open_mpi_handler doc: '' has_kwargs: false has_varargs: false - name: open_mpi_handler + lineno: 56 decorator: - lineno: 68 parameters: - name: handler + name: decorator doc: '' has_kwargs: false has_varargs: false - name: decorator + lineno: 68 wrapper: - lineno: 73 + name: wrapper doc: '' has_kwargs: true has_varargs: false - name: wrapper + lineno: 73 translate: outputs: - doc: 'A tuple of:' - type: Tuple[str, pd.DataFrame, dict] - lineno: 135 + type: tuple[str, pd.DataFrame, dict] parameters: - name: data_path - type: Union[str, List[str], Path] doc: A directory of text files or a single file or a list of files to translate. - name: output_directory type: str @@ -79,6 +94,7 @@ spec: type: bool doc: 'Whether to present logs of a progress bar and errors. Default: True.' default: false + name: translate doc: 'Translate text files using a transformer model from Huggingface''s hub according to the source and target languages @@ -93,24 +109,7 @@ spec: * translation_file - The translation text file name in the output directory.' has_kwargs: false has_varargs: false - name: translate - disable_auto_mount: false - image: '' + lineno: 135 + command: '' + description: Translate text files from one language to another default_handler: translate - build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAyMyBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgIGh0dHA6Ly93d3cuYXBhY2hlLm9yZy9saWNlbnNlcy9MSUNFTlNFLTIuMAojCiMgVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQojIGRpc3RyaWJ1dGVkIHVuZGVyIHRoZSBMaWNlbnNlIGlzIGRpc3RyaWJ1dGVkIG9uIGFuICJBUyBJUyIgQkFTSVMsCiMgV0lUSE9VVCBXQVJSQU5USUVTIE9SIENPTkRJVElPTlMgT0YgQU5ZIEtJTkQsIGVpdGhlciBleHByZXNzIG9yIGltcGxpZWQuCiMgU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAojIGxpbWl0YXRpb25zIHVuZGVyIHRoZSBMaWNlbnNlLgoKaW1wb3J0IGxvZ2dpbmcKaW1wb3J0IG9wZXJhdG9yCmltcG9ydCBwYXRobGliCmZyb20gZnVuY3Rvb2xzIGltcG9ydCByZWR1Y2UsIHdyYXBzCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIFR1cGxlLCBVbmlvbgoKaW1wb3J0IHBhbmRhcyBhcyBwZAppbXBvcnQgdHJhbnNmb3JtZXJzCmZyb20gdHFkbSBpbXBvcnQgdHFkbQoKIyBHZXQgdGhlIGdsb2JhbCBsb2dnZXI6Cl9MT0dHRVIgPSBsb2dnaW5nLmdldExvZ2dlcigpCgoKZGVmIF9jaGVja19tbHJ1bl9hbmRfb3Blbl9tcGkoKSAtPiBUdXBsZVsibWxydW4uTUxDbGllbnRDdHgiLCAibXBpNHB5Lk1QSS5JbnRyYWNvbW0iXToKICAgIGlzX21waSA9IEZhbHNlCiAgICB0cnk6CiAgICAgICAgaW1wb3J0IG1scnVuCgogICAgICAgIGNvbnRleHQgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX2N0eChuYW1lPSJtbHJ1biIpCiAgICAgICAgaXNfbXBpID0gY29udGV4dC5sYWJlbHMuZ2V0KCJraW5kIiwgImpvYiIpID09ICJtcGlqb2IiCgogICAgICAgIGlmIGlzX21waToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnJvbSBtcGk0cHkgaW1wb3J0IE1QSQoKICAgICAgICAgICAgICAgIHJldHVybiBjb250ZXh0LCBNUEkuQ09NTV9XT1JMRAogICAgICAgICAgICBleGNlcHQgTW9kdWxlTm90Rm91bmRFcnJvciBhcyBtcGk0cHlfbm90X2ZvdW5kOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgIlRvIGRpc3RyaWJ1dGUgdGhlIGZ1bmN0aW9uIHVzaW5nIE1MUnVuJ3MgJ21waWpvYicgeW91IG5lZWQgdG8gaGF2ZSBgbXBpNHB5YCBwYWNrYWdlIGluIHlvdXIgIgogICAgICAgICAgICAgICAgICAgICJpbnRlcnByZXRlci4gUGxlYXNlIHJ1biBgcGlwIGluc3RhbGwgbXBpNHB5YCBhbmQgbWFrZSBzdXJlIHlvdSBoYXZlIG9wZW4tbXBpLiIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIG1waTRweV9ub3RfZm91bmQKICAgICAgICBlbHNlOgogICAgICAgICAgICByZXR1cm4gY29udGV4dCwgTm9uZQogICAgZXhjZXB0IE1vZHVsZU5vdEZvdW5kRXJyb3IgYXMgbW9kdWxlX25vdF9mb3VuZDoKICAgICAgICBpZiBpc19tcGk6CiAgICAgICAgICAgIHJhaXNlIG1vZHVsZV9ub3RfZm91bmQKICAgIHJldHVybiBOb25lLCBOb25lCgoKZGVmIG9wZW5fbXBpX2hhbmRsZXIoCiAgICB3b3JrZXJfaW5wdXRzOiBMaXN0W3N0cl0sIHJvb3Rfd29ya2VyX2lucHV0czogRGljdFtzdHIsIEFueV0gPSBOb25lCik6CiAgICBnbG9iYWwgX0xPR0dFUgoKICAgICMgQ2hlY2sgZm9yIE1MUnVuIGFuZCBPcGVuTVBJIGF2YWlsYWJpbGl0eToKICAgIGNvbnRleHQsIGNvbW0gPSBfY2hlY2tfbWxydW5fYW5kX29wZW5fbXBpKCkKCiAgICAjIENoZWNrIGlmIE1MUnVuIGlzIGF2YWlsYWJsZSwgc2V0IHRoZSBnbG9iYWwgbG9nZ2VyIHRvIE1MUnVuJ3M6CiAgICBpZiBjb250ZXh0OgogICAgICAgIF9MT0dHRVIgPSBjb250ZXh0LmxvZ2dlcgoKICAgIGRlZiBkZWNvcmF0b3IoaGFuZGxlcik6CiAgICAgICAgaWYgY29tbSBpcyBOb25lIG9yIGNvbW0uR2V0X3NpemUoKSA9PSAxOgogICAgICAgICAgICByZXR1cm4gaGFuZGxlcgoKICAgICAgICBAd3JhcHMoaGFuZGxlcikKICAgICAgICBkZWYgd3JhcHBlcigqKmt3YXJncyk6CiAgICAgICAgICAgICMgR2V0IHRoZSBvcGVuIG1waSBlbnZpcm9ubWVudCBwcm9wZXJ0aWVzOgogICAgICAgICAgICBzaXplID0gY29tbS5HZXRfc2l6ZSgpCiAgICAgICAgICAgIHJhbmsgPSBjb21tLkdldF9yYW5rKCkKCiAgICAgICAgICAgICMgR2l2ZSB0aGUgY29ycmVjdCBjaHVuayBvZiB0aGUgd29ya2VycyBpbnB1dHM6CiAgICAgICAgICAgIGZvciB3b3JrZXJfaW5wdXQgaW4gd29ya2VyX2lucHV0czoKICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0ga3dhcmdzW3dvcmtlcl9pbnB1dF0KICAgICAgICAgICAgICAgIGlmIGlucHV0X2FyZ3VtZW50IGlzIE5vbmU6CiAgICAgICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgICAgIGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIChzdHIsIHBhdGhsaWIuUGF0aCkpOgogICAgICAgICAgICAgICAgICAgIGlucHV0X2FyZ3VtZW50ID0gX2dldF90ZXh0X2ZpbGVzKAogICAgICAgICAgICAgICAgICAgICAgICBkYXRhX3BhdGg9cGF0aGxpYi5QYXRoKGlucHV0X2FyZ3VtZW50KS5hYnNvbHV0ZSgpCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgaWYgbGVuKGlucHV0X2FyZ3VtZW50KSA8IHNpemU6CiAgICAgICAgICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgICAgICAgICAgICAgZiJDYW5ub3Qgc3BsaXQgdGhlIGlucHV0ICd7d29ya2VyX2lucHV0fScgb2YgbGVuZ3RoIHtsZW4oaW5wdXRfYXJndW1lbnQpfSB0byB7c2l6ZX0gd29ya2Vycy4gIgogICAgICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSByZWR1Y2UgdGhlIGFtb3VudCBvZiB3b3JrZXJzIGZvciB0aGlzIGlucHV0LiIKICAgICAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBldmVuX2NodW5rX3NpemUgPSBsZW4oaW5wdXRfYXJndW1lbnQpIC8vIHNpemUKICAgICAgICAgICAgICAgIGNodW5rX3N0YXJ0ID0gcmFuayAqIGV2ZW5fY2h1bmtfc2l6ZQogICAgICAgICAgICAgICAgY2h1bmtfZW5kID0gKAogICAgICAgICAgICAgICAgICAgIChyYW5rICsgMSkgKiBldmVuX2NodW5rX3NpemUKICAgICAgICAgICAgICAgICAgICBpZiByYW5rICsgMSA8IHNpemUKICAgICAgICAgICAgICAgICAgICBlbHNlIGxlbihpbnB1dF9hcmd1bWVudCkKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSYW5rICN7cmFua306IFByb2Nlc3NpbmcgaW5wdXQgY2h1bmsgb2YgJ3t3b3JrZXJfaW5wdXR9JyAiCiAgICAgICAgICAgICAgICAgICAgZiJmcm9tIGluZGV4IHtjaHVua19zdGFydH0gdG8ge2NodW5rX2VuZH0uIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShpbnB1dF9hcmd1bWVudCwgbGlzdCk6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBpbnB1dF9hcmd1bWVudFtjaHVua19zdGFydDpjaHVua19lbmRdCiAgICAgICAgICAgICAgICBlbGlmIGlzaW5zdGFuY2UoaW5wdXRfYXJndW1lbnQsIHBkLkRhdGFGcmFtZSk6CiAgICAgICAgICAgICAgICAgICAgaW5wdXRfYXJndW1lbnQgPSBpbnB1dF9hcmd1bWVudC5pbG9jW2NodW5rX3N0YXJ0OmNodW5rX2VuZDosIDpdCiAgICAgICAgICAgICAgICBrd2FyZ3Nbd29ya2VyX2lucHV0XSA9IGlucHV0X2FyZ3VtZW50CgogICAgICAgICAgICAjIFNldCB0aGUgcm9vdCB3b3JrZXIgb25seSBhcmd1bWVudHM6CiAgICAgICAgICAgIGlmIHJhbmsgPT0gMCBhbmQgcm9vdF93b3JrZXJfaW5wdXRzOgogICAgICAgICAgICAgICAga3dhcmdzLnVwZGF0ZShyb290X3dvcmtlcl9pbnB1dHMpCgogICAgICAgICAgICAjIFJ1biB0aGUgd29ya2VyOgogICAgICAgICAgICBvdXRwdXQgPSBoYW5kbGVyKCoqa3dhcmdzKQoKICAgICAgICAgICAgIyBTZW5kIHRoZSBvdXRwdXQgdG8gdGhlIHJvb3QgcmFuayAocmFuayAjMCk6CiAgICAgICAgICAgIG91dHB1dCA9IGNvbW0uZ2F0aGVyKG91dHB1dCwgcm9vdD0wKQogICAgICAgICAgICBpZiByYW5rID09IDA6CiAgICAgICAgICAgICAgICAjIEpvaW4gdGhlIG91dHB1dHM6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKCJDb2xsZWN0aW5nIGRhdGEgZnJvbSB3b3JrZXJzIHRvIHJvb3Qgd29ya2VyLiIpCiAgICAgICAgICAgICAgICBvdXRwdXRfZGlyZWN0b3J5ID0gb3V0cHV0WzBdWzBdCiAgICAgICAgICAgICAgICBkYXRhZnJhbWUgPSBwZC5jb25jYXQob2Jqcz1bZGYgZm9yIF8sIGRmLCBfIGluIG91dHB1dF0sIGF4aXM9MCkKICAgICAgICAgICAgICAgIGVycm9yc19kaWN0aW9uYXJ5ID0gcmVkdWNlKAogICAgICAgICAgICAgICAgICAgIG9wZXJhdG9yLmlvciwgW2VyciBmb3IgXywgXywgZXJyIGluIG91dHB1dF0sIHt9CiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByZXR1cm4gb3V0cHV0X2RpcmVjdG9yeSwgZGF0YWZyYW1lLCBlcnJvcnNfZGljdGlvbmFyeQogICAgICAgICAgICByZXR1cm4gTm9uZQoKICAgICAgICByZXR1cm4gd3JhcHBlcgoKICAgIHJldHVybiBkZWNvcmF0b3IKCgpAb3Blbl9tcGlfaGFuZGxlcih3b3JrZXJfaW5wdXRzPVsiZGF0YV9wYXRoIl0sIHJvb3Rfd29ya2VyX2lucHV0cz17InZlcmJvc2UiOiBUcnVlfSkKZGVmIHRyYW5zbGF0ZSgKICAgIGRhdGFfcGF0aDogVW5pb25bc3RyLCBMaXN0W3N0cl0sIHBhdGhsaWIuUGF0aF0sCiAgICBvdXRwdXRfZGlyZWN0b3J5OiBzdHIsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgc291cmNlX2xhbmd1YWdlOiBzdHIgPSBOb25lLAogICAgdGFyZ2V0X2xhbmd1YWdlOiBzdHIgPSBOb25lLAogICAgZGV2aWNlOiBzdHIgPSBOb25lLAogICAgbW9kZWxfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIGJhdGNoX3NpemU6IGludCA9IDEsCiAgICB0cmFuc2xhdGlvbl9rd2FyZ3M6IGRpY3QgPSBOb25lLAogICAgdmVyYm9zZTogYm9vbCA9IEZhbHNlLAopIC0+IFR1cGxlW3N0ciwgcGQuRGF0YUZyYW1lLCBkaWN0XToKICAgICIiIgogICAgVHJhbnNsYXRlIHRleHQgZmlsZXMgdXNpbmcgYSB0cmFuc2Zvcm1lciBtb2RlbCBmcm9tIEh1Z2dpbmdmYWNlJ3MgaHViIGFjY29yZGluZyB0byB0aGUgc291cmNlIGFuZCB0YXJnZXQgbGFuZ3VhZ2VzCiAgICBnaXZlbiAob3IgdXNpbmcgdGhlIGRpcmVjdGx5IHByb3ZpZGVkIG1vZGVsIG5hbWUpLiBUaGUgZW5kIHJlc3VsdCBpcyBhIGRpcmVjdG9yeSBvZiB0cmFuc2xhdGVkIHRleHQgZmlsZXMgYW5kIGEKICAgIGRhdGFmcmFtZSBjb250YWluaW5nIHRoZSBmb2xsb3dpbmcgY29sdW1uczoKCiAgICAqIHRleHRfZmlsZSAtIFRoZSB0ZXh0IGZpbGUgcGF0aC4KICAgICogdHJhbnNsYXRpb25fZmlsZSAtIFRoZSB0cmFuc2xhdGlvbiB0ZXh0IGZpbGUgbmFtZSBpbiB0aGUgb3V0cHV0IGRpcmVjdG9yeS4KCiAgICA6cGFyYW0gZGF0YV9wYXRoOiAgICAgICAgICBBIGRpcmVjdG9yeSBvZiB0ZXh0IGZpbGVzIG9yIGEgc2luZ2xlIGZpbGUgb3IgYSBsaXN0IG9mIGZpbGVzIHRvIHRyYW5zbGF0ZS4KICAgIDpwYXJhbSBvdXRwdXRfZGlyZWN0b3J5OiAgIERpcmVjdG9yeSB3aGVyZSB0aGUgdHJhbnNsYXRlZCBmaWxlcyB3aWxsIGJlIHNhdmVkLgogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgVGhlIG5hbWUgb2YgYSBtb2RlbCB0byBsb2FkLiBJZiBOb25lLCB0aGUgbW9kZWwgbmFtZSBpcyBjb25zdHJ1Y3RlZCB1c2luZyB0aGUgc291cmNlIGFuZAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGFyZ2V0IGxhbmd1YWdlcyBwYXJhbWV0ZXJzLgogICAgOnBhcmFtIHNvdXJjZV9sYW5ndWFnZTogICAgVGhlIHNvdXJjZSBsYW5ndWFnZSBjb2RlIChlLmcuLCAnZW4nIGZvciBFbmdsaXNoKS4KICAgIDpwYXJhbSB0YXJnZXRfbGFuZ3VhZ2U6ICAgIFRoZSB0YXJnZXQgbGFuZ3VhZ2UgY29kZSAoZS5nLiwgJ2VuJyBmb3IgRW5nbGlzaCkuCiAgICA6cGFyYW0gbW9kZWxfa3dhcmdzOiAgICAgICBLZXl3b3JkIGFyZ3VtZW50cyB0byBwYXNzIHJlZ2FyZGluZyB0aGUgbG9hZGluZyBvZiB0aGUgbW9kZWwgaW4gSHVnZ2luZ0ZhY2UncyBgcGlwZWxpbmVgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmdW5jdGlvbi4KICAgIDpwYXJhbSBkZXZpY2U6ICAgICAgICAgICAgIFRoZSBkZXZpY2UgaW5kZXggZm9yIHRyYW5zZm9ybWVycy4gRGVmYXVsdCB3aWxsIHByZWZlciBjdWRhIGlmIGF2YWlsYWJsZS4KICAgIDpwYXJhbSBiYXRjaF9zaXplOiAgICAgICAgIFRoZSBudW1iZXIgb2YgYmF0Y2hlcyB0byB1c2UgaW4gdHJhbnNsYXRpb24uIFRoZSBmaWxlcyBhcmUgdHJhbnNsYXRlZCBvbmUgYnkgb25lLCBidXQgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzZW50ZW5jZXMgY2FuIGJlIGJhdGNoZWQuCiAgICA6cGFyYW0gdHJhbnNsYXRpb25fa3dhcmdzOiBBZGRpdGlvbmFsIGtleXdvcmQgYXJndW1lbnRzIHRvIHBhc3MgdG8gYSBgdHJhbnNmb3JtZXJzLlRyYW5zbGF0aW9uUGlwZWxpbmVgIHdoZW4gZG9pbmcKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoZSB0cmFuc2xhdGlvbiBpbmZlcmVuY2UuIE5vdGljZSB0aGUgYmF0Y2ggc2l6ZSBoZXJlIGlzIGJlaW5nIGFkZGVkIGF1dG9tYXRpY2FsbHkuCiAgICA6cGFyYW0gdmVyYm9zZTogICAgICAgICAgICBXaGV0aGVyIHRvIHByZXNlbnQgbG9ncyBvZiBhIHByb2dyZXNzIGJhciBhbmQgZXJyb3JzLiBEZWZhdWx0OiBUcnVlLgoKICAgIDpyZXR1cm5zOiBBIHR1cGxlIG9mOgoKICAgICAgICAgICAgICAqIFBhdGggdG8gdGhlIG91dHB1dCBkaXJlY3RvcnkuCiAgICAgICAgICAgICAgKiBBIGRhdGFmcmFtZSBkYXRhc2V0IG9mIHRoZSB0cmFuc2xhdGVkIGZpbGUgbmFtZXMuCiAgICAgICAgICAgICAgKiBBIGRpY3Rpb25hcnkgb2YgZXJyb3JlZCBmaWxlcyB0aGF0IHdlcmUgbm90IHRyYW5zbGF0ZWQuCiAgICAiIiIKICAgIGdsb2JhbCBfTE9HR0VSCgogICAgIyBHZXQgdGhlIGlucHV0IHRleHQgZmlsZXMgdG8gdHJhbnNsYXRlOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oIkNvbGxlY3RpbmcgdGV4dCBmaWxlcy4iKQogICAgaWYgaXNpbnN0YW5jZShkYXRhX3BhdGgsIHN0cik6CiAgICAgICAgZGF0YV9wYXRoID0gcGF0aGxpYi5QYXRoKGRhdGFfcGF0aCkuYWJzb2x1dGUoKQogICAgICAgIHRleHRfZmlsZXMgPSBfZ2V0X3RleHRfZmlsZXMoZGF0YV9wYXRoPWRhdGFfcGF0aCkKICAgIGVsc2U6CiAgICAgICAgdGV4dF9maWxlcyA9IGRhdGFfcGF0aAogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oZiJDb2xsZWN0ZWQge2xlbih0ZXh0X2ZpbGVzKX0gdGV4dCBmaWxlcy4iKQoKICAgICMgR2V0IHRoZSB0cmFuc2xhdGlvbiBwaXBlbGluZToKICAgIGlmIHZlcmJvc2U6CiAgICAgICAgX0xPR0dFUi5pbmZvKGYiTG9hZGluZyBtb2RlbCAtIHVzaW5nIGRldmljZSAne2RldmljZX0nLiIpCiAgICB0cmFuc2xhdGlvbl9waXBlbGluZSwgbW9kZWxfbmFtZSA9IF9nZXRfdHJhbnNsYXRpb25fcGlwZWxpbmUoCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIHNvdXJjZV9sYW5ndWFnZT1zb3VyY2VfbGFuZ3VhZ2UsCiAgICAgICAgdGFyZ2V0X2xhbmd1YWdlPXRhcmdldF9sYW5ndWFnZSwKICAgICAgICBkZXZpY2U9ZGV2aWNlLAogICAgICAgIG1vZGVsX2t3YXJncz1tb2RlbF9rd2FyZ3MsCiAgICAgICAgYmF0Y2hfc2l6ZT1iYXRjaF9zaXplIGlmIGJhdGNoX3NpemUgIT0gMSBlbHNlIE5vbmUsCiAgICApCiAgICBpZiB2ZXJib3NlOgogICAgICAgIF9MT0dHRVIuaW5mbyhmIk1vZGVsICd7bW9kZWxfbmFtZX0nIHdhcyBsb2FkZWQgc3VjY2Vzc2Z1bGx5LiIpCgogICAgIyBQcmVwYXJlIHRoZSBzdWNjZXNzZXMgZGF0YWZyYW1lIGFuZCBlcnJvcnMgZGljdGlvbmFyeSB0byBiZSByZXR1cm5lZDoKICAgIHN1Y2Nlc3NlcyA9IFtdCiAgICBlcnJvcnMgPSB7fQoKICAgICMgQ3JlYXRlIHRoZSBvdXRwdXQgZGlyZWN0b3J5OgogICAgb3V0cHV0X2RpcmVjdG9yeSA9IHBhdGhsaWIuUGF0aChvdXRwdXRfZGlyZWN0b3J5KQogICAgb3V0cHV0X2RpcmVjdG9yeS5ta2RpcihwYXJlbnRzPVRydWUsIGV4aXN0X29rPVRydWUpCgogICAgIyBQcmVwYXJlIHRoZSB0cmFuc2xhdGlvbiBrZXl3b3JkIGFyZ3VtZW50czoKICAgIHRyYW5zbGF0aW9uX2t3YXJncyA9IHRyYW5zbGF0aW9uX2t3YXJncyBvciB7fQoKICAgICMgR28gb3ZlciB0aGUgYXVkaW8gZmlsZXMgYW5kIHRyYW5zY3JpYmU6CiAgICBmb3IgdGV4dF9maWxlIGluIHRxZG0oCiAgICAgICAgdGV4dF9maWxlcywgZGVzYz0iVHJhbnNsYXRpbmciLCB1bml0PSJmaWxlIiwgZGlzYWJsZT1ub3QgdmVyYm9zZQogICAgKToKICAgICAgICB0cnk6CiAgICAgICAgICAgICMgVHJhbnNsYXRlOgogICAgICAgICAgICB0cmFuc2xhdGlvbiA9IF90cmFuc2xhdGUoCiAgICAgICAgICAgICAgICB0ZXh0X2ZpbGU9dGV4dF9maWxlLAogICAgICAgICAgICAgICAgdHJhbnNsYXRpb25fcGlwZWxpbmU9dHJhbnNsYXRpb25fcGlwZWxpbmUsCiAgICAgICAgICAgICAgICB0cmFuc2xhdGlvbl9rd2FyZ3M9dHJhbnNsYXRpb25fa3dhcmdzLAogICAgICAgICAgICApCiAgICAgICAgICAgICMgV3JpdGUgdGhlIHRyYW5zY3JpcHRpb24gdG8gZmlsZToKICAgICAgICAgICAgdHJhbnNsYXRpb25fZmlsZSA9IF9zYXZlX3RvX2ZpbGUoCiAgICAgICAgICAgICAgICB0cmFuc2xhdGlvbj10cmFuc2xhdGlvbiwKICAgICAgICAgICAgICAgIGZpbGVfbmFtZT10ZXh0X2ZpbGUuc3RlbSwKICAgICAgICAgICAgICAgIG91dHB1dF9kaXJlY3Rvcnk9b3V0cHV0X2RpcmVjdG9yeSwKICAgICAgICAgICAgKQogICAgICAgICAgICAjIE5vdGUgYXMgYSBzdWNjZXNzIGluIHRoZSBsaXN0OgogICAgICAgICAgICBzdWNjZXNzZXMuYXBwZW5kKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIHRleHRfZmlsZS5uYW1lLAogICAgICAgICAgICAgICAgICAgIHRyYW5zbGF0aW9uX2ZpbGUubmFtZSwKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgICAgIGV4Y2VwdCBFeGNlcHRpb24gYXMgZXhjZXB0aW9uOgogICAgICAgICAgICAjIE5vdGUgdGhlIGV4Y2VwdGlvbiBhcyBlcnJvciBpbiB0aGUgZGljdGlvbmFyeToKICAgICAgICAgICAgaWYgdmVyYm9zZToKICAgICAgICAgICAgICAgIF9MT0dHRVIud2FybmluZyhmIkVycm9yIGluIGZpbGU6ICd7dGV4dF9maWxlLm5hbWV9JyIpCiAgICAgICAgICAgIGVycm9yc1tzdHIodGV4dF9maWxlLm5hbWUpXSA9IHN0cihleGNlcHRpb24pCiAgICAgICAgICAgIGNvbnRpbnVlCgogICAgIyBDb25zdHJ1Y3QgdGhlIHRyYW5zbGF0aW9ucyBkYXRhZnJhbWU6CiAgICBjb2x1bW5zID0gWwogICAgICAgICJ0ZXh0X2ZpbGUiLAogICAgICAgICJ0cmFuc2xhdGlvbl9maWxlIiwKICAgIF0KICAgIHN1Y2Nlc3NlcyA9IHBkLkRhdGFGcmFtZSgKICAgICAgICBzdWNjZXNzZXMsCiAgICAgICAgY29sdW1ucz1jb2x1bW5zLAogICAgKQoKICAgICMgUHJpbnQgdGhlIGhlYWQgb2YgdGhlIHByb2R1Y2VkIGRhdGFmcmFtZSBhbmQgcmV0dXJuOgogICAgaWYgdmVyYm9zZToKICAgICAgICBfTE9HR0VSLmluZm8oCiAgICAgICAgICAgIGYiRG9uZSAoe3N1Y2Nlc3Nlcy5zaGFwZVswXX0ve2xlbih0ZXh0X2ZpbGVzKX0pXG4iCiAgICAgICAgICAgIGYiVHJhbnNsYXRpb25zIHN1bW1hcnk6XG4iCiAgICAgICAgICAgIGYie3N1Y2Nlc3Nlcy5oZWFkKCl9IgogICAgICAgICkKICAgIHJldHVybiBzdHIob3V0cHV0X2RpcmVjdG9yeSksIHN1Y2Nlc3NlcywgZXJyb3JzCgoKZGVmIF9nZXRfdGV4dF9maWxlcygKICAgIGRhdGFfcGF0aDogcGF0aGxpYi5QYXRoLAopIC0+IExpc3RbcGF0aGxpYi5QYXRoXToKICAgICMgQ2hlY2sgaWYgdGhlIHBhdGggaXMgb2YgYSBkaXJlY3Rvcnkgb3IgYSBmaWxlOgogICAgaWYgZGF0YV9wYXRoLmlzX2RpcigpOgogICAgICAgICMgR2V0IGFsbCBmaWxlcyBpbnNpZGUgdGhlIGRpcmVjdG9yeToKICAgICAgICB0ZXh0X2ZpbGVzID0gbGlzdChkYXRhX3BhdGguZ2xvYigiKi4qIikpCiAgICBlbGlmIGRhdGFfcGF0aC5pc19maWxlKCk6CiAgICAgICAgdGV4dF9maWxlcyA9IFtkYXRhX3BhdGhdCiAgICBlbHNlOgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoCiAgICAgICAgICAgIGYiVW5yZWNvZ25pemVkIGRhdGEgcGF0aC4gVGhlIHBhcmFtZXRlciBgZGF0YV9wYXRoYCBtdXN0IGJlIGVpdGhlciBhIGRpcmVjdG9yeSBwYXRoIG9yIGEgZmlsZSBwYXRoLiAiCiAgICAgICAgICAgIGYiR2l2ZW46IHtzdHIoZGF0YV9wYXRoKX0gIgogICAgICAgICkKCiAgICByZXR1cm4gdGV4dF9maWxlcwoKCmRlZiBfZ2V0X3RyYW5zbGF0aW9uX3BpcGVsaW5lKAogICAgbW9kZWxfbmFtZTogc3RyID0gTm9uZSwKICAgIHNvdXJjZV9sYW5ndWFnZTogc3RyID0gTm9uZSwKICAgIHRhcmdldF9sYW5ndWFnZTogc3RyID0gTm9uZSwKICAgIGRldmljZTogc3RyID0gTm9uZSwKICAgIG1vZGVsX2t3YXJnczogZGljdCA9IE5vbmUsCiAgICBiYXRjaF9zaXplOiBpbnQgPSBOb25lLAopIC0+IFR1cGxlW3RyYW5zZm9ybWVycy5QaXBlbGluZSwgc3RyXToKICAgICMgQ29uc3RydWN0IHRoZSBtb2RlbCBuYW1lIC0gaWYgbW9kZWwgbmFtZSBpcyBwcm92aWRlZCAobm90IE5vbmUpIHRoZW4gd2UgdGFrZSBpdCwgb3RoZXJ3aXNlIHdlIGNoZWNrIGJvdGggc291cmNlCiAgICAjIGFuZCB0YXJnZXQgd2VyZSBwcm92aWRlZCB0byBjb25zdHJ1Y3QgdGhlIG1vZGVsIG5hbWU6CiAgICBpZiBtb2RlbF9uYW1lIGlzIE5vbmUgYW5kIChzb3VyY2VfbGFuZ3VhZ2UgaXMgTm9uZSBvciB0YXJnZXRfbGFuZ3VhZ2UgaXMgTm9uZSk6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigKICAgICAgICAgICAgIk5vIG1vZGVsIG5hbWUgd2VyZSBnaXZlbiBhbmQgbWlzc2luZyBzb3VyY2UgYW5kIC8gb3IgdGFyZ2V0IGxhbmd1YWdlcy4gSW4gb3JkZXIgdG8gdHJhbnNsYXRlIHlvdSBtdXN0ICIKICAgICAgICAgICAgInBhc3MgYSBgbW9kZWxfbmFtZWAgb3IgYm90aCBgc291cmNlX2xhbmd1YWdlYCBhbmQgYHRhcmdldF9sYW5ndWFnZWAuIgogICAgICAgICkKICAgIGVsaWYgbW9kZWxfbmFtZSBpcyBOb25lOgogICAgICAgIG1vZGVsX25hbWUgPSBmIkhlbHNpbmtpLU5MUC9vcHVzLW10LXtzb3VyY2VfbGFuZ3VhZ2V9LXt0YXJnZXRfbGFuZ3VhZ2V9IgoKICAgICMgSW5pdGlhbGl6ZSB0aGUgdHJhbnNsYXRpb24gcGlwZWxpbmU6CiAgICB0cnk6CiAgICAgICAgdHJhbnNsYXRpb25fcGlwZWxpbmUgPSB0cmFuc2Zvcm1lcnMucGlwZWxpbmUoCiAgICAgICAgICAgIHRhc2s9InRyYW5zbGF0aW9uIiwKICAgICAgICAgICAgbW9kZWw9bW9kZWxfbmFtZSwKICAgICAgICAgICAgdG9rZW5pemVyPW1vZGVsX25hbWUsCiAgICAgICAgICAgIGRldmljZT1kZXZpY2UsCiAgICAgICAgICAgIG1vZGVsX2t3YXJncz1tb2RlbF9rd2FyZ3MsCiAgICAgICAgICAgIGJhdGNoX3NpemU9YmF0Y2hfc2l6ZSwKICAgICAgICApCiAgICBleGNlcHQgT1NFcnJvciBhcyBsb2FkX2V4Y2VwdGlvbjoKICAgICAgICBpZiAoCiAgICAgICAgICAgICJpcyBub3QgYSB2YWxpZCBtb2RlbCBpZGVudGlmaWVyIGxpc3RlZCBvbiAnaHR0cHM6Ly9odWdnaW5nZmFjZS5jby9tb2RlbHMnIgogICAgICAgICAgICBpbiBzdHIobG9hZF9leGNlcHRpb24pCiAgICAgICAgICAgIGFuZCBzb3VyY2VfbGFuZ3VhZ2UKICAgICAgICApOgogICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yKAogICAgICAgICAgICAgICAgZiJUaGUgbW9kZWwgJ3ttb2RlbF9uYW1lfScgaXMgbm90IGEgdmFsaWQgbW9kZWwgaWRlbnRpZmllci4gIgogICAgICAgICAgICAgICAgZiJUaGUgcGFyYW1ldGVycyBgc291cmNlX2xhbmd1YWdlYCBhbmQgYHRhcmdldF9sYW5ndWFnZWAgYXJlIHVzZWQgdG8gY29uc3RydWN0IGEgSGVsc2lua2kgbW9kZWwgZm9yICIKICAgICAgICAgICAgICAgIGYidGV4dCB0byB0ZXh0IGdlbmVyYXRpb24sIGJ1dCB0aGUgbW9kZWwgY3JlYXRlZCBmcm9tIHRoZSBnaXZlbiBsYW5ndWFnZXMgZG9lcyBub3QgZXhpc3QuICIKICAgICAgICAgICAgICAgIGYiWW91IG1heSBjaGVjayBsYW5ndWFnZSBpZGVudGlmaWVycyBhdCAiCiAgICAgICAgICAgICAgICBmImh0dHBzOi8vZGV2ZWxvcGVycy5nb29nbGUuY29tL2FkbWluLXNkay9kaXJlY3RvcnkvdjEvbGFuZ3VhZ2VzLCBhbmQgaWYgdGhlIGVycm9yIHdhcyBub3QgZml4ZWQsIG9uZSAiCiAgICAgICAgICAgICAgICBmIm9yIG1vcmUgbGFuZ3VhZ2UgY29kZSBtaWdodCBiZSB3aXRoIDMgbGV0dGVycyBhbmQgbmVlZHMgdG8gYmUgZm91bmQgb25saW5lLiAiCiAgICAgICAgICAgICAgICBmIlJlbWVtYmVyLCB5b3UgY2FuIGFsd2F5cyBjaG9vc2UgYSBtb2RlbCBkaXJlY3RseSBmcm9tIHRoZSBIdWdnaW5nZmFjZSBodWIgYnkgdXNpbmcgdGhlIGBtb2RlbF9uYW1lYCAiCiAgICAgICAgICAgICAgICBmInBhcmFtZXRlci4iCiAgICAgICAgICAgICkgZnJvbSBsb2FkX2V4Y2VwdGlvbgogICAgICAgIHJhaXNlIGxvYWRfZXhjZXB0aW9uCgogICAgcmV0dXJuIHRyYW5zbGF0aW9uX3BpcGVsaW5lLCBtb2RlbF9uYW1lCgoKZGVmIF90cmFuc2xhdGUoCiAgICB0ZXh0X2ZpbGU6IHBhdGhsaWIuUGF0aCwKICAgIHRyYW5zbGF0aW9uX3BpcGVsaW5lOiB0cmFuc2Zvcm1lcnMuUGlwZWxpbmUsCiAgICB0cmFuc2xhdGlvbl9rd2FyZ3M6IGRpY3QsCikgLT4gc3RyOgogICAgIyBSZWFkIHRoZSB0ZXh0IGZyb20gZmlsZToKICAgIHdpdGggb3Blbih0ZXh0X2ZpbGUsICJyIikgYXMgZnA6CiAgICAgICAgdGV4dCA9IGZwLnJlYWQoKQoKICAgICMgU3BsaXQgdG8gcGFyYWdyYXBocyBhbmQgZWFjaCBwYXJhZ3JhcGggdG8gc2VudGVuY2VzOgogICAgcGFyYWdyYXBocyA9IFtwYXJhZ3JhcGguc3BsaXQoIi4iKSBmb3IgcGFyYWdyYXBoIGluIHRleHQuc3BsaXQoIlxuIildCgogICAgIyBEaXNjb3ZlciB0aGUgbmV3bGluZSBpbmRleGVzIHRvIHJlc3RvcmUgdGhlIGZpbGUgdG8gaXRzIHN0cnVjdHVyZSBwb3N0IHRyYW5zbGF0aW9uOgogICAgbmV3bGluZXNfaW5kZXhlcyA9IFtdCiAgICBmb3IgcGFyYWdyYXBoIGluIHBhcmFncmFwaHNbOi0xXToKICAgICAgICBpZiBsZW4obmV3bGluZXNfaW5kZXhlcykgPT0gMDoKICAgICAgICAgICAgbmV3bGluZXNfaW5kZXhlcy5hcHBlbmQobGVuKHBhcmFncmFwaCkgLSAxKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIG5ld2xpbmVzX2luZGV4ZXMuYXBwZW5kKG5ld2xpbmVzX2luZGV4ZXNbLTFdICsgbGVuKHBhcmFncmFwaCkpCgogICAgIyBQcmVwYXJlIHRoZSBiYXRjaGVzIChlYWNoIHNlbnRlbmNlIGZyb20gdGhlIHBhcmFncmFwaHMpLiBOb3RpY2Ugd2UgYWRkIGEgZG90IG5vdCBvbmx5IHRvIHJlc3RvcmUgdGhlIHNlbnRlbmNlCiAgICAjIHN0cnVjdHVyZSBidXQgdG8gaWdub3JlIGVtcHR5IHN0cmluZ3MgYXMgaXQgd2lsbCBydWluIHRoZSB0cmFuc2xhdGlvbjoKICAgIHNlbnRlbmNlcyA9IFtmIntsaW5lfS4iIGZvciBwYXJhZ3JhcGggaW4gcGFyYWdyYXBocyBmb3IgbGluZSBpbiBwYXJhZ3JhcGhdCgogICAgIyBUcmFuc2xhdGUgdGhlIHNlbnRlbmNlczoKICAgIHRyYW5zbGF0aW9ucyA9IHRyYW5zbGF0aW9uX3BpcGVsaW5lKHNlbnRlbmNlcywgKip0cmFuc2xhdGlvbl9rd2FyZ3MpCgogICAgIyBSZXN0cnVjdHVyZSB0aGUgZnVsbCB0ZXh0IGZyb20gdGhlIHNlbnRlbmNlczoKICAgIHRyYW5zbGF0ZWRfdGV4dCA9IFtdCiAgICBuZXdsaW5lX2luZGV4ID0gbmV3bGluZXNfaW5kZXhlcy5wb3AoMCkgaWYgbmV3bGluZXNfaW5kZXhlcyBlbHNlIE5vbmUKICAgIGZvciBpLCB0cmFuc2xhdGlvbiBpbiBlbnVtZXJhdGUodHJhbnNsYXRpb25zKToKICAgICAgICAjIEdldCB0aGUgdHJhbnNsYXRpb246CiAgICAgICAgdGV4dCA9IHRyYW5zbGF0aW9uWyJ0cmFuc2xhdGlvbl90ZXh0Il0KICAgICAgICAjIFZhbGlkYXRlIGlmIGl0IHdhcyBhbiBlbXB0eSBzZW50ZW5jZSBiZWZvcmU6CiAgICAgICAgaWYgdGV4dCA9PSAiLiI6CiAgICAgICAgICAgIHRleHQgPSAiIgogICAgICAgICMgQ2hlY2sgaWYgbmVlZGVkIHRvIGluc2VydCBhIG5ld2xpbmU6CiAgICAgICAgaWYgbmV3bGluZV9pbmRleCBhbmQgbmV3bGluZV9pbmRleCA9PSBpOgogICAgICAgICAgICB0ZXh0ICs9ICJcbiIKICAgICAgICAgICAgbmV3bGluZV9pbmRleCA9IG5ld2xpbmVzX2luZGV4ZXMucG9wKDApIGlmIG5ld2xpbmVzX2luZGV4ZXMgZWxzZSBOb25lCiAgICAgICAgIyBDb2xsZWN0IGl0OgogICAgICAgIHRyYW5zbGF0ZWRfdGV4dC5hcHBlbmQodGV4dCkKICAgIHRyYW5zbGF0ZWRfdGV4dCA9ICIiLmpvaW4odHJhbnNsYXRlZF90ZXh0KQoKICAgIHJldHVybiB0cmFuc2xhdGVkX3RleHQKCgpkZWYgX3NhdmVfdG9fZmlsZSgKICAgIHRyYW5zbGF0aW9uOiBzdHIsIGZpbGVfbmFtZTogc3RyLCBvdXRwdXRfZGlyZWN0b3J5OiBwYXRobGliLlBhdGgKKSAtPiBwYXRobGliLlBhdGg6CiAgICAjIFByZXBhcmUgdGhlIGZpbGUgZnVsbCBwYXRoIChjaGVja2luZyBmb3Igbm8gZHVwbGljYXRpb25zKToKICAgIHRyYW5zbGF0aW9uX2ZpbGUgPSBvdXRwdXRfZGlyZWN0b3J5IC8gZiJ7ZmlsZV9uYW1lfS50eHQiCiAgICBpID0gMQogICAgd2hpbGUgdHJhbnNsYXRpb25fZmlsZS5leGlzdHMoKToKICAgICAgICBpICs9IDEKICAgICAgICB0cmFuc2xhdGlvbl9maWxlID0gb3V0cHV0X2RpcmVjdG9yeSAvIGYie2ZpbGVfbmFtZX1fe2l9LnR4dCIKCiAgICAjIE1ha2Ugc3VyZSBhbGwgZGlyZWN0b3JpZXMgYXJlIGNyZWF0ZWQ6CiAgICB0cmFuc2xhdGlvbl9maWxlLnBhcmVudC5ta2RpcihleGlzdF9vaz1UcnVlLCBwYXJlbnRzPVRydWUpCgogICAgIyBXcml0ZSB0byBmaWxlOgogICAgd2l0aCBvcGVuKHRyYW5zbGF0aW9uX2ZpbGUsICJ3IikgYXMgZnA6CiAgICAgICAgZnAud3JpdGUodHJhbnNsYXRpb24pCgogICAgcmV0dXJuIHRyYW5zbGF0aW9uX2ZpbGUK - origin_filename: '' - base_image: mlrun/mlrun - requirements: - - transformers - - sentencepiece - - torch>=2.6 - - tqdm - code_origin: '' -kind: job -metadata: - tag: '' - categories: - - genai - - NLP - name: translate diff --git a/functions/src/translate/item.yaml b/functions/src/translate/item.yaml index 68f176ac2..24424748b 100644 --- a/functions/src/translate/item.yaml +++ b/functions/src/translate/item.yaml @@ -12,7 +12,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.10.0-rc41 +mlrunVersion: 1.10.0 name: translate platformVersion: 3.5.3 spec: diff --git a/functions/src/translate/test_translate.py b/functions/src/translate/test_translate.py index a22dc899a..e56572546 100644 --- a/functions/src/translate/test_translate.py +++ b/functions/src/translate/test_translate.py @@ -19,7 +19,9 @@ def test_translate(): project = mlrun.new_project("test-translate") - translate_fn = project.set_function("translate.py", "translate", image="mlrun/mlrun") + translate_fn = project.set_function( + "translate.py", "translate", image="mlrun/mlrun" + ) input_text = "Ali her gece bir kitap okur." expected_translation = "Ali reads a book every night." @@ -48,4 +50,3 @@ def test_translate(): assert translate_run.status.state == "completed" with open(os.path.join(test_dir, "test_tr.txt")) as f: assert f.read() == expected_translation - diff --git a/functions/src/translate/translate.py b/functions/src/translate/translate.py index 360fa6203..a5e05f2d2 100644 --- a/functions/src/translate/translate.py +++ b/functions/src/translate/translate.py @@ -16,7 +16,7 @@ import operator import pathlib from functools import reduce, wraps -from typing import Any, Dict, List, Tuple, Union +from typing import Any import pandas as pd import transformers @@ -26,7 +26,7 @@ _LOGGER = logging.getLogger() -def _check_mlrun_and_open_mpi() -> Tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intracomm"]: +def _check_mlrun_and_open_mpi() -> tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intracomm"]: is_mpi = False try: import mlrun @@ -54,7 +54,7 @@ def _check_mlrun_and_open_mpi() -> Tuple["mlrun.MLClientCtx", "mpi4py.MPI.Intrac def open_mpi_handler( - worker_inputs: List[str], root_worker_inputs: Dict[str, Any] = None + worker_inputs: list[str], root_worker_inputs: dict[str, Any] = None ): global _LOGGER @@ -133,7 +133,7 @@ def wrapper(**kwargs): @open_mpi_handler(worker_inputs=["data_path"], root_worker_inputs={"verbose": True}) def translate( - data_path: Union[str, List[str], pathlib.Path], + data_path: str | list[str] | pathlib.Path, output_directory: str, model_name: str = None, source_language: str = None, @@ -143,7 +143,7 @@ def translate( batch_size: int = 1, translation_kwargs: dict = None, verbose: bool = False, -) -> Tuple[str, pd.DataFrame, dict]: +) -> tuple[str, pd.DataFrame, dict]: """ Translate text files using a transformer model from Huggingface's hub according to the source and target languages given (or using the directly provided model name). The end result is a directory of translated text files and a @@ -264,7 +264,7 @@ def translate( def _get_text_files( data_path: pathlib.Path, -) -> List[pathlib.Path]: +) -> list[pathlib.Path]: # Check if the path is of a directory or a file: if data_path.is_dir(): # Get all files inside the directory: @@ -287,7 +287,7 @@ def _get_translation_pipeline( device: str = None, model_kwargs: dict = None, batch_size: int = None, -) -> Tuple[transformers.Pipeline, str]: +) -> tuple[transformers.Pipeline, str]: # Construct the model name - if model name is provided (not None) then we take it, otherwise we check both source # and target were provided to construct the model name: if model_name is None and (source_language is None or target_language is None): @@ -335,7 +335,7 @@ def _translate( translation_kwargs: dict, ) -> str: # Read the text from file: - with open(text_file, "r") as fp: + with open(text_file) as fp: text = fp.read() # Split to paragraphs and each paragraph to sentences: diff --git a/functions/src/v2_model_server/function.yaml b/functions/src/v2_model_server/function.yaml index 5ecfec9ba..4a2b6dd81 100644 --- a/functions/src/v2_model_server/function.yaml +++ b/functions/src/v2_model_server/function.yaml @@ -1,87 +1,29 @@ -kind: serving metadata: - name: v2-model-server tag: '' - hash: ad85919d3b9cf2acae43a3434ba56e01b005755e - project: '' - labels: - author: Iguazio - framework: sklearn + name: v2-model-server categories: - model-serving - machine-learning +verbose: false +kind: serving spec: - command: '' - args: [] image: mlrun/mlrun - entry_points: - load: - name: load - doc: load and initialize the model and/or other elements - parameters: - - name: self - default: '' - outputs: - - default: '' - lineno: 16 - predict: - name: predict - doc: Generate model predictions from sample. - parameters: - - name: self - default: '' - - name: body - type: dict - default: '' - outputs: - - default: '' - type: List - lineno: 21 - init_context: - name: init_context - doc: '' - parameters: - - name: context - default: '' - outputs: - - default: '' - lineno: 39 - handler: - name: handler - doc: '' - parameters: - - name: context - default: '' - - name: event - default: '' - outputs: - - default: '' - lineno: 42 - description: generic sklearn model server + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgppbXBvcnQgbWxydW4KaW1wb3J0IG51bXB5IGFzIG5wCmZyb20gY2xvdWRwaWNrbGUgaW1wb3J0IGxvYWQKCndhcm5pbmdzLmZpbHRlcndhcm5pbmdzKCJpZ25vcmUiKQoKCmNsYXNzIENsYXNzaWZpZXJNb2RlbChtbHJ1bi5zZXJ2aW5nLlYyTW9kZWxTZXJ2ZXIpOgogICAgZGVmIGxvYWQoc2VsZik6CiAgICAgICAgIiIibG9hZCBhbmQgaW5pdGlhbGl6ZSB0aGUgbW9kZWwgYW5kL29yIG90aGVyIGVsZW1lbnRzIiIiCiAgICAgICAgbW9kZWxfZmlsZSwgZXh0cmFfZGF0YSA9IHNlbGYuZ2V0X21vZGVsKCIucGtsIikKICAgICAgICBzZWxmLm1vZGVsID0gbG9hZChvcGVuKG1vZGVsX2ZpbGUsICJyYiIpKQoKICAgIGRlZiBwcmVkaWN0KHNlbGYsIGJvZHk6IGRpY3QpIC0+IGxpc3Q6CiAgICAgICAgIiIiR2VuZXJhdGUgbW9kZWwgcHJlZGljdGlvbnMgZnJvbSBzYW1wbGUuIiIiCiAgICAgICAgZmVhdHMgPSBucC5hc2FycmF5KGJvZHlbImlucHV0cyJdKQogICAgICAgIHJlc3VsdDogbnAubmRhcnJheSA9IHNlbGYubW9kZWwucHJlZGljdChmZWF0cykKICAgICAgICByZXR1cm4gcmVzdWx0LnRvbGlzdCgpCgpmcm9tIG1scnVuLnJ1bnRpbWVzIGltcG9ydCBudWNsaW9faW5pdF9ob29rCmRlZiBpbml0X2NvbnRleHQoY29udGV4dCk6CiAgICBudWNsaW9faW5pdF9ob29rKGNvbnRleHQsIGdsb2JhbHMoKSwgJ3NlcnZpbmdfdjInKQoKZGVmIGhhbmRsZXIoY29udGV4dCwgZXZlbnQpOgogICAgcmV0dXJuIGNvbnRleHQubWxydW5faGFuZGxlcihjb250ZXh0LCBldmVudCkK + code_origin: '' + filename: v2_model_server.py + default_class: ClassifierModel min_replicas: 1 - max_replicas: 4 - env: [] - base_spec: - apiVersion: nuclio.io/v1 - kind: Function - metadata: - name: v2-model-server - labels: {} - annotations: - nuclio.io/generated_by: function generated from /home/michaell/projects/functions/v2_model_server/v2_model_server.py - spec: - runtime: python:3.9 - handler: v2_model_server:handler - env: [] - volumes: [] - build: - commands: [] - noBaseImagesPull: true - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IG1scnVuCgpmcm9tIGNsb3VkcGlja2xlIGltcG9ydCBsb2FkCmZyb20gdHlwaW5nIGltcG9ydCBMaXN0CmZyb20gc2tsZWFybi5kYXRhc2V0cyBpbXBvcnQgbG9hZF9pcmlzCmltcG9ydCBudW1weSBhcyBucAoKaW1wb3J0IHdhcm5pbmdzCgp3YXJuaW5ncy5maWx0ZXJ3YXJuaW5ncygiaWdub3JlIikKCgpjbGFzcyBDbGFzc2lmaWVyTW9kZWwobWxydW4uc2VydmluZy5WMk1vZGVsU2VydmVyKToKICAgIGRlZiBsb2FkKHNlbGYpOgogICAgICAgICIiImxvYWQgYW5kIGluaXRpYWxpemUgdGhlIG1vZGVsIGFuZC9vciBvdGhlciBlbGVtZW50cyIiIgogICAgICAgIG1vZGVsX2ZpbGUsIGV4dHJhX2RhdGEgPSBzZWxmLmdldF9tb2RlbCgiLnBrbCIpCiAgICAgICAgc2VsZi5tb2RlbCA9IGxvYWQob3Blbihtb2RlbF9maWxlLCAicmIiKSkKCiAgICBkZWYgcHJlZGljdChzZWxmLCBib2R5OiBkaWN0KSAtPiBMaXN0OgogICAgICAgICIiIkdlbmVyYXRlIG1vZGVsIHByZWRpY3Rpb25zIGZyb20gc2FtcGxlLiIiIgogICAgICAgIGZlYXRzID0gbnAuYXNhcnJheShib2R5WyJpbnB1dHMiXSkKICAgICAgICByZXN1bHQ6IG5wLm5kYXJyYXkgPSBzZWxmLm1vZGVsLnByZWRpY3QoZmVhdHMpCiAgICAgICAgcmV0dXJuIHJlc3VsdC50b2xpc3QoKQpmcm9tIG1scnVuLnJ1bnRpbWVzIGltcG9ydCBudWNsaW9faW5pdF9ob29rCmRlZiBpbml0X2NvbnRleHQoY29udGV4dCk6CiAgICBudWNsaW9faW5pdF9ob29rKGNvbnRleHQsIGdsb2JhbHMoKSwgJ3NlcnZpbmdfdjInKQoKZGVmIGhhbmRsZXIoY29udGV4dCwgZXZlbnQpOgogICAgcmV0dXJuIGNvbnRleHQubWxydW5faGFuZGxlcihjb250ZXh0LCBldmVudCkK + command: '' + default_handler: '' source: '' + max_replicas: 4 + base_image_pull: false + description: generic sklearn model server function_kind: serving_v2 - default_class: ClassifierModel - build: - commands: [] - code_origin: https://github.com/Michaelliv/functions.git#0e79859b0adccb92a9b65b02d438ed3dfa3e785f:/home/michaell/projects/functions/v2_model_server/v2_model_server.py -verbose: false + function_handler: v2-model-server-nuclio:handler + env: + - name: MLRUN_HTTPDB__NUCLIO__EXPLICIT_ACK + value: enabled diff --git a/functions/src/v2_model_server/v2_model_server.py b/functions/src/v2_model_server/v2_model_server.py index 572f1680d..d2d54793d 100644 --- a/functions/src/v2_model_server/v2_model_server.py +++ b/functions/src/v2_model_server/v2_model_server.py @@ -14,14 +14,11 @@ # # Generated by nuclio.export.NuclioExporter -import mlrun +import warnings -from cloudpickle import load -from typing import List -from sklearn.datasets import load_iris +import mlrun import numpy as np - -import warnings +from cloudpickle import load warnings.filterwarnings("ignore") @@ -32,7 +29,7 @@ def load(self): model_file, extra_data = self.get_model(".pkl") self.model = load(open(model_file, "rb")) - def predict(self, body: dict) -> List: + def predict(self, body: dict) -> list: """Generate model predictions from sample.""" feats = np.asarray(body["inputs"]) result: np.ndarray = self.model.predict(feats) diff --git a/functions/src/v2_model_tester/function.yaml b/functions/src/v2_model_tester/function.yaml index c9562b097..c70ec5e49 100644 --- a/functions/src/v2_model_tester/function.yaml +++ b/functions/src/v2_model_tester/function.yaml @@ -1,35 +1,29 @@ -kind: job metadata: - name: v2-model-tester tag: '' - hash: 72d3f664ff2aa870109e44f52f975bda2ac13682 - project: '' - labels: - author: Iguazio + name: v2-model-tester categories: - model-testing - machine-learning +verbose: false +kind: job spec: - command: '' - args: [] image: mlrun/mlrun - env: [] - default_handler: model_server_tester + disable_auto_mount: false + build: + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IGpzb24KZnJvbSBkYXRldGltZSBpbXBvcnQgZGF0ZXRpbWUKCmltcG9ydCBudW1weSBhcyBucAppbXBvcnQgcmVxdWVzdHMKZnJvbSBtbHJ1bi5hcnRpZmFjdHMgaW1wb3J0IENoYXJ0QXJ0aWZhY3QKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCgoKZGVmIG1vZGVsX3NlcnZlcl90ZXN0ZXIoCiAgICBjb250ZXh0LAogICAgdGFibGU6IERhdGFJdGVtLAogICAgYWRkcjogc3RyLAogICAgbGFiZWxfY29sdW1uOiBzdHIgPSAibGFiZWwiLAogICAgbW9kZWw6IHN0ciA9ICIiLAogICAgbWF0Y2hfZXJyOiBib29sID0gRmFsc2UsCiAgICByb3dzOiBpbnQgPSAyMCwKKToKICAgICIiIlRlc3QgYSBtb2RlbCBzZXJ2ZXIKCiAgICA6cGFyYW0gdGFibGU6ICAgICAgICAgY3N2L3BhcnF1ZXQgdGFibGUgd2l0aCB0ZXN0IGRhdGEKICAgIDpwYXJhbSBhZGRyOiAgICAgICAgICBmdW5jdGlvbiBhZGRyZXNzL3VybAogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogIG5hbWUgb2YgdGhlIGxhYmVsIGNvbHVtbiBpbiB0YWJsZQogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgIHRlc3RlZCBtb2RlbCBuYW1lCiAgICA6cGFyYW0gbWF0Y2hfZXJyOiAgICAgcmFpc2UgZXJyb3Igb24gdmFsaWRhdGlvbiAocmVxdWlyZSBwcm9wZXIgdGVzdCBzZXQpCiAgICA6cGFyYW0gcm93czogICAgICAgICAgbnVtYmVyIG9mIHJvd3MgdG8gdXNlIGZyb20gdGVzdCBzZXQKICAgICIiIgoKICAgIHRhYmxlID0gdGFibGUuYXNfZGYoKQoKICAgIHlfbGlzdCA9IHRhYmxlLnBvcChsYWJlbF9jb2x1bW4pLnZhbHVlcy50b2xpc3QoKQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInRlc3Rpbmcgd2l0aCBkYXRhc2V0IGFnYWluc3Qge2FkZHJ9LCBtb2RlbDoge21vZGVsfSIpCiAgICBpZiByb3dzIGFuZCByb3dzIDwgdGFibGUuc2hhcGVbMF06CiAgICAgICAgdGFibGUgPSB0YWJsZS5zYW1wbGUocm93cykKCiAgICBjb3VudCA9IGVycl9jb3VudCA9IG1hdGNoID0gMAogICAgdGltZXMgPSBbXQogICAgZm9yIHgsIHkgaW4gemlwKHRhYmxlLnZhbHVlcywgeV9saXN0KToKICAgICAgICBjb3VudCArPSAxCiAgICAgICAgZXZlbnRfZGF0YSA9IGpzb24uZHVtcHMoeyJpbnB1dHMiOiBbeC50b2xpc3QoKV19KQogICAgICAgIGhhZF9lcnIgPSBGYWxzZQogICAgICAgIHRyeToKICAgICAgICAgICAgc3RhcnQgPSBkYXRldGltZS5ub3coKQogICAgICAgICAgICByZXNwID0gcmVxdWVzdHMucHV0KGYie2FkZHJ9L3YyL21vZGVscy97bW9kZWx9L2luZmVyIiwganNvbj1ldmVudF9kYXRhKQogICAgICAgICAgICBpZiBub3QgcmVzcC5vazoKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKGYiYmFkIGZ1bmN0aW9uIHJlc3AhIVxue3Jlc3AudGV4dH0iKQogICAgICAgICAgICAgICAgZXJyX2NvdW50ICs9IDEKICAgICAgICAgICAgICAgIGNvbnRpbnVlCiAgICAgICAgICAgIHRpbWVzLmFwcGVuZCgoZGF0ZXRpbWUubm93KCkgLSBzdGFydCkubWljcm9zZWNvbmRzKQoKICAgICAgICBleGNlcHQgT1NFcnJvciBhcyBlcnI6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKGYiZXJyb3IgaW4gcmVxdWVzdCwgZGF0YTp7ZXZlbnRfZGF0YX0sIGVycm9yOiB7ZXJyfSIpCiAgICAgICAgICAgIGVycl9jb3VudCArPSAxCiAgICAgICAgICAgIGNvbnRpbnVlCgogICAgICAgIHJlc3BfZGF0YSA9IHJlc3AuanNvbigpCiAgICAgICAgcHJpbnQocmVzcF9kYXRhKQogICAgICAgIHlfcmVzcCA9IHJlc3BfZGF0YVsib3V0cHV0cyJdWzBdCiAgICAgICAgaWYgeSA9PSB5X3Jlc3A6CiAgICAgICAgICAgIG1hdGNoICs9IDEKCiAgICBjb250ZXh0LmxvZ19yZXN1bHQoInRvdGFsX3Rlc3RzIiwgY291bnQpCiAgICBjb250ZXh0LmxvZ19yZXN1bHQoImVycm9ycyIsIGVycl9jb3VudCkKICAgIGNvbnRleHQubG9nX3Jlc3VsdCgibWF0Y2giLCBtYXRjaCkKICAgIGlmIGNvdW50IC0gZXJyX2NvdW50ID4gMDoKICAgICAgICB0aW1lc19hcnIgPSBucC5hcnJheSh0aW1lcykKICAgICAgICBjb250ZXh0LmxvZ19yZXN1bHQoImF2Z19sYXRlbmN5IiwgaW50KG5wLm1lYW4odGltZXNfYXJyKSkpCiAgICAgICAgY29udGV4dC5sb2dfcmVzdWx0KCJtaW5fbGF0ZW5jeSIsIGludChucC5hbWluKHRpbWVzX2FycikpKQogICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdCgibWF4X2xhdGVuY3kiLCBpbnQobnAuYW1heCh0aW1lc19hcnIpKSkKCiAgICAgICAgY2hhcnQgPSBDaGFydEFydGlmYWN0KCJsYXRlbmN5IiwgaGVhZGVyPVsiVGVzdCIsICJMYXRlbmN5IChtaWNyb3NlYykiXSkKICAgICAgICBmb3IgaSBpbiByYW5nZShsZW4odGltZXMpKToKICAgICAgICAgICAgY2hhcnQuYWRkX3JvdyhbaSArIDEsIGludCh0aW1lc1tpXSldKQogICAgICAgIGNvbnRleHQubG9nX2FydGlmYWN0KGNoYXJ0KQoKICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgZiJydW4ge2NvdW50fSB0ZXN0cywge2Vycl9jb3VudH0gZXJyb3JzIGFuZCB7bWF0Y2h9IG1hdGNoIGV4cGVjdGVkIHZhbHVlIgogICAgKQoKICAgIGlmIGVycl9jb3VudDoKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKGYiZmFpbGVkIG9uIHtlcnJfY291bnR9IHRlc3RzIG9mIHtjb3VudH0iKQoKICAgIGlmIG1hdGNoX2VyciBhbmQgbWF0Y2ggIT0gY291bnQ6CiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcihmIm9ubHkge21hdGNofSByZXN1bHRzIG1hdGNoIG91dCBvZiB7Y291bnR9IikK + code_origin: '' + filename: v2_model_tester.py entry_points: model_server_tester: - name: model_server_tester - doc: Test a model server parameters: - name: context - default: '' - name: table type: DataItem doc: csv/parquet table with test data - default: '' - name: addr type: str doc: function address/url - default: '' - name: label_column type: str doc: name of the label column in table @@ -46,13 +40,11 @@ spec: type: int doc: number of rows to use from test set default: 20 - outputs: - - default: '' - lineno: 13 + name: model_server_tester + doc: Test a model server + has_kwargs: false + has_varargs: false + lineno: 26 + command: '' description: test v2 model servers - build: - functionSourceCode: IyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IG9zCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IHJlcXVlc3RzCmltcG9ydCBqc29uCmltcG9ydCBudW1weSBhcyBucApmcm9tIGRhdGV0aW1lIGltcG9ydCBkYXRldGltZQpmcm9tIG1scnVuLmRhdGFzdG9yZSBpbXBvcnQgRGF0YUl0ZW0KZnJvbSBtbHJ1bi5hcnRpZmFjdHMgaW1wb3J0IENoYXJ0QXJ0aWZhY3QKCgpkZWYgbW9kZWxfc2VydmVyX3Rlc3RlcigKICAgIGNvbnRleHQsCiAgICB0YWJsZTogRGF0YUl0ZW0sCiAgICBhZGRyOiBzdHIsCiAgICBsYWJlbF9jb2x1bW46IHN0ciA9ICJsYWJlbCIsCiAgICBtb2RlbDogc3RyID0gIiIsCiAgICBtYXRjaF9lcnI6IGJvb2wgPSBGYWxzZSwKICAgIHJvd3M6IGludCA9IDIwLAopOgogICAgIiIiVGVzdCBhIG1vZGVsIHNlcnZlcgoKICAgIDpwYXJhbSB0YWJsZTogICAgICAgICBjc3YvcGFycXVldCB0YWJsZSB3aXRoIHRlc3QgZGF0YQogICAgOnBhcmFtIGFkZHI6ICAgICAgICAgIGZ1bmN0aW9uIGFkZHJlc3MvdXJsCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uOiAgbmFtZSBvZiB0aGUgbGFiZWwgY29sdW1uIGluIHRhYmxlCiAgICA6cGFyYW0gbW9kZWw6ICAgICAgICAgdGVzdGVkIG1vZGVsIG5hbWUKICAgIDpwYXJhbSBtYXRjaF9lcnI6ICAgICByYWlzZSBlcnJvciBvbiB2YWxpZGF0aW9uIChyZXF1aXJlIHByb3BlciB0ZXN0IHNldCkKICAgIDpwYXJhbSByb3dzOiAgICAgICAgICBudW1iZXIgb2Ygcm93cyB0byB1c2UgZnJvbSB0ZXN0IHNldAogICAgIiIiCgogICAgdGFibGUgPSB0YWJsZS5hc19kZigpCgogICAgeV9saXN0ID0gdGFibGUucG9wKGxhYmVsX2NvbHVtbikudmFsdWVzLnRvbGlzdCgpCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidGVzdGluZyB3aXRoIGRhdGFzZXQgYWdhaW5zdCB7YWRkcn0sIG1vZGVsOiB7bW9kZWx9IikKICAgIGlmIHJvd3MgYW5kIHJvd3MgPCB0YWJsZS5zaGFwZVswXToKICAgICAgICB0YWJsZSA9IHRhYmxlLnNhbXBsZShyb3dzKQoKICAgIGNvdW50ID0gZXJyX2NvdW50ID0gbWF0Y2ggPSAwCiAgICB0aW1lcyA9IFtdCiAgICBmb3IgeCwgeSBpbiB6aXAodGFibGUudmFsdWVzLCB5X2xpc3QpOgogICAgICAgIGNvdW50ICs9IDEKICAgICAgICBldmVudF9kYXRhID0ganNvbi5kdW1wcyh7ImlucHV0cyI6IFt4LnRvbGlzdCgpXX0pCiAgICAgICAgaGFkX2VyciA9IEZhbHNlCiAgICAgICAgdHJ5OgogICAgICAgICAgICBzdGFydCA9IGRhdGV0aW1lLm5vdygpCiAgICAgICAgICAgIHJlc3AgPSByZXF1ZXN0cy5wdXQoZiJ7YWRkcn0vdjIvbW9kZWxzL3ttb2RlbH0vaW5mZXIiLCBqc29uPWV2ZW50X2RhdGEpCiAgICAgICAgICAgIGlmIG5vdCByZXNwLm9rOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoZiJiYWQgZnVuY3Rpb24gcmVzcCEhXG57cmVzcC50ZXh0fSIpCiAgICAgICAgICAgICAgICBlcnJfY291bnQgKz0gMQogICAgICAgICAgICAgICAgY29udGludWUKICAgICAgICAgICAgdGltZXMuYXBwZW5kKChkYXRldGltZS5ub3coKSAtIHN0YXJ0KS5taWNyb3NlY29uZHMpCgogICAgICAgIGV4Y2VwdCBPU0Vycm9yIGFzIGVycjoKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoZiJlcnJvciBpbiByZXF1ZXN0LCBkYXRhOntldmVudF9kYXRhfSwgZXJyb3I6IHtlcnJ9IikKICAgICAgICAgICAgZXJyX2NvdW50ICs9IDEKICAgICAgICAgICAgY29udGludWUKCiAgICAgICAgcmVzcF9kYXRhID0gcmVzcC5qc29uKCkKICAgICAgICBwcmludChyZXNwX2RhdGEpCiAgICAgICAgeV9yZXNwID0gcmVzcF9kYXRhWyJvdXRwdXRzIl1bMF0KICAgICAgICBpZiB5ID09IHlfcmVzcDoKICAgICAgICAgICAgbWF0Y2ggKz0gMQoKICAgIGNvbnRleHQubG9nX3Jlc3VsdCgidG90YWxfdGVzdHMiLCBjb3VudCkKICAgIGNvbnRleHQubG9nX3Jlc3VsdCgiZXJyb3JzIiwgZXJyX2NvdW50KQogICAgY29udGV4dC5sb2dfcmVzdWx0KCJtYXRjaCIsIG1hdGNoKQogICAgaWYgY291bnQgLSBlcnJfY291bnQgPiAwOgogICAgICAgIHRpbWVzX2FyciA9IG5wLmFycmF5KHRpbWVzKQogICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdCgiYXZnX2xhdGVuY3kiLCBpbnQobnAubWVhbih0aW1lc19hcnIpKSkKICAgICAgICBjb250ZXh0LmxvZ19yZXN1bHQoIm1pbl9sYXRlbmN5IiwgaW50KG5wLmFtaW4odGltZXNfYXJyKSkpCiAgICAgICAgY29udGV4dC5sb2dfcmVzdWx0KCJtYXhfbGF0ZW5jeSIsIGludChucC5hbWF4KHRpbWVzX2FycikpKQoKICAgICAgICBjaGFydCA9IENoYXJ0QXJ0aWZhY3QoImxhdGVuY3kiLCBoZWFkZXI9WyJUZXN0IiwgIkxhdGVuY3kgKG1pY3Jvc2VjKSJdKQogICAgICAgIGZvciBpIGluIHJhbmdlKGxlbih0aW1lcykpOgogICAgICAgICAgICBjaGFydC5hZGRfcm93KFtpICsgMSwgaW50KHRpbWVzW2ldKV0pCiAgICAgICAgY29udGV4dC5sb2dfYXJ0aWZhY3QoY2hhcnQpCgogICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICBmInJ1biB7Y291bnR9IHRlc3RzLCB7ZXJyX2NvdW50fSBlcnJvcnMgYW5kIHttYXRjaH0gbWF0Y2ggZXhwZWN0ZWQgdmFsdWUiCiAgICApCgogICAgaWYgZXJyX2NvdW50OgogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IoZiJmYWlsZWQgb24ge2Vycl9jb3VudH0gdGVzdHMgb2Yge2NvdW50fSIpCgogICAgaWYgbWF0Y2hfZXJyIGFuZCBtYXRjaCAhPSBjb3VudDoKICAgICAgICByYWlzZSBWYWx1ZUVycm9yKGYib25seSB7bWF0Y2h9IHJlc3VsdHMgbWF0Y2ggb3V0IG9mIHtjb3VudH0iKQo= - commands: [] - code_origin: https://github.com/daniels290813/functions.git#55a79c32be5d233cc11efcf40cd3edbe309bfdef:/home/kali/functions/v2_model_tester/v2_model_tester.py - affinity: null -verbose: false + default_handler: model_server_tester diff --git a/functions/src/v2_model_tester/v2_model_tester.py b/functions/src/v2_model_tester/v2_model_tester.py index 74590acdc..3d41ad37b 100644 --- a/functions/src/v2_model_tester/v2_model_tester.py +++ b/functions/src/v2_model_tester/v2_model_tester.py @@ -14,14 +14,13 @@ # # Generated by nuclio.export.NuclioExporter -import os -import pandas as pd -import requests import json -import numpy as np from datetime import datetime -from mlrun.datastore import DataItem + +import numpy as np +import requests from mlrun.artifacts import ChartArtifact +from mlrun.datastore import DataItem def model_server_tester( diff --git a/modules/src/agent_deployer/agent_deployer.py b/modules/src/agent_deployer/agent_deployer.py index 9af0dd632..9a4ab415a 100644 --- a/modules/src/agent_deployer/agent_deployer.py +++ b/modules/src/agent_deployer/agent_deployer.py @@ -12,18 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Optional import os import mlrun.errors -from mlrun import get_current_project, code_to_function, mlconf -from mlrun.runtimes import ServingRuntime -from mlrun.serving import ModelRunnerStep +from mlrun import code_to_function, get_current_project, mlconf from mlrun.datastore.datastore_profile import ( - DatastoreProfileV3io, DatastoreProfileKafkaStream, DatastoreProfileTDEngine, + DatastoreProfileV3io, ) +from mlrun.runtimes import ServingRuntime +from mlrun.serving import ModelRunnerStep from mlrun.utils import logger @@ -33,10 +32,10 @@ def __init__( agent_name: str, model_class_name: str, function: str, - result_path: Optional[str] = None, - inputs_path: Optional[str] = None, - outputs: Optional[list[str]] = None, - requirements: Optional[list[str]] = None, + result_path: str | None = None, + inputs_path: str | None = None, + outputs: list[str] | None = None, + requirements: list[str] | None = None, image: str = "mlrun/mlrun", set_model_monitoring: bool = False, **model_params, diff --git a/modules/src/agent_deployer/test_agent_deployer.py b/modules/src/agent_deployer/test_agent_deployer.py index 0bb3adc8b..7700bfeea 100644 --- a/modules/src/agent_deployer/test_agent_deployer.py +++ b/modules/src/agent_deployer/test_agent_deployer.py @@ -14,14 +14,13 @@ # import unittest -from unittest.mock import patch, MagicMock -from agent_deployer import AgentDeployer -import mlrun.errors +from unittest.mock import MagicMock, patch +import mlrun.errors +from agent_deployer import AgentDeployer class TestAgentDeployer(unittest.TestCase): - def setUp(self): # Common parameters for a minimal AgentDeployer instance self.deployer_params = { @@ -33,7 +32,9 @@ def setUp(self): # --- Test Cases for Properties --- - @patch('agent_deployer.get_current_project') # Patch the import in the *module* you are testing + @patch( + "agent_deployer.get_current_project" + ) # Patch the import in the *module* you are testing def test_project_property_returns_project(self, mock_get_current_project): """Test that the project property returns the project if it exists.""" mock_proj = MagicMock() @@ -42,13 +43,13 @@ def test_project_property_returns_project(self, mock_get_current_project): self.assertEqual(self.deployer.project, mock_proj) mock_get_current_project.assert_called_once_with(silent=True) - @patch('agent_deployer.get_current_project', return_value=None) + @patch("agent_deployer.get_current_project", return_value=None) def test_project_name_raises_error_if_no_project(self, mock_get_current_project): """Test that project_name raises an error when no project is found.""" with self.assertRaises(mlrun.errors.MLRunInvalidArgumentError): _ = self.deployer.project_name - @patch('agent_deployer.get_current_project') + @patch("agent_deployer.get_current_project") def test_project_name_returns_name(self, mock_get_current_project): """Test that project_name correctly retrieves the name from the project metadata.""" mock_proj = MagicMock() @@ -57,15 +58,18 @@ def test_project_name_returns_name(self, mock_get_current_project): self.assertEqual(self.deployer.project_name, "test-project-name") - - @patch('agent_deployer.AgentDeployer.project', new_callable=unittest.mock.PropertyMock) + @patch( + "agent_deployer.AgentDeployer.project", new_callable=unittest.mock.PropertyMock + ) def test_configure_model_monitoring_handles_conflict_error(self, mock_project_prop): """Test that the method handles expected exceptions during enable_model_monitoring.""" mock_project = MagicMock() # Simulate an expected error that should be caught and passed over - mock_project.enable_model_monitoring.side_effect = mlrun.errors.MLRunConflictError("Already deployed") + mock_project.enable_model_monitoring.side_effect = ( + mlrun.errors.MLRunConflictError("Already deployed") + ) mock_project_prop.return_value = mock_project # This should run without raising an uncaught exception self.deployer.configure_model_monitoring() - mock_project.enable_model_monitoring.assert_called_once() \ No newline at end of file + mock_project.enable_model_monitoring.assert_called_once() diff --git a/modules/src/count_events/count_events.py b/modules/src/count_events/count_events.py index 1c6d97621..4f04366ac 100644 --- a/modules/src/count_events/count_events.py +++ b/modules/src/count_events/count_events.py @@ -13,21 +13,22 @@ # limitations under the License. # +import mlrun.model_monitoring.applications.context as mm_context from mlrun.model_monitoring.applications import ( - ModelMonitoringApplicationBase, ModelMonitoringApplicationMetric, + ModelMonitoringApplicationBase, + ModelMonitoringApplicationMetric, ) -import mlrun.model_monitoring.applications.context as mm_context class CountApp(ModelMonitoringApplicationBase): """ Model Monitoring Application that counts the number of events in the given time window. """ + def do_tracking( - self, - monitoring_context: mm_context.MonitoringApplicationContext + self, monitoring_context: mm_context.MonitoringApplicationContext ) -> ModelMonitoringApplicationMetric: - """" + """ " he do_tracking method implementation for the CountApp class. It counts the number of events in the sample data-frame and logs the count. @@ -47,4 +48,4 @@ def do_tracking( return ModelMonitoringApplicationMetric( name="count", value=count, - ) \ No newline at end of file + ) diff --git a/modules/src/count_events/item.yaml b/modules/src/count_events/item.yaml index 049651ddb..723ebc4a9 100644 --- a/modules/src/count_events/item.yaml +++ b/modules/src/count_events/item.yaml @@ -7,7 +7,7 @@ generationDate: 2025-09-16:12-25 hidden: false labels: author: Iguazio -mlrunVersion: 1.10.0-rc41 +mlrunVersion: 1.10.0 name: count_events spec: filename: count_events.py diff --git a/modules/src/count_events/test_count_events.py b/modules/src/count_events/test_count_events.py index 66a94c932..fc3e76a4e 100644 --- a/modules/src/count_events/test_count_events.py +++ b/modules/src/count_events/test_count_events.py @@ -14,15 +14,15 @@ # -from mlrun.model_monitoring.applications import ModelMonitoringApplicationMetric -import mlrun.model_monitoring.applications.context as mm_context - -from count_events import CountApp - -from unittest.mock import Mock from datetime import datetime +from unittest.mock import Mock + +import mlrun.model_monitoring.applications.context as mm_context import pandas as pd import pytest +from count_events import CountApp +from mlrun.model_monitoring.applications import ModelMonitoringApplicationMetric + class TestCountApp: """Test suite for CountApp class.""" @@ -30,6 +30,7 @@ class TestCountApp: def setup_method(self): """Set up test fixtures before each test method.""" self.count_app = CountApp() + @staticmethod def _create_mock_monitoring_context(sample_df, model_endpoint_name="test-model"): """Helper method to create a mock monitoring context.""" @@ -53,7 +54,6 @@ def _create_mock_monitoring_context(sample_df, model_endpoint_name="test-model") return mock_context - @pytest.mark.parametrize("df_size", [0, 1, 10, 100, 1000]) def test_do_tracking_with_various_dataframe_sizes(self, df_size): """Test do_tracking with various dataframe sizes using parametrized test.""" @@ -72,4 +72,3 @@ def test_do_tracking_with_various_dataframe_sizes(self, df_size): assert isinstance(result, ModelMonitoringApplicationMetric) assert result.value == df_size assert result.name == "count" - diff --git a/modules/src/evidently_iris/evidently_iris.py b/modules/src/evidently_iris/evidently_iris.py index e7a9f3ef9..375c1d3f8 100644 --- a/modules/src/evidently_iris/evidently_iris.py +++ b/modules/src/evidently_iris/evidently_iris.py @@ -14,18 +14,8 @@ from typing import Optional -import pandas as pd -from sklearn.datasets import load_iris - import mlrun.model_monitoring.applications.context as mm_context -from mlrun.common.schemas.model_monitoring.constants import ( - ResultKindApp, - ResultStatusApp, -) -from mlrun.feature_store.api import norm_column_name -from mlrun.model_monitoring.applications import ModelMonitoringApplicationResult -from mlrun.model_monitoring.applications.evidently import EvidentlyModelMonitoringApplicationBase - +import pandas as pd from evidently.core.report import Report, Snapshot from evidently.metrics import DatasetMissingValueCount, ValueDrift from evidently.presets import DataDriftPreset, DataSummaryPreset @@ -33,6 +23,16 @@ STR_UUID, OrgID, ) +from mlrun.common.schemas.model_monitoring.constants import ( + ResultKindApp, + ResultStatusApp, +) +from mlrun.feature_store.api import norm_column_name +from mlrun.model_monitoring.applications import ModelMonitoringApplicationResult +from mlrun.model_monitoring.applications.evidently import ( + EvidentlyModelMonitoringApplicationBase, +) +from sklearn.datasets import load_iris _PROJECT_NAME = "Iris Monitoring" _PROJECT_DESCRIPTION = "Test project using iris dataset" @@ -43,12 +43,13 @@ class EvidentlyIrisMonitoringApp(EvidentlyModelMonitoringApplicationBase): This model monitoring application is a simple example of integrating MLRun with Evidently for data monitoring, which you can adapt to fit your own project needs or use as a reference implementation. """ + NAME = "Evidently-App-Test" def __init__( self, evidently_project_id: Optional["STR_UUID"] = None, - evidently_workspace_path: Optional[str] = None, + evidently_workspace_path: str | None = None, cloud_workspace: bool = False, evidently_organization_id: Optional["OrgID"] = None, ) -> None: diff --git a/modules/src/evidently_iris/item.yaml b/modules/src/evidently_iris/item.yaml index 42c5c10cb..f8aa203fa 100644 --- a/modules/src/evidently_iris/item.yaml +++ b/modules/src/evidently_iris/item.yaml @@ -8,7 +8,7 @@ generationDate: 2025-11-09:12-25 hidden: false labels: author: Iguazio -mlrunVersion: 1.10.0-rc41 +mlrunVersion: 1.10.0 name: evidently_iris spec: filename: evidently_iris.py diff --git a/modules/src/evidently_iris/test_evidently_iris.py b/modules/src/evidently_iris/test_evidently_iris.py index 6488768fd..a9d12d75a 100644 --- a/modules/src/evidently_iris/test_evidently_iris.py +++ b/modules/src/evidently_iris/test_evidently_iris.py @@ -20,14 +20,12 @@ import pytest import semver - +from evidently_iris import EvidentlyIrisMonitoringApp from mlrun.errors import MLRunIncompatibleVersionError from mlrun.model_monitoring.applications.evidently.base import ( _check_evidently_version, ) -from evidently_iris import EvidentlyIrisMonitoringApp - @pytest.mark.parametrize( ("cur", "ref", "expectation"), diff --git a/modules/src/histogram_data_drift/histogram_data_drift.py b/modules/src/histogram_data_drift/histogram_data_drift.py index b8cdcf299..59df3df06 100644 --- a/modules/src/histogram_data_drift/histogram_data_drift.py +++ b/modules/src/histogram_data_drift/histogram_data_drift.py @@ -13,16 +13,14 @@ # limitations under the License. from dataclasses import dataclass -from typing import Final, Optional, Protocol, Union, cast - -import numpy as np -from pandas import DataFrame, Series +from typing import Final, Protocol, cast import mlrun.artifacts import mlrun.common.model_monitoring.helpers import mlrun.model_monitoring.applications.context as mm_context import mlrun.model_monitoring.applications.results as mm_results import mlrun.model_monitoring.features_drift_table as mm_drift_table +import numpy as np from mlrun.common.schemas.model_monitoring.constants import ( ResultKindApp, ResultStatusApp, @@ -37,6 +35,7 @@ KullbackLeiblerDivergence, TotalVarianceDistance, ) +from pandas import DataFrame, Series class InvalidMetricValueError(ValueError): @@ -134,7 +133,7 @@ class HistogramDataDriftApplication(ModelMonitoringApplicationBase): def __init__( self, - value_classifier: Optional[ValueClassifier] = None, + value_classifier: ValueClassifier | None = None, produce_json_artifact: bool = False, produce_plotly_artifact: bool = False, ) -> None: @@ -145,9 +144,9 @@ def __init__( :param produce_plotly_artifact: Whether to produce the Plotly artifact or not, ``False`` by default. """ self._value_classifier = value_classifier or DataDriftClassifier() - assert self._REQUIRED_METRICS <= set( - self.metrics - ), "TVD and Hellinger distance are required for the general data drift result" + assert self._REQUIRED_METRICS <= set(self.metrics), ( + "TVD and Hellinger distance are required for the general data drift result" + ) self._produce_json_artifact = produce_json_artifact self._produce_plotly_artifact = produce_plotly_artifact @@ -349,11 +348,9 @@ def _log_drift_artifacts( def do_tracking( self, monitoring_context: mm_context.MonitoringApplicationContext ) -> list[ - Union[ - mm_results.ModelMonitoringApplicationResult, - mm_results.ModelMonitoringApplicationMetric, - mm_results._ModelMonitoringApplicationStats, - ] + mm_results.ModelMonitoringApplicationResult + | mm_results.ModelMonitoringApplicationMetric + | mm_results._ModelMonitoringApplicationStats ]: """ Calculate and return the data drift metrics, averaged over the features. diff --git a/modules/src/histogram_data_drift/item.yaml b/modules/src/histogram_data_drift/item.yaml index f516ae071..83d0f0c99 100644 --- a/modules/src/histogram_data_drift/item.yaml +++ b/modules/src/histogram_data_drift/item.yaml @@ -8,7 +8,7 @@ generationDate: 2025-11-06:12-25 hidden: false labels: author: Iguazio -mlrunVersion: 1.10.0-rc41 +mlrunVersion: 1.10.0 name: histogram_data_drift spec: filename: histogram_data_drift.py diff --git a/modules/src/histogram_data_drift/test_histogram_data_drift.py b/modules/src/histogram_data_drift/test_histogram_data_drift.py index 018edaa86..c731e2c9b 100644 --- a/modules/src/histogram_data_drift/test_histogram_data_drift.py +++ b/modules/src/histogram_data_drift/test_histogram_data_drift.py @@ -16,25 +16,24 @@ from pathlib import Path from unittest.mock import Mock -import pandas as pd -import pytest -from hypothesis import given -from hypothesis import strategies as st - import mlrun.common.model_monitoring.helpers import mlrun.model_monitoring.applications import mlrun.model_monitoring.applications.context as mm_context import mlrun.utils -from mlrun.common.schemas.model_monitoring.constants import ( - ResultKindApp, - ResultStatusApp, -) +import pandas as pd +import pytest from histogram_data_drift import ( DataDriftClassifier, HistogramDataDriftApplication, InvalidMetricValueError, InvalidThresholdValueError, ) +from hypothesis import given +from hypothesis import strategies as st +from mlrun.common.schemas.model_monitoring.constants import ( + ResultKindApp, + ResultStatusApp, +) assets_folder = Path(__file__).parent / "assets" @@ -99,9 +98,9 @@ def classifier() -> DataDriftClassifier: def test_status( classifier: DataDriftClassifier, value: float, expected_status: ResultStatusApp ) -> None: - assert ( - classifier.value_to_status(value) == expected_status - ), "The status is different than expected" + assert classifier.value_to_status(value) == expected_status, ( + "The status is different than expected" + ) class TestApplication: @@ -205,15 +204,15 @@ def test( res, mlrun.model_monitoring.applications.ModelMonitoringApplicationResult, ): - assert ( - res.kind == ResultKindApp.data_drift - ), "The kind should be data drift" - assert ( - res.name == "general_drift" - ), "The result name should be general_drift" - assert ( - res.status == ResultStatusApp.potential_detection - ), "Expected potential detection in the general drift" + assert res.kind == ResultKindApp.data_drift, ( + "The kind should be data drift" + ) + assert res.name == "general_drift", ( + "The result name should be general_drift" + ) + assert res.status == ResultStatusApp.potential_detection, ( + "Expected potential detection in the general drift" + ) elif isinstance( res, mlrun.model_monitoring.applications.ModelMonitoringApplicationMetric, @@ -274,6 +273,6 @@ def test_compute_metrics_per_feature( assert set(metrics_per_feature.columns) == { metric.NAME for metric in application.metrics }, "Different metrics than expected" - assert set(metrics_per_feature.index) == set( - feature_stats.columns - ), "The features are different than expected" + assert set(metrics_per_feature.index) == set(feature_stats.columns), ( + "The features are different than expected" + ) diff --git a/modules/src/openai_proxy_app/openai_proxy_app.py b/modules/src/openai_proxy_app/openai_proxy_app.py index 65bfbf7c9..9132d9bff 100644 --- a/modules/src/openai_proxy_app/openai_proxy_app.py +++ b/modules/src/openai_proxy_app/openai_proxy_app.py @@ -12,13 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # -#This module acts as a lightweight gateway to OpenAI-compatible APIs. -#You can send chat prompts, create embeddings, or get model responses without worrying about authentication or endpoint differences. -#It simplifies access so you can test, analyze, or integrate AI features directly into your projects or notebooks with minimal setup. +# This module acts as a lightweight gateway to OpenAI-compatible APIs. +# You can send chat prompts, create embeddings, or get model responses without worrying about authentication or endpoint differences. +# It simplifies access so you can test, analyze, or integrate AI features directly into your projects or notebooks with minimal setup. BASE64 = "IyBvcGVuYWlfcHJveHkvb3BlbmFpLnB5CgppbXBvcnQgb3MKaW1wb3J0IGpzb24KZnJvbSB1cmxsaWIucGFyc2UgaW1wb3J0IHVybGpvaW4KZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgRGljdCwgTGlzdCwgT3B0aW9uYWwKCmltcG9ydCByZXF1ZXN0cwpmcm9tIGZhc3RhcGkgaW1wb3J0IEZhc3RBUEksIFJlcXVlc3QsIFJlc3BvbnNlLCBCb2R5CgphcHAgPSBGYXN0QVBJKAogICAgdGl0bGU9Ik9wZW5BSSBQcm94eSBBcHAiLAogICAgZGVzY3JpcHRpb249IkxvY2FsIEZhc3RBUEkgcHJveHkgZm9yIE9wZW5BSSBzdHlsZSBlbmRwb2ludHMiLAogICAgdmVyc2lvbj0iMS4wLjAiLAopCgpPUEVOQUlfQkFTRV9VUkwgPSBvcy5nZXRlbnYoIk9QRU5BSV9CQVNFX1VSTCIsICJodHRwczovL2FwaS5vcGVuYWkuY29tIikucnN0cmlwKCIvIikKT1BFTkFJX0FQSV9LRVkgPSBvcy5nZXRlbnYoIk9QRU5BSV9BUElfS0VZIiwgIiIpCk9QRU5BSV9ERUZBVUxUX01PREVMID0gb3MuZ2V0ZW52KCJPUEVOQUlfREVGQVVMVF9NT0RFTCIsICJncHQtNG8tbWluaSIpCgoKZGVmIGJ1aWxkX2hlYWRlcnMoaW5jb21pbmc6IGRpY3QpIC0+IGRpY3Q6CiAgICBoZWFkZXJzID0ge30KICAgIGF1dGggPSBpbmNvbWluZy5nZXQoImF1dGhvcml6YXRpb24iKSBvciBpbmNvbWluZy5nZXQoIkF1dGhvcml6YXRpb24iKQogICAgaWYgYXV0aDoKICAgICAgICBoZWFkZXJzWyJBdXRob3JpemF0aW9uIl0gPSBhdXRoCiAgICBlbGlmIE9QRU5BSV9BUElfS0VZOgogICAgICAgIGhlYWRlcnNbIkF1dGhvcml6YXRpb24iXSA9IGYiQmVhcmVyIHtPUEVOQUlfQVBJX0tFWX0iCiAgICBjdHlwZSA9IGluY29taW5nLmdldCgiY29udGVudC10eXBlIikgb3IgaW5jb21pbmcuZ2V0KCJDb250ZW50LVR5cGUiKSBvciAiYXBwbGljYXRpb24vanNvbiIKICAgIGhlYWRlcnNbIkNvbnRlbnQtVHlwZSJdID0gY3R5cGUKICAgIHJldHVybiBoZWFkZXJzCgoKZGVmIGJ1aWxkX3RhcmdldChwYXRoOiBzdHIpIC0+IHN0cjoKICAgIGJhc2UgPSBPUEVOQUlfQkFTRV9VUkwKICAgIGlmIGJhc2UuZW5kc3dpdGgoIi92MSIpIG9yIGJhc2UuZW5kc3dpdGgoIi92MS8iKToKICAgICAgICBiYXNlID0gYmFzZVs6LTNdIGlmIGJhc2UuZW5kc3dpdGgoIi92MSIpIGVsc2UgYmFzZVs6LTRdCiAgICByZXR1cm4gdXJsam9pbihiYXNlICsgIi8iLCBwYXRoLmxzdHJpcCgiLyIpKQoKCmRlZiBmb3J3YXJkX2pzb24ocGF0aDogc3RyLCBib2R5OiBkaWN0LCBoZWFkZXJzOiBkaWN0LCBxdWVyeTogZGljdCk6CiAgICB0YXJnZXQgPSBidWlsZF90YXJnZXQocGF0aCkKICAgIHJlc3AgPSByZXF1ZXN0cy5wb3N0KAogICAgICAgIHRhcmdldCwKICAgICAgICBoZWFkZXJzPWhlYWRlcnMsCiAgICAgICAgcGFyYW1zPXF1ZXJ5LAogICAgICAgIGpzb249Ym9keSwKICAgICAgICB0aW1lb3V0PTYwLAogICAgKQogICAgcmV0dXJuIHJlc3AKCkBhcHAuZ2V0KCIvIikKZGVmIGhlYWx0aCgpOgogICAgcmV0dXJuIHsic3RhdHVzIjogIm9rIn0KCgojIHJlbGF4ZWQgY2hhdCBlbmRwb2ludCwgYWNjZXB0cyBhbnkgSlNPTiB0aGF0IGluY2x1ZGVzIG1lc3NhZ2VzCkBhcHAucG9zdCgiL3YxL2NoYXQvY29tcGxldGlvbnMiKQphc3luYyBkZWYgY2hhdF9jb21wbGV0aW9ucygKICAgIHJlcXVlc3Q6IFJlcXVlc3QsCiAgICBwYXlsb2FkOiBEaWN0W3N0ciwgQW55XSA9IEJvZHkoLi4uKSwKKToKICAgIGlmICJtZXNzYWdlcyIgbm90IGluIHBheWxvYWQgb3Igbm90IGlzaW5zdGFuY2UocGF5bG9hZFsibWVzc2FnZXMiXSwgbGlzdCk6CiAgICAgICAgcmV0dXJuIFJlc3BvbnNlKAogICAgICAgICAgICBjb250ZW50PWpzb24uZHVtcHMoeyJlcnJvciI6ICJtZXNzYWdlcyBtdXN0IGJlIGEgbGlzdCBvZiBjaGF0IG1lc3NhZ2VzIn0pLAogICAgICAgICAgICBzdGF0dXNfY29kZT00MDAsCiAgICAgICAgICAgIG1lZGlhX3R5cGU9ImFwcGxpY2F0aW9uL2pzb24iLAogICAgICAgICkKCiAgICBpZiAibW9kZWwiIG5vdCBpbiBwYXlsb2FkIG9yIHBheWxvYWRbIm1vZGVsIl0gaXMgTm9uZToKICAgICAgICBwYXlsb2FkWyJtb2RlbCJdID0gT1BFTkFJX0RFRkFVTFRfTU9ERUwKCiAgICBoZWFkZXJzID0gYnVpbGRfaGVhZGVycyhkaWN0KHJlcXVlc3QuaGVhZGVycykpCiAgICByZXNwID0gZm9yd2FyZF9qc29uKCIvdjEvY2hhdC9jb21wbGV0aW9ucyIsIHBheWxvYWQsIGhlYWRlcnMsIGRpY3QocmVxdWVzdC5xdWVyeV9wYXJhbXMpKQogICAgcmV0dXJuIFJlc3BvbnNlKAogICAgICAgIGNvbnRlbnQ9cmVzcC5jb250ZW50LAogICAgICAgIHN0YXR1c19jb2RlPXJlc3Auc3RhdHVzX2NvZGUsCiAgICAgICAgbWVkaWFfdHlwZT1yZXNwLmhlYWRlcnMuZ2V0KCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpLAogICAgKQoKCkBhcHAucG9zdCgiL3YxL2VtYmVkZGluZ3MiKQphc3luYyBkZWYgZW1iZWRkaW5ncygKICAgIHJlcXVlc3Q6IFJlcXVlc3QsCiAgICBwYXlsb2FkOiBEaWN0W3N0ciwgQW55XSA9IEJvZHkoLi4uKSwKKToKICAgIGlmICJtb2RlbCIgbm90IGluIHBheWxvYWQgb3Igbm90IHBheWxvYWRbIm1vZGVsIl06CiAgICAgICAgcGF5bG9hZFsibW9kZWwiXSA9ICJ0ZXh0LWVtYmVkZGluZy0zLXNtYWxsIgogICAgaGVhZGVycyA9IGJ1aWxkX2hlYWRlcnMoZGljdChyZXF1ZXN0LmhlYWRlcnMpKQogICAgcmVzcCA9IGZvcndhcmRfanNvbigiL3YxL2VtYmVkZGluZ3MiLCBwYXlsb2FkLCBoZWFkZXJzLCBkaWN0KHJlcXVlc3QucXVlcnlfcGFyYW1zKSkKICAgIHJldHVybiBSZXNwb25zZSgKICAgICAgICBjb250ZW50PXJlc3AuY29udGVudCwKICAgICAgICBzdGF0dXNfY29kZT1yZXNwLnN0YXR1c19jb2RlLAogICAgICAgIG1lZGlhX3R5cGU9cmVzcC5oZWFkZXJzLmdldCgiQ29udGVudC1UeXBlIiwgImFwcGxpY2F0aW9uL2pzb24iKSwKICAgICkKCgpAYXBwLnBvc3QoIi92MS9yZXNwb25zZXMiKQphc3luYyBkZWYgcmVzcG9uc2VzX2FwaSgKICAgIHJlcXVlc3Q6IFJlcXVlc3QsCiAgICBwYXlsb2FkOiBEaWN0W3N0ciwgQW55XSA9IEJvZHkoLi4uKSwKKToKICAgIGlmICJtb2RlbCIgbm90IGluIHBheWxvYWQgb3IgcGF5bG9hZFsibW9kZWwiXSBpcyBOb25lOgogICAgICAgIHBheWxvYWRbIm1vZGVsIl0gPSBPUEVOQUlfREVGQVVMVF9NT0RFTAogICAgaGVhZGVycyA9IGJ1aWxkX2hlYWRlcnMoZGljdChyZXF1ZXN0LmhlYWRlcnMpKQogICAgcmVzcCA9IGZvcndhcmRfanNvbigiL3YxL3Jlc3BvbnNlcyIsIHBheWxvYWQsIGhlYWRlcnMsIGRpY3QocmVxdWVzdC5xdWVyeV9wYXJhbXMpKQogICAgcmV0dXJuIFJlc3BvbnNlKAogICAgICAgIGNvbnRlbnQ9cmVzcC5jb250ZW50LAogICAgICAgIHN0YXR1c19jb2RlPXJlc3Auc3RhdHVzX2NvZGUsCiAgICAgICAgbWVkaWFfdHlwZT1yZXNwLmhlYWRlcnMuZ2V0KCJDb250ZW50LVR5cGUiLCAiYXBwbGljYXRpb24vanNvbiIpLAogICAgKQoKCiMgLS0tLS0tLS0tLS0tLS0tLSBjbGllbnQgLS0tLS0tLS0tLS0tLS0tLQpjbGFzcyBPcGVuQUlQcm94eUNsaWVudDoKICAgICIiIgogICAgU2ltcGxlIGNsaWVudCBmb3IgdGhlIGxvY2FsIHByb3h5LgogICAgRGVmYXVsdCBiYXNlIHVybCBpcyBodHRwOi8vbG9jYWxob3N0OjgwMDAKICAgIElmIGFwaV9rZXkgaXMgbm90IHByb3ZpZGVkLCBpdCB1c2VzIE9QRU5BSV9BUElfS0VZIGZyb20gZW52aXJvbm1lbnQuCiAgICAiIiIKCiAgICBkZWYgX19pbml0X18oc2VsZiwgYmFzZV91cmw6IHN0ciA9ICJodHRwOi8vbG9jYWxob3N0OjgwMDAiLCBhcGlfa2V5OiBPcHRpb25hbFtzdHJdID0gTm9uZSk6CiAgICAgICAgc2VsZi5iYXNlX3VybCA9IGJhc2VfdXJsLnJzdHJpcCgiLyIpCiAgICAgICAgc2VsZi5hcGlfa2V5ID0gYXBpX2tleQoKICAgIGRlZiBfaGVhZGVycyhzZWxmKSAtPiBEaWN0W3N0ciwgc3RyXToKICAgICAgICBoZWFkZXJzID0geyJDb250ZW50LVR5cGUiOiAiYXBwbGljYXRpb24vanNvbiJ9CiAgICAgICAga2V5ID0gc2VsZi5hcGlfa2V5IG9yIG9zLmdldGVudigiT1BFTkFJX0FQSV9LRVkiLCAiIikKICAgICAgICBpZiBrZXk6CiAgICAgICAgICAgIGhlYWRlcnNbIkF1dGhvcml6YXRpb24iXSA9IGYiQmVhcmVyIHtrZXl9IgogICAgICAgIHJldHVybiBoZWFkZXJzCgogICAgZGVmIGNoYXQoc2VsZiwgbWVzc2FnZXM6IExpc3RbRGljdFtzdHIsIHN0cl1dLCBtb2RlbDogT3B0aW9uYWxbc3RyXSA9IE5vbmUpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgICAgIGJvZHk6IERpY3Rbc3RyLCBBbnldID0geyJtZXNzYWdlcyI6IG1lc3NhZ2VzfQogICAgICAgIGlmIG1vZGVsOgogICAgICAgICAgICBib2R5WyJtb2RlbCJdID0gbW9kZWwKICAgICAgICByZXNwID0gcmVxdWVzdHMucG9zdCgKICAgICAgICAgICAgZiJ7c2VsZi5iYXNlX3VybH0vdjEvY2hhdC9jb21wbGV0aW9ucyIsCiAgICAgICAgICAgIGhlYWRlcnM9c2VsZi5faGVhZGVycygpLAogICAgICAgICAgICBqc29uPWJvZHksCiAgICAgICAgICAgIHRpbWVvdXQ9NjAsCiAgICAgICAgKQogICAgICAgIHJlc3AucmFpc2VfZm9yX3N0YXR1cygpCiAgICAgICAgcmV0dXJuIHJlc3AuanNvbigpCgogICAgZGVmIGVtYmVkZGluZ3Moc2VsZiwgdGV4dDogQW55LCBtb2RlbDogT3B0aW9uYWxbc3RyXSA9IE5vbmUpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgICAgIGJvZHk6IERpY3Rbc3RyLCBBbnldID0geyJpbnB1dCI6IHRleHR9CiAgICAgICAgaWYgbW9kZWw6CiAgICAgICAgICAgIGJvZHlbIm1vZGVsIl0gPSBtb2RlbAogICAgICAgIHJlc3AgPSByZXF1ZXN0cy5wb3N0KAogICAgICAgICAgICBmIntzZWxmLmJhc2VfdXJsfS92MS9lbWJlZGRpbmdzIiwKICAgICAgICAgICAgaGVhZGVycz1zZWxmLl9oZWFkZXJzKCksCiAgICAgICAgICAgIGpzb249Ym9keSwKICAgICAgICAgICAgdGltZW91dD02MCwKICAgICAgICApCiAgICAgICAgcmVzcC5yYWlzZV9mb3Jfc3RhdHVzKCkKICAgICAgICByZXR1cm4gcmVzcC5qc29uKCkKCiAgICBkZWYgcmVzcG9uc2VzKHNlbGYsIGlucHV0X3RleHQ6IEFueSwgbW9kZWw6IE9wdGlvbmFsW3N0cl0gPSBOb25lKSAtPiBEaWN0W3N0ciwgQW55XToKICAgICAgICBib2R5OiBEaWN0W3N0ciwgQW55XSA9IHsiaW5wdXQiOiBpbnB1dF90ZXh0fQogICAgICAgIGlmIG1vZGVsOgogICAgICAgICAgICBib2R5WyJtb2RlbCJdID0gbW9kZWwKICAgICAgICByZXNwID0gcmVxdWVzdHMucG9zdCgKICAgICAgICAgICAgZiJ7c2VsZi5iYXNlX3VybH0vdjEvcmVzcG9uc2VzIiwKICAgICAgICAgICAgaGVhZGVycz1zZWxmLl9oZWFkZXJzKCksCiAgICAgICAgICAgIGpzb249Ym9keSwKICAgICAgICAgICAgdGltZW91dD02MCwKICAgICAgICApCiAgICAgICAgcmVzcC5yYWlzZV9mb3Jfc3RhdHVzKCkKICAgICAgICByZXR1cm4gcmVzcC5qc29uKCkKCgojIG9wdGlvbmFsIHF1aWNrIHNlbGYgdGVzdCB3aGVuIHJ1bm5pbmcgdGhpcyBmaWxlIGRpcmVjdGx5CmlmIF9fbmFtZV9fID09ICJfX21haW5fXyI6CiAgICAjIHN0YXJ0IHRoZSBzZXJ2ZXIgaW4gYW5vdGhlciB0ZXJtaW5hbCBmaXJzdDoKICAgICMgdXZpY29ybiBvcGVuYWlfcHJveHkub3BlbmFpOmFwcCAtLWhvc3QgMC4wLjAuMCAtLXBvcnQgODAwMCAtLXJlbG9hZAogICAgYyA9IE9wZW5BSVByb3h5Q2xpZW50KCkKICAgIHRyeToKICAgICAgICBwcmludCgiSGVhbHRoOiIsIHJlcXVlc3RzLmdldChmIntjLmJhc2VfdXJsfS8iKS5qc29uKCkpCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgcHJpbnQoIlNlcnZlciBub3QgcnVubmluZzoiLCBlKQo=" -CMD = r''' +CMD = r""" set -e python - <<'PY' import os, base64, pathlib @@ -34,23 +34,24 @@ --bind 0.0.0.0:8000 \ --worker-class uvicorn.workers.UvicornWorker \ --log-level info -'''.strip() +""".strip() + + class OpenAIModule: - def __init__(self,project): + def __init__(self, project): self.project = project - self.openai_proxy_app = self.project.set_function(name="openai",kind="application",image="python:3.11") - self.openai_proxy_app.with_requirements([ + self.openai_proxy_app = self.project.set_function( + name="openai", kind="application", image="python:3.11" + ) + self.openai_proxy_app.with_requirements( + [ "fastapi==0.124.0", "uvicorn[standard]==0.38.0", "gunicorn==23.0.0", "requests=2.32.5", - ]) - self.openai_proxy_app.set_env("BASE64",BASE64) + ] + ) + self.openai_proxy_app.set_env("BASE64", BASE64) self.openai_proxy_app.set_internal_application_port(8000) self.openai_proxy_app.spec.command = "/bin/sh" self.openai_proxy_app.spec.args = ["-c", CMD] - - - - - diff --git a/modules/src/openai_proxy_app/test_openai_proxy_app.py b/modules/src/openai_proxy_app/test_openai_proxy_app.py index 79fbc726a..957222325 100644 --- a/modules/src/openai_proxy_app/test_openai_proxy_app.py +++ b/modules/src/openai_proxy_app/test_openai_proxy_app.py @@ -13,8 +13,9 @@ # limitations under the License. # -from openai_proxy_app import OpenAIModule import mlrun +from openai_proxy_app import OpenAIModule + class TestOpenAIProxyApp: """Test suite for TestOpenAIProxyApp class.""" @@ -26,6 +27,7 @@ def setup_method(self): def test_openai_proxy_app(self): """Test do_tracking with various dataframe sizes using parametrized test.""" - assert type(self.TestOpenAIProxyApp.openai_proxy_app) == mlrun.runtimes.nuclio.application.application.ApplicationRuntime - - + assert ( + type(self.TestOpenAIProxyApp.openai_proxy_app) + == mlrun.runtimes.nuclio.application.application.ApplicationRuntime + ) diff --git a/modules/src/vllm_module/test_vllm_module.py b/modules/src/vllm_module/test_vllm_module.py index 3a5f422ae..4de1be16a 100644 --- a/modules/src/vllm_module/test_vllm_module.py +++ b/modules/src/vllm_module/test_vllm_module.py @@ -13,8 +13,8 @@ # limitations under the License. # -from vllm_module import VLLMModule import mlrun +from vllm_module import VLLMModule class TestVllmModule: @@ -30,6 +30,7 @@ def setup_method(self): ) def test_vllm_module(self): - assert ( - type(self.TestVllmModule.vllm_app) == mlrun.runtimes.nuclio.application.application.ApplicationRuntime + assert isinstance( + self.TestVllmModule.vllm_app, + mlrun.runtimes.nuclio.application.application.ApplicationRuntime, ) diff --git a/modules/src/vllm_module/vllm_module.py b/modules/src/vllm_module/vllm_module.py index 50bc9f038..39ecff28f 100644 --- a/modules/src/vllm_module/vllm_module.py +++ b/modules/src/vllm_module/vllm_module.py @@ -12,20 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. # -#This module acts as a lightweight gateway to OpenAI-compatible APIs. -#You can send chat prompts, create embeddings, or get model responses without worrying about authentication or endpoint differences. -#It simplifies access so you can test, analyze, or integrate AI features directly into your projects or notebooks with minimal setup. +# This module acts as a lightweight gateway to OpenAI-compatible APIs. +# You can send chat prompts, create embeddings, or get model responses without worrying about authentication or endpoint differences. +# It simplifies access so you can test, analyze, or integrate AI features directly into your projects or notebooks with minimal setup. -from typing import Dict, Optional, List - class VLLMModule: """ VLLMModule - + This module provides a lightweight wrapper for deploying a vLLM (OpenAI-compatible) large language model server as an MLRun application runtime. - + The VLLMModule is responsible for: - Creating an MLRun application runtime based on a vLLM container image - Configuring GPU resources, memory limits, and Kubernetes node selection @@ -34,35 +32,33 @@ class VLLMModule: - Automatically configuring shared memory (/dev/shm) when using multiple GPUs - Exposing an OpenAI-compatible API (e.g. /v1/chat/completions) for inference - Providing a simple Python interface for deployment and invocation from Jupyter notebooks - + The module is designed to be used in Jupyter notebooks and MLRun pipelines, allowing users to deploy and test large language models on Kubernetes with minimal configuration. """ def __init__( - self, - project: str, - *, - node_selector: Optional[Dict[str, str]] = None, - name: str = "vllm", - image: str = "vllm/vllm-openai:latest", - model: str = "Qwen/Qwen2.5-Omni-3B", - gpus: int = 1, - mem: str = "10G", - port: int = 8000, - dtype: str = "auto", - uvicorn_log_level: str = "info", - max_tokens: int = 500, + self, + project: str, + *, + node_selector: dict[str, str] | None = None, + name: str = "vllm", + image: str = "vllm/vllm-openai:latest", + model: str = "Qwen/Qwen2.5-Omni-3B", + gpus: int = 1, + mem: str = "10G", + port: int = 8000, + dtype: str = "auto", + uvicorn_log_level: str = "info", + max_tokens: int = 500, ): if gpus < 1: raise ValueError("gpus must be >= 1") - - if node_selector is None: node_selector = {"alpha.eksctl.io/nodegroup-name": "added-gpu"} - + if not isinstance(max_tokens, int): raise TypeError("max_tokens must be an integer") @@ -94,7 +90,7 @@ def __init__( self.vllm_app.set_internal_application_port(self.port) - args: List[str] = [ + args: list[str] = [ "serve", self.model, "--dtype", @@ -110,10 +106,12 @@ def __init__( args += ["--tensor-parallel-size", str(gpus)] # For more than one GPU you should create a share volume for the multiple GPUs - self.vllm_app.spec.volumes = [{"name": "dshm", "emptyDir": {"medium": "Memory"}}] - self.vllm_app.spec.volume_mounts = [{"name": "dshm", "mountPath": "/dev/shm"}] - - + self.vllm_app.spec.volumes = [ + {"name": "dshm", "emptyDir": {"medium": "Memory"}} + ] + self.vllm_app.spec.volume_mounts = [ + {"name": "dshm", "mountPath": "/dev/shm"} + ] self.vllm_app.spec.command = "vllm" self.vllm_app.spec.args = args @@ -124,8 +122,9 @@ def __init__( def get_runtime(self): return self.vllm_app - def add_args(self, extra_args: List[str]): - if not isinstance(extra_args, list) or not all(isinstance(x, str) for x in extra_args): + def add_args(self, extra_args: list[str]): + if not isinstance(extra_args, list) or not all( + isinstance(x, str) for x in extra_args + ): raise ValueError("extra_args must be a list of strings") self.vllm_app.spec.args += extra_args - diff --git a/pyproject.toml b/pyproject.toml index d7813821d..869e3356b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "mlrun-hub" version = "0.1.0" description = "MLRun Hub - centralized location for open source contributions of mlrun hub components" readme = "README.md" -requires-python = ">=3.11" +requires-python = ">=3.10.18" license = { file = "LICENSE" } authors = [ { name = "MLRun Team" } @@ -33,8 +33,12 @@ dependencies = [ mlrun-functions = "cli.cli:cli" [tool.ruff] -target-version = "py311" +target-version = "py310" required-version = ">=0.8.0" +exclude = [ + "**/*.ipynb", +] + [tool.ruff.lint] extend-select = [ diff --git a/steps/src/verify_schema/test_verify_schema.py b/steps/src/verify_schema/test_verify_schema.py index 5a7e08b53..bebb0a5b4 100644 --- a/steps/src/verify_schema/test_verify_schema.py +++ b/steps/src/verify_schema/test_verify_schema.py @@ -15,25 +15,19 @@ from verify_schema import VerifySchema + class TestVerifySchema: def test_verify_schema(self): schema = ["id", "name", "active"] verifier = VerifySchema(schema=schema, allow_unexpected_keys=False) # Test with valid event - event = { - "id": 1, - "name": "Test Event", - "active": True - } + event = {"id": 1, "name": "Test Event", "active": True} result = verifier.do(event) assert result == event # Test with missing key - event_missing_key = { - "id": 1, - "name": "Test Event" - } + event_missing_key = {"id": 1, "name": "Test Event"} try: verifier.do(event_missing_key) except KeyError as e: @@ -44,7 +38,7 @@ def test_verify_schema(self): "id": 1, "name": "Test Event", "active": True, - "extra": "unexpected" + "extra": "unexpected", } try: verifier.do(event_unexpected_key) @@ -56,11 +50,6 @@ def test_verify_schema_allow_unexpected(self): verifier = VerifySchema(schema=schema, allow_unexpected_keys=True) # Test with valid event and unexpected key - event = { - "id": 1, - "name": "Test Event", - "active": True, - "extra": "unexpected" - } + event = {"id": 1, "name": "Test Event", "active": True, "extra": "unexpected"} result = verifier.do(event) - assert result == event \ No newline at end of file + assert result == event diff --git a/steps/src/verify_schema/verify_schema.py b/steps/src/verify_schema/verify_schema.py index 80a379560..81cc46353 100644 --- a/steps/src/verify_schema/verify_schema.py +++ b/steps/src/verify_schema/verify_schema.py @@ -13,6 +13,7 @@ # limitations under the License. # + class VerifySchema: """ This step validates that an event dictionary contains exactly the keys defined in the schema, @@ -27,7 +28,9 @@ def do(self, event: dict): # Check if all keys in the expected schema are present in the event missing = set(self.schema) - set(event) if missing: - raise KeyError(f"Schema verification failed: missing keys {missing} in event: {event}") + raise KeyError( + f"Schema verification failed: missing keys {missing} in event: {event}" + ) if self.allow_unexpected_keys: return event @@ -35,6 +38,8 @@ def do(self, event: dict): # Check if there are any unexpected keys in the event unexpected = set(event) - set(self.schema) if unexpected: - raise KeyError(f"Schema verification failed: unexpected keys {unexpected} in event: {event}") + raise KeyError( + f"Schema verification failed: unexpected keys {unexpected} in event: {event}" + ) return event