diff --git a/functions/src/auto_trainer/auto_trainer.py b/functions/src/auto_trainer/auto_trainer.py index 7b4764700..4e53e5b7e 100755 --- a/functions/src/auto_trainer/auto_trainer.py +++ b/functions/src/auto_trainer/auto_trainer.py @@ -67,30 +67,14 @@ def _get_dataframe( Classification tasks. :param drop_columns: str/int or a list of strings/ints that represent the column names/indices to drop. """ - store_uri_prefix, _ = mlrun.datastore.parse_store_uri(dataset.artifact_url) - - # Getting the dataset: - if mlrun.utils.StorePrefix.FeatureVector == store_uri_prefix: - label_columns = label_columns or dataset.meta.status.label_column - context.logger.info(f"label columns: {label_columns}") - # FeatureVector case: - try: - fv = mlrun.datastore.get_store_resource(dataset.artifact_url) - dataset = fv.get_offline_features(drop_columns=drop_columns).to_dataframe() - except AttributeError: - # Leave here for backwards compatibility - dataset = fs.get_offline_features( - dataset.meta.uri, drop_columns=drop_columns - ).to_dataframe() - - elif not label_columns: - context.logger.info( - "label_columns not provided, mandatory when dataset is not a FeatureVector" - ) - raise ValueError - - elif isinstance(dataset, (list, dict)): + # Check if dataset is list/dict first (before trying to access artifact_url) + if isinstance(dataset, (list, dict)): # list/dict case: + if not label_columns: + context.logger.info( + "label_columns not provided, mandatory when dataset is not a FeatureVector" + ) + raise ValueError dataset = pd.DataFrame(dataset) # Checking if drop_columns provided by integer type: if drop_columns: @@ -103,17 +87,38 @@ def _get_dataframe( ) raise ValueError dataset.drop(drop_columns, axis=1, inplace=True) - else: - # simple URL case: - dataset = dataset.as_df() - if drop_columns: - if all(col in dataset for col in drop_columns): - dataset = dataset.drop(drop_columns, axis=1) - else: + # Dataset is a DataItem with artifact_url (URI or FeatureVector) + store_uri_prefix, _ = mlrun.datastore.parse_store_uri(dataset.artifact_url) + + # Getting the dataset: + if mlrun.utils.StorePrefix.FeatureVector == store_uri_prefix: + label_columns = label_columns or dataset.meta.status.label_column + context.logger.info(f"label columns: {label_columns}") + # FeatureVector case: + try: + fv = mlrun.datastore.get_store_resource(dataset.artifact_url) + dataset = fv.get_offline_features(drop_columns=drop_columns).to_dataframe() + except AttributeError: + # Leave here for backwards compatibility + dataset = fs.get_offline_features( + dataset.meta.uri, drop_columns=drop_columns + ).to_dataframe() + else: + # simple URL case: + if not label_columns: context.logger.info( - "not all of the columns to drop in the dataset, drop columns process skipped" + "label_columns not provided, mandatory when dataset is not a FeatureVector" ) + raise ValueError + dataset = dataset.as_df() + if drop_columns: + if all(col in dataset for col in drop_columns): + dataset = dataset.drop(drop_columns, axis=1) + else: + context.logger.info( + "not all of the columns to drop in the dataset, drop columns process skipped" + ) return dataset, label_columns diff --git a/functions/src/auto_trainer/function.yaml b/functions/src/auto_trainer/function.yaml index 0920b1033..50a36e750 100644 --- a/functions/src/auto_trainer/function.yaml +++ b/functions/src/auto_trainer/function.yaml @@ -1,22 +1,19 @@ -metadata: - categories: - - machine-learning - - model-training - tag: '' - name: auto-trainer +verbose: false +kind: job spec: - image: mlrun/mlrun - build: - origin_filename: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICBzdG9yZV91cmlfcHJlZml4LCBfID0gbWxydW4uZGF0YXN0b3JlLnBhcnNlX3N0b3JlX3VyaShkYXRhc2V0LmFydGlmYWN0X3VybCkKCiAgICAjIEdldHRpbmcgdGhlIGRhdGFzZXQ6CiAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IGxhYmVsX2NvbHVtbnMgb3IgZGF0YXNldC5tZXRhLnN0YXR1cy5sYWJlbF9jb2x1bW4KICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYibGFiZWwgY29sdW1uczoge2xhYmVsX2NvbHVtbnN9IikKICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICB0cnk6CiAgICAgICAgICAgIGZ2ID0gbWxydW4uZGF0YXN0b3JlLmdldF9zdG9yZV9yZXNvdXJjZShkYXRhc2V0LmFydGlmYWN0X3VybCkKICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZXhjZXB0IEF0dHJpYnV0ZUVycm9yOgogICAgICAgICAgICAjIExlYXZlIGhlcmUgZm9yIGJhY2t3YXJkcyBjb21wYXRpYmlsaXR5CiAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgIGRhdGFzZXQubWV0YS51cmksIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMKICAgICAgICAgICAgKS50b19kYXRhZnJhbWUoKQoKICAgIGVsaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgImxhYmVsX2NvbHVtbnMgbm90IHByb3ZpZGVkLCBtYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBlbGlmIGlzaW5zdGFuY2UoZGF0YXNldCwgKGxpc3QsIGRpY3QpKToKICAgICAgICAjIGxpc3QvZGljdCBjYXNlOgogICAgICAgIGRhdGFzZXQgPSBwZC5EYXRhRnJhbWUoZGF0YXNldCkKICAgICAgICAjIENoZWNraW5nIGlmIGRyb3BfY29sdW1ucyBwcm92aWRlZCBieSBpbnRlZ2VyIHR5cGU6CiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICBpZiBpc2luc3RhbmNlKGRyb3BfY29sdW1ucywgc3RyKSBvciAoCiAgICAgICAgICAgICAgICBpc2luc3RhbmNlKGRyb3BfY29sdW1ucywgbGlzdCkKICAgICAgICAgICAgICAgIGFuZCBhbnkoaXNpbnN0YW5jZShjb2wsIHN0cikgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpCiAgICAgICAgICAgICk6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIG11c3QgYmUgYW4gaW50ZWdlci9saXN0IG9mIGludGVnZXJzIGlmIG5vdCBwcm92aWRlZCB3aXRoIGEgVVJJL0ZlYXR1cmVWZWN0b3IgZGF0YXNldCIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xLCBpbnBsYWNlPVRydWUpCgogICAgZWxzZToKICAgICAgICAjIHNpbXBsZSBVUkwgY2FzZToKICAgICAgICBkYXRhc2V0ID0gZGF0YXNldC5hc19kZigpCiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSkKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgIm5vdCBhbGwgb2YgdGhlIGNvbHVtbnMgdG8gZHJvcCBpbiB0aGUgZGF0YXNldCwgZHJvcCBjb2x1bW5zIHByb2Nlc3Mgc2tpcHBlZCIKICAgICAgICAgICAgICAgICkKCiAgICByZXR1cm4gZGF0YXNldCwgbGFiZWxfY29sdW1ucwoKCmRlZiB0cmFpbigKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBtb2RlbF9jbGFzczogc3RyLAogICAgbGFiZWxfY29sdW1uczogT3B0aW9uYWxbVW5pb25bc3RyLCBMaXN0W3N0cl1dXSA9IE5vbmUsCiAgICBkcm9wX2NvbHVtbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSAibW9kZWwiLAogICAgdGFnOiBzdHIgPSAiIiwKICAgIHNhbXBsZV9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRlc3Rfc2V0OiBEYXRhSXRlbSA9IE5vbmUsCiAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemU6IGZsb2F0ID0gTm9uZSwKICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gTm9uZSwKICAgIGxhYmVsczogZGljdCA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgVHJhaW5pbmcgYSBtb2RlbCB3aXRoIHRoZSBnaXZlbiBkYXRhc2V0LgoKICAgIGV4YW1wbGU6OgoKICAgICAgICBpbXBvcnQgbWxydW4KICAgICAgICBwcm9qZWN0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9wcm9qZWN0KCJteS1wcm9qZWN0IikKICAgICAgICBwcm9qZWN0LnNldF9mdW5jdGlvbigiaHViOi8vYXV0b190cmFpbmVyIiwgInRyYWluIikKICAgICAgICB0cmFpbmVyX3J1biA9IHByb2plY3QucnVuKAogICAgICAgICAgICBuYW1lPSJ0cmFpbiIsCiAgICAgICAgICAgIGhhbmRsZXI9InRyYWluIiwKICAgICAgICAgICAgaW5wdXRzPXsiZGF0YXNldCI6ICIuL3BhdGgvdG8vZGF0YXNldC5jc3YifSwKICAgICAgICAgICAgcGFyYW1zPXsKICAgICAgICAgICAgICAgICJtb2RlbF9jbGFzcyI6ICJza2xlYXJuLmxpbmVhcl9tb2RlbC5Mb2dpc3RpY1JlZ3Jlc3Npb24iLAogICAgICAgICAgICAgICAgImxhYmVsX2NvbHVtbnMiOiAibGFiZWwiLAogICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyI6ICJpZCIsCiAgICAgICAgICAgICAgICAibW9kZWxfbmFtZSI6ICJteS1tb2RlbCIsCiAgICAgICAgICAgICAgICAidGFnIjogInYxLjAuMCIsCiAgICAgICAgICAgICAgICAic2FtcGxlX3NldCI6ICIuL3BhdGgvdG8vc2FtcGxlX3NldC5jc3YiLAogICAgICAgICAgICAgICAgInRlc3Rfc2V0IjogIi4vcGF0aC90by90ZXN0X3NldC5jc3YiLAogICAgICAgICAgICAgICAgIkNMQVNTX3NvbHZlciI6ICJsaWJsaW5lYXIiLAogICAgICAgICAgICB9LAogICAgICAgICkKCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQKICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3RvcgogICAgOnBhcmFtIG1vZGVsX2NsYXNzOiAgICAgICAgICAgICBUaGUgY2xhc3Mgb2YgdGhlIG1vZGVsLCBlLmcuIGBza2xlYXJuLmxpbmVhcl9tb2RlbC5Mb2dpc3RpY1JlZ3Jlc3Npb25gCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uczogICAgICAgICAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2xhc3NpZmljYXRpb24gdGFza3MuIE1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyIG9yIGEgbGlzdCBvZiBzdHJpbmdzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW5zIHRvIGRyb3AKICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgICAgICAgICAgICAgVGhlIG1vZGVsJ3MgbmFtZSB0byB1c2UgZm9yIHN0b3JpbmcgdGhlIG1vZGVsIGFydGlmYWN0LCBkZWZhdWx0IHRvICdtb2RlbCcKICAgIDpwYXJhbSB0YWc6ICAgICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsJ3MgdGFnIHRvIGxvZyB3aXRoCiAgICA6cGFyYW0gc2FtcGxlX3NldDogICAgICAgICAgICAgIEEgc2FtcGxlIHNldCBvZiBpbnB1dHMgZm9yIHRoZSBtb2RlbCBmb3IgbG9nZ2luZyBpdHMgc3RhdHMgYWxvbmcgdGhlIG1vZGVsIGluIGZhdm91cgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvZiBtb2RlbCBtb25pdG9yaW5nLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3RvcgogICAgOnBhcmFtIHRlc3Rfc2V0OiAgICAgICAgICAgICAgICBUaGUgdGVzdCBzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIHdpdGguCiAgICA6cGFyYW0gdHJhaW5fdGVzdF9zcGxpdF9zaXplOiAgIGlmIHRlc3Rfc2V0IHdhcyBwcm92aWRlZCB0aGVuIHRoaXMgYXJndW1lbnQgaXMgaWdub3JlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgU2hvdWxkIGJlIGJldHdlZW4gMC4wIGFuZCAxLjAgYW5kIHJlcHJlc2VudCB0aGUgcHJvcG9ydGlvbiBvZiB0aGUgZGF0YXNldCB0byBpbmNsdWRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGluIHRoZSB0ZXN0IHNwbGl0LiBUaGUgc2l6ZSBvZiB0aGUgVHJhaW5pbmcgc2V0IGlzIHNldCB0byB0aGUgY29tcGxlbWVudCBvZiB0aGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHZhbHVlLiBEZWZhdWx0ID0gMC4yCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICAgICAgIFJlbGV2YW50IG9ubHkgd2hlbiB1c2luZyB0cmFpbl90ZXN0X3NwbGl0X3NpemUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIEEgcmFuZG9tIHN0YXRlIHNlZWQgdG8gc2h1ZmZsZSB0aGUgZGF0YS4gRm9yIG1vcmUgaW5mb3JtYXRpb24sIHNlZToKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaHR0cHM6Ly9zY2lraXQtbGVhcm4ub3JnL3N0YWJsZS9nbG9zc2FyeS5odG1sI3Rlcm0tcmFuZG9tX3N0YXRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE5vdGljZSB0aGF0IGhlcmUgd2Ugb25seSBwYXNzIGludGVnZXIgdmFsdWVzLgogICAgOnBhcmFtIGxhYmVsczogICAgICAgICAgICAgICAgICBMYWJlbHMgdG8gbG9nIHdpdGggdGhlIG1vZGVsCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHdpdGggcHJlZml4ZXMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoYXQgd2lsbCBiZSBwYXJzZWQgYW5kIHBhc3NlZCB0byB0aGUgcmVsZXZhbnQgZnVuY3Rpb24sIGJ5IHRoZSBmb2xsb3dpbmcgcHJlZml4ZXM6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYENMQVNTX2AgLSBmb3IgdGhlIG1vZGVsIGNsYXNzIGFyZ3VtZW50cwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAtIGBGSVRfYCAtIGZvciB0aGUgYGZpdGAgZnVuY3Rpb24gYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYFRSQUlOX2AgLSBmb3IgdGhlIGB0cmFpbmAgZnVuY3Rpb24gKGluIHhnYiBvciBsZ2JtIHRyYWluIGZ1bmN0aW9uIC0gZnV0dXJlKQoKICAgICIiIgogICAgIyBWYWxpZGF0ZSBpbnB1dHM6CiAgICAjIENoZWNrIGlmIGV4YWN0bHkgb25lIG9mIHRoZW0gaXMgc3VwcGxpZWQ6CiAgICBpZiB0ZXN0X3NldCBpcyBOb25lOgogICAgICAgIGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSBpcyBOb25lOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgInRlc3Rfc2V0IG9yIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSBhcmUgbm90IHByb3ZpZGVkLCBzZXR0aW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB0byAwLjIiCiAgICAgICAgICAgICkKICAgICAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gMC4yCgogICAgZWxpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgInRlc3Rfc2V0IHByb3ZpZGVkLCBpZ25vcmluZyBnaXZlbiB0cmFpbl90ZXN0X3NwbGl0X3NpemUgdmFsdWUiCiAgICAgICAgKQogICAgICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSA9IE5vbmUKCiAgICAjIEdldCBEYXRhRnJhbWUgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIEdldHRpbmcgdGhlIHNhbXBsZSBzZXQ6CiAgICBpZiBzYW1wbGVfc2V0IGlzIE5vbmU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgZiJTYW1wbGUgc2V0IG5vdCBnaXZlbiwgdXNpbmcgdGhlIHdob2xlIHRyYWluaW5nIHNldCBhcyB0aGUgc2FtcGxlIHNldCIKICAgICAgICApCiAgICAgICAgc2FtcGxlX3NldCA9IGRhdGFzZXQKICAgIGVsc2U6CiAgICAgICAgc2FtcGxlX3NldCwgXyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgICAgIGRhdGFzZXQ9c2FtcGxlX3NldCwKICAgICAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgICAgICkKCiAgICAjIFBhcnNpbmcga3dhcmdzOgogICAgIyBUT0RPOiBVc2UgaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24uCiAgICB0cmFpbl9rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeChzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLlRSQUlOKQogICAgZml0X2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuRklUKQogICAgbW9kZWxfY2xhc3Nfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoCiAgICAgICAgc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5NT0RFTF9DTEFTUwogICAgKQoKICAgICMgQ2hlY2sgaWYgbW9kZWwgb3IgZnVuY3Rpb246CiAgICBpZiBoYXNhdHRyKG1vZGVsX2NsYXNzLCAidHJhaW4iKToKICAgICAgICAjIFRPRE86IE5lZWQgdG8gY2FsbDogbW9kZWwoKSwgYWZ0ZXJ3YXJkcyB0byBzdGFydCB0aGUgdHJhaW4gZnVuY3Rpb24uCiAgICAgICAgIyBtb2RlbCA9IGNyZWF0ZV9mdW5jdGlvbihmInttb2RlbF9jbGFzc30udHJhaW4iKQogICAgICAgIHJhaXNlIE5vdEltcGxlbWVudGVkRXJyb3IKICAgIGVsc2U6CiAgICAgICAgIyBDcmVhdGluZyBtb2RlbCBpbnN0YW5jZToKICAgICAgICBtb2RlbCA9IGNyZWF0ZV9jbGFzcyhtb2RlbF9jbGFzcykoKiptb2RlbF9jbGFzc19rd2FyZ3MpCgogICAgeCA9IGRhdGFzZXQuZHJvcChsYWJlbF9jb2x1bW5zLCBheGlzPTEpCiAgICB5ID0gZGF0YXNldFtsYWJlbF9jb2x1bW5zXQogICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplOgogICAgICAgIHhfdHJhaW4sIHhfdGVzdCwgeV90cmFpbiwgeV90ZXN0ID0gdHJhaW5fdGVzdF9zcGxpdCgKICAgICAgICAgICAgeCwgeSwgdGVzdF9zaXplPXRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSwgcmFuZG9tX3N0YXRlPXJhbmRvbV9zdGF0ZQogICAgICAgICkKICAgIGVsc2U6CiAgICAgICAgeF90cmFpbiwgeV90cmFpbiA9IHgsIHkKCiAgICAgICAgdGVzdF9zZXQgPSB0ZXN0X3NldC5hc19kZigpCiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICB0ZXN0X3NldCA9IGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSkKCiAgICAgICAgeF90ZXN0LCB5X3Rlc3QgPSB0ZXN0X3NldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSksIHRlc3Rfc2V0W2xhYmVsX2NvbHVtbnNdCgogICAgQXV0b01MUnVuLmFwcGx5X21scnVuKAogICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgIG1vZGVsX25hbWU9bW9kZWxfbmFtZSwKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgdGFnPXRhZywKICAgICAgICBzYW1wbGVfc2V0PXNhbXBsZV9zZXQsCiAgICAgICAgeV9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgdGVzdF9zZXQ9dGVzdF9zZXQsCiAgICAgICAgeF90ZXN0PXhfdGVzdCwKICAgICAgICB5X3Rlc3Q9eV90ZXN0LAogICAgICAgIGFydGlmYWN0cz1jb250ZXh0LmFydGlmYWN0cywKICAgICAgICBsYWJlbHM9bGFiZWxzLAogICAgKQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInRyYWluaW5nICd7bW9kZWxfbmFtZX0nIikKICAgIG1vZGVsLmZpdCh4X3RyYWluLCB5X3RyYWluLCAqKmZpdF9rd2FyZ3MpCgoKZGVmIGV2YWx1YXRlKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBtb2RlbDogc3RyLAogICAgZGF0YXNldDogbWxydW4uRGF0YUl0ZW0sCiAgICBkcm9wX2NvbHVtbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBFdmFsdWF0aW5nIGEgbW9kZWwuIEFydGlmYWN0cyBnZW5lcmF0ZWQgYnkgdGhlIE1MSGFuZGxlci4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gbW9kZWw6ICAgICAgICAgICAgICAgICAgIFRoZSBtb2RlbCBTdG9yZSBwYXRoLgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byBldmFsdWF0ZSB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcC4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIFBhcnNpbmcgbGFiZWxfY29sdW1uczoKICAgIHBhcnNlZF9sYWJlbF9jb2x1bW5zID0gW10KICAgIGlmIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9ICgKICAgICAgICAgICAgbGFiZWxfY29sdW1ucyBpZiBpc2luc3RhbmNlKGxhYmVsX2NvbHVtbnMsIGxpc3QpIGVsc2UgW2xhYmVsX2NvbHVtbnNdCiAgICAgICAgKQogICAgICAgIGZvciBsYyBpbiBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICBpZiBmcy5jb21tb24uZmVhdHVyZV9zZXBhcmF0b3IgaW4gbGM6CiAgICAgICAgICAgICAgICBmZWF0dXJlX3NldF9uYW1lLCBsYWJlbF9uYW1lLCBhbGlhcyA9IGZzLmNvbW1vbi5wYXJzZV9mZWF0dXJlX3N0cmluZyhsYykKICAgICAgICAgICAgICAgIHBhcnNlZF9sYWJlbF9jb2x1bW5zLmFwcGVuZChhbGlhcyBvciBsYWJlbF9uYW1lKQogICAgICAgIGlmIHBhcnNlZF9sYWJlbF9jb2x1bW5zOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gcGFyc2VkX2xhYmVsX2NvbHVtbnMKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCgogICAgIyBMb2FkaW5nIHRoZSBtb2RlbCBhbmQgcHJlZGljdGluZzoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbCgKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQsIG1vZGVsX25hbWU9Im1vZGVsX0xpbmVhclJlZ3Jlc3Npb24iCiAgICApCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4obW9kZWxfaGFuZGxlci5tb2RlbCwgeV90ZXN0PXksIG1vZGVsX3BhdGg9bW9kZWwpCgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmImV2YWx1YXRpbmcgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoeCwgKiprd2FyZ3MpCgoKZGVmIHByZWRpY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIHJlc3VsdF9zZXQ6IE9wdGlvbmFsW3N0cl0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIFByZWRpY3RpbmcgZGF0YXNldCBieSBhIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIHByZWRpY3QgdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJLCBhIEZlYXR1cmVWZWN0b3Igb3IgYQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgaW4gYSBzaGFwZSBvZiBhIGxpc3QvZGljdC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV2hlbiBwYXNzaW5nIGEgc2FtcGxlLCBwYXNzIHRoZSBkYXRhc2V0IGFzIGEgZmllbGQgaW4gYHBhcmFtc2AgaW5zdGVhZCBvZiBgaW5wdXRzYC4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHRoZSBkYXRhc2V0IGlzIGEgbGlzdC9kaWN0IHRoaXMgcGFyYW1ldGVyIHNob3VsZCBiZSByZXByZXNlbnRlZCBieSBpbnRlZ2Vycy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIHJlc3VsdF9zZXQ6ICAgICAgICAgICAgICBUaGUgZGIga2V5IHRvIHNldCBuYW1lIG9mIHRoZSBwcmVkaWN0aW9uIHJlc3VsdCBhbmQgdGhlIGZpbGVuYW1lLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0IHRvICdwcmVkaWN0aW9uJy4KICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgdG8gdGhlIHByZWRpY3QgZnVuY3Rpb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKFBSRURJQ1RfIHByZWZpeCBpcyBub3QgcmVxdWlyZWQpLgogICAgIiIiCiAgICAjIEdldCBkYXRhc2V0IGJ5IFVSTCBvciBieSBGZWF0dXJlVmVjdG9yOgogICAgZGF0YXNldCwgbGFiZWxfY29sdW1ucyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICBkYXRhc2V0PWRhdGFzZXQsCiAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICApCgogICAgIyBsb2FkaW5nIHRoZSBtb2RlbCwgYW5kIGdldHRpbmcgdGhlIG1vZGVsIGhhbmRsZXI6CiAgICBtb2RlbF9oYW5kbGVyID0gQXV0b01MUnVuLmxvYWRfbW9kZWwobW9kZWxfcGF0aD1tb2RlbCwgY29udGV4dD1jb250ZXh0KQoKICAgICMgRHJvcHBpbmcgbGFiZWwgY29sdW1ucyBpZiBuZWNlc3Nhcnk6CiAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW10KICAgIGVsaWYgaXNpbnN0YW5jZShsYWJlbF9jb2x1bW5zLCBzdHIpOgogICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbbGFiZWxfY29sdW1uc10KCiAgICAjIFByZWRpY3Rpbmc6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYibWFraW5nIHByZWRpY3Rpb24gYnkgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICB5X3ByZWQgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoZGF0YXNldCwgKiprd2FyZ3MpCgogICAgIyBQcmVwYXJpbmcgYW5kIHZhbGlkYXRpbmcgbGFiZWwgY29sdW1ucyBmb3IgdGhlIGRhdGFmcmFtZSBvZiB0aGUgcHJlZGljdGlvbiByZXN1bHQ6CiAgICBudW1fcHJlZGljdGVkID0gMSBpZiBsZW4oeV9wcmVkLnNoYXBlKSA9PSAxIGVsc2UgeV9wcmVkLnNoYXBlWzFdCgogICAgaWYgbnVtX3ByZWRpY3RlZCA+IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBpZiBudW1fcHJlZGljdGVkID09IDE6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbInByZWRpY3RlZCBsYWJlbHMiXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMuZXh0ZW5kKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIGYicHJlZGljdGVkX2xhYmVsX3tpICsgMSArIGxlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgICAgICAgICAgICAgZm9yIGkgaW4gcmFuZ2UobnVtX3ByZWRpY3RlZCAtIGxlbihsYWJlbF9jb2x1bW5zKSkKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgZWxpZiBudW1fcHJlZGljdGVkIDwgbGVuKGxhYmVsX2NvbHVtbnMpOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIm51bWJlciBvZiBwcmVkaWN0ZWQgbGFiZWxzOiB7bnVtX3ByZWRpY3RlZH0gaXMgc21hbGxlciB0aGFuIG51bWJlciBvZiBsYWJlbCBjb2x1bW5zOiB7bGVuKGxhYmVsX2NvbHVtbnMpfSIKICAgICAgICApCiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcgoKICAgIGFydGlmYWN0X25hbWUgPSByZXN1bHRfc2V0IG9yICJwcmVkaWN0aW9uIgogICAgbGFiZWxzX2luc2lkZV9kZiA9IHNldChsYWJlbF9jb2x1bW5zKSAmIHNldChkYXRhc2V0LmNvbHVtbnMudG9saXN0KCkpCiAgICBpZiBsYWJlbHNfaW5zaWRlX2RmOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIlRoZSBsYWJlbHM6IHtsYWJlbHNfaW5zaWRlX2RmfSBhcmUgYWxyZWFkeSBleGlzdGVkIGluIHRoZSBkYXRhZnJhbWUiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgIHByZWRfZGYgPSBwZC5jb25jYXQoW2RhdGFzZXQsIHBkLkRhdGFGcmFtZSh5X3ByZWQsIGNvbHVtbnM9bGFiZWxfY29sdW1ucyldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KGFydGlmYWN0X25hbWUsIHByZWRfZGYsIGRiX2tleT1yZXN1bHRfc2V0KQo= - code_origin: '' - description: Automatic train, evaluate and predict functions for the ML frameworks - - Scikit-Learn, XGBoost and LightGBM. - disable_auto_mount: false - default_handler: train entry_points: train: - lineno: 121 + doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ + \ project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ + hub://auto_trainer\", \"train\")\n trainer_run = project.run(\n \ + \ name=\"train\",\n handler=\"train\",\n inputs={\"dataset\"\ + : \"./path/to/dataset.csv\"},\n params={\n \"model_class\"\ + : \"sklearn.linear_model.LogisticRegression\",\n \"label_columns\"\ + : \"label\",\n \"drop_columns\": \"id\",\n \"model_name\"\ + : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ + : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ + ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" + has_kwargs: true parameters: - name: context type: MLClientCtx @@ -70,21 +67,12 @@ spec: type: dict doc: Labels to log with the model default: null - has_varargs: false + lineno: 126 name: train - has_kwargs: true - doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ - \ project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ - hub://auto_trainer\", \"train\")\n trainer_run = project.run(\n \ - \ name=\"train\",\n handler=\"train\",\n inputs={\"dataset\"\ - : \"./path/to/dataset.csv\"},\n params={\n \"model_class\"\ - : \"sklearn.linear_model.LogisticRegression\",\n \"label_columns\"\ - : \"label\",\n \"drop_columns\": \"id\",\n \"model_name\"\ - : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ - : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ - ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" + has_varargs: false evaluate: - lineno: 273 + doc: Evaluating a model. Artifacts generated by the MLHandler. + has_kwargs: true parameters: - name: context type: MLClientCtx @@ -104,12 +92,12 @@ spec: doc: The target label(s) of the column(s) in the dataset. for Regression or Classification tasks. Mandatory when dataset is not a FeatureVector. default: null - has_varargs: false + lineno: 278 name: evaluate - has_kwargs: true - doc: Evaluating a model. Artifacts generated by the MLHandler. + has_varargs: false predict: - lineno: 327 + doc: Predicting dataset by a model. + has_kwargs: true parameters: - name: context type: MLClientCtx @@ -138,10 +126,22 @@ spec: doc: The db key to set name of the prediction result and the filename. Default to 'prediction'. default: null - has_varargs: false + lineno: 332 name: predict - has_kwargs: true - doc: Predicting dataset by a model. + has_varargs: false + build: + code_origin: '' + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICAjIENoZWNrIGlmIGRhdGFzZXQgaXMgbGlzdC9kaWN0IGZpcnN0IChiZWZvcmUgdHJ5aW5nIHRvIGFjY2VzcyBhcnRpZmFjdF91cmwpCiAgICBpZiBpc2luc3RhbmNlKGRhdGFzZXQsIChsaXN0LCBkaWN0KSk6CiAgICAgICAgIyBsaXN0L2RpY3QgY2FzZToKICAgICAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICJsYWJlbF9jb2x1bW5zIG5vdCBwcm92aWRlZCwgbWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yIgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICBkYXRhc2V0ID0gcGQuRGF0YUZyYW1lKGRhdGFzZXQpCiAgICAgICAgIyBDaGVja2luZyBpZiBkcm9wX2NvbHVtbnMgcHJvdmlkZWQgYnkgaW50ZWdlciB0eXBlOgogICAgICAgIGlmIGRyb3BfY29sdW1uczoKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIHN0cikgb3IgKAogICAgICAgICAgICAgICAgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIGxpc3QpCiAgICAgICAgICAgICAgICBhbmQgYW55KGlzaW5zdGFuY2UoY29sLCBzdHIpIGZvciBjb2wgaW4gZHJvcF9jb2x1bW5zKQogICAgICAgICAgICApOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyBtdXN0IGJlIGFuIGludGVnZXIvbGlzdCBvZiBpbnRlZ2VycyBpZiBub3QgcHJvdmlkZWQgd2l0aCBhIFVSSS9GZWF0dXJlVmVjdG9yIGRhdGFzZXQiCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICAgICAgICAgIGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSwgaW5wbGFjZT1UcnVlKQogICAgZWxzZToKICAgICAgICAjIERhdGFzZXQgaXMgYSBEYXRhSXRlbSB3aXRoIGFydGlmYWN0X3VybCAoVVJJIG9yIEZlYXR1cmVWZWN0b3IpCiAgICAgICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGF0YXNldC5hcnRpZmFjdF91cmwpCgogICAgICAgICMgR2V0dGluZyB0aGUgZGF0YXNldDoKICAgICAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBsYWJlbF9jb2x1bW5zIG9yIGRhdGFzZXQubWV0YS5zdGF0dXMubGFiZWxfY29sdW1uCiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJsYWJlbCBjb2x1bW5zOiB7bGFiZWxfY29sdW1uc30iKQogICAgICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnYgPSBtbHJ1bi5kYXRhc3RvcmUuZ2V0X3N0b3JlX3Jlc291cmNlKGRhdGFzZXQuYXJ0aWZhY3RfdXJsKQogICAgICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgICAgIGV4Y2VwdCBBdHRyaWJ1dGVFcnJvcjoKICAgICAgICAgICAgICAgICMgTGVhdmUgaGVyZSBmb3IgYmFja3dhcmRzIGNvbXBhdGliaWxpdHkKICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgICAgICBkYXRhc2V0Lm1ldGEudXJpLCBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zCiAgICAgICAgICAgICAgICApLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgIyBzaW1wbGUgVVJMIGNhc2U6CiAgICAgICAgICAgIGlmIG5vdCBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyBub3QgcHJvdmlkZWQsIG1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3RvciIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuYXNfZGYoKQogICAgICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBkYXRhc2V0LmRyb3AoZHJvcF9jb2x1bW5zLCBheGlzPTEpCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgICAgICJub3QgYWxsIG9mIHRoZSBjb2x1bW5zIHRvIGRyb3AgaW4gdGhlIGRhdGFzZXQsIGRyb3AgY29sdW1ucyBwcm9jZXNzIHNraXBwZWQiCiAgICAgICAgICAgICAgICAgICAgKQoKICAgIHJldHVybiBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zCgoKZGVmIHRyYWluKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIG1vZGVsX2NsYXNzOiBzdHIsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIG1vZGVsX25hbWU6IHN0ciA9ICJtb2RlbCIsCiAgICB0YWc6IHN0ciA9ICIiLAogICAgc2FtcGxlX3NldDogRGF0YUl0ZW0gPSBOb25lLAogICAgdGVzdF9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZTogZmxvYXQgPSBOb25lLAogICAgcmFuZG9tX3N0YXRlOiBpbnQgPSBOb25lLAogICAgbGFiZWxzOiBkaWN0ID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBUcmFpbmluZyBhIG1vZGVsIHdpdGggdGhlIGdpdmVuIGRhdGFzZXQuCgogICAgZXhhbXBsZTo6CgogICAgICAgIGltcG9ydCBtbHJ1bgogICAgICAgIHByb2plY3QgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX3Byb2plY3QoIm15LXByb2plY3QiKQogICAgICAgIHByb2plY3Quc2V0X2Z1bmN0aW9uKCJodWI6Ly9hdXRvX3RyYWluZXIiLCAidHJhaW4iKQogICAgICAgIHRyYWluZXJfcnVuID0gcHJvamVjdC5ydW4oCiAgICAgICAgICAgIG5hbWU9InRyYWluIiwKICAgICAgICAgICAgaGFuZGxlcj0idHJhaW4iLAogICAgICAgICAgICBpbnB1dHM9eyJkYXRhc2V0IjogIi4vcGF0aC90by9kYXRhc2V0LmNzdiJ9LAogICAgICAgICAgICBwYXJhbXM9ewogICAgICAgICAgICAgICAgIm1vZGVsX2NsYXNzIjogInNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyI6ICJsYWJlbCIsCiAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIjogImlkIiwKICAgICAgICAgICAgICAgICJtb2RlbF9uYW1lIjogIm15LW1vZGVsIiwKICAgICAgICAgICAgICAgICJ0YWciOiAidjEuMC4wIiwKICAgICAgICAgICAgICAgICJzYW1wbGVfc2V0IjogIi4vcGF0aC90by9zYW1wbGVfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAidGVzdF9zZXQiOiAiLi9wYXRoL3RvL3Rlc3Rfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAiQ0xBU1Nfc29sdmVyIjogImxpYmxpbmVhciIsCiAgICAgICAgICAgIH0sCiAgICAgICAgKQoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dAogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICAgICAgICAgIFRoZSBjbGFzcyBvZiB0aGUgbW9kZWwsIGUuZy4gYHNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbmAKICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcAogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICAgICBUaGUgbW9kZWwncyBuYW1lIHRvIHVzZSBmb3Igc3RvcmluZyB0aGUgbW9kZWwgYXJ0aWZhY3QsIGRlZmF1bHQgdG8gJ21vZGVsJwogICAgOnBhcmFtIHRhZzogICAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwncyB0YWcgdG8gbG9nIHdpdGgKICAgIDpwYXJhbSBzYW1wbGVfc2V0OiAgICAgICAgICAgICAgQSBzYW1wbGUgc2V0IG9mIGlucHV0cyBmb3IgdGhlIG1vZGVsIGZvciBsb2dnaW5nIGl0cyBzdGF0cyBhbG9uZyB0aGUgbW9kZWwgaW4gZmF2b3VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9mIG1vZGVsIG1vbml0b3JpbmcuIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gdGVzdF9zZXQ6ICAgICAgICAgICAgICAgIFRoZSB0ZXN0IHNldCB0byB0cmFpbiB0aGUgbW9kZWwgd2l0aC4KICAgIDpwYXJhbSB0cmFpbl90ZXN0X3NwbGl0X3NpemU6ICAgaWYgdGVzdF9zZXQgd2FzIHByb3ZpZGVkIHRoZW4gdGhpcyBhcmd1bWVudCBpcyBpZ25vcmVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTaG91bGQgYmUgYmV0d2VlbiAwLjAgYW5kIDEuMCBhbmQgcmVwcmVzZW50IHRoZSBwcm9wb3J0aW9uIG9mIHRoZSBkYXRhc2V0IHRvIGluY2x1ZGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaW4gdGhlIHRlc3Qgc3BsaXQuIFRoZSBzaXplIG9mIHRoZSBUcmFpbmluZyBzZXQgaXMgc2V0IHRvIHRoZSBjb21wbGVtZW50IG9mIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWUuIERlZmF1bHQgPSAwLjIKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgUmVsZXZhbnQgb25seSB3aGVuIHVzaW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQSByYW5kb20gc3RhdGUgc2VlZCB0byBzaHVmZmxlIHRoZSBkYXRhLiBGb3IgbW9yZSBpbmZvcm1hdGlvbiwgc2VlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL3NjaWtpdC1sZWFybi5vcmcvc3RhYmxlL2dsb3NzYXJ5Lmh0bWwjdGVybS1yYW5kb21fc3RhdGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTm90aWNlIHRoYXQgaGVyZSB3ZSBvbmx5IHBhc3MgaW50ZWdlciB2YWx1ZXMuCiAgICA6cGFyYW0gbGFiZWxzOiAgICAgICAgICAgICAgICAgIExhYmVscyB0byBsb2cgd2l0aCB0aGUgbW9kZWwKICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgd2l0aCBwcmVmaXhlcywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCB3aWxsIGJlIHBhcnNlZCBhbmQgcGFzc2VkIHRvIHRoZSByZWxldmFudCBmdW5jdGlvbiwgYnkgdGhlIGZvbGxvd2luZyBwcmVmaXhlczoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgQ0xBU1NfYCAtIGZvciB0aGUgbW9kZWwgY2xhc3MgYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYEZJVF9gIC0gZm9yIHRoZSBgZml0YCBmdW5jdGlvbiBhcmd1bWVudHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgVFJBSU5fYCAtIGZvciB0aGUgYHRyYWluYCBmdW5jdGlvbiAoaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24gLSBmdXR1cmUpCgogICAgIiIiCiAgICAjIFZhbGlkYXRlIGlucHV0czoKICAgICMgQ2hlY2sgaWYgZXhhY3RseSBvbmUgb2YgdGhlbSBpcyBzdXBwbGllZDoKICAgIGlmIHRlc3Rfc2V0IGlzIE5vbmU6CiAgICAgICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGlzIE5vbmU6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAidGVzdF9zZXQgb3IgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGFyZSBub3QgcHJvdmlkZWQsIHNldHRpbmcgdHJhaW5fdGVzdF9zcGxpdF9zaXplIHRvIDAuMiIKICAgICAgICAgICAgKQogICAgICAgICAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemUgPSAwLjIKCiAgICBlbGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAidGVzdF9zZXQgcHJvdmlkZWQsIGlnbm9yaW5nIGdpdmVuIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB2YWx1ZSIKICAgICAgICApCiAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gTm9uZQoKICAgICMgR2V0IERhdGFGcmFtZSBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgR2V0dGluZyB0aGUgc2FtcGxlIHNldDoKICAgIGlmIHNhbXBsZV9zZXQgaXMgTm9uZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIlNhbXBsZSBzZXQgbm90IGdpdmVuLCB1c2luZyB0aGUgd2hvbGUgdHJhaW5pbmcgc2V0IGFzIHRoZSBzYW1wbGUgc2V0IgogICAgICAgICkKICAgICAgICBzYW1wbGVfc2V0ID0gZGF0YXNldAogICAgZWxzZToKICAgICAgICBzYW1wbGVfc2V0LCBfID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICAgICAgZGF0YXNldD1zYW1wbGVfc2V0LAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICAgICAgKQoKICAgICMgUGFyc2luZyBrd2FyZ3M6CiAgICAjIFRPRE86IFVzZSBpbiB4Z2Igb3IgbGdibSB0cmFpbiBmdW5jdGlvbi4KICAgIHRyYWluX2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuVFJBSU4pCiAgICBmaXRfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5GSVQpCiAgICBtb2RlbF9jbGFzc19rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeCgKICAgICAgICBzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLk1PREVMX0NMQVNTCiAgICApCgogICAgIyBDaGVjayBpZiBtb2RlbCBvciBmdW5jdGlvbjoKICAgIGlmIGhhc2F0dHIobW9kZWxfY2xhc3MsICJ0cmFpbiIpOgogICAgICAgICMgVE9ETzogTmVlZCB0byBjYWxsOiBtb2RlbCgpLCBhZnRlcndhcmRzIHRvIHN0YXJ0IHRoZSB0cmFpbiBmdW5jdGlvbi4KICAgICAgICAjIG1vZGVsID0gY3JlYXRlX2Z1bmN0aW9uKGYie21vZGVsX2NsYXNzfS50cmFpbiIpCiAgICAgICAgcmFpc2UgTm90SW1wbGVtZW50ZWRFcnJvcgogICAgZWxzZToKICAgICAgICAjIENyZWF0aW5nIG1vZGVsIGluc3RhbmNlOgogICAgICAgIG1vZGVsID0gY3JlYXRlX2NsYXNzKG1vZGVsX2NsYXNzKSgqKm1vZGVsX2NsYXNzX2t3YXJncykKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCiAgICBpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgeF90cmFpbiwgeF90ZXN0LCB5X3RyYWluLCB5X3Rlc3QgPSB0cmFpbl90ZXN0X3NwbGl0KAogICAgICAgICAgICB4LCB5LCB0ZXN0X3NpemU9dHJhaW5fdGVzdF9zcGxpdF9zaXplLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICB4X3RyYWluLCB5X3RyYWluID0geCwgeQoKICAgICAgICB0ZXN0X3NldCA9IHRlc3Rfc2V0LmFzX2RmKCkKICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgIHRlc3Rfc2V0ID0gZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xKQoKICAgICAgICB4X3Rlc3QsIHlfdGVzdCA9IHRlc3Rfc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKSwgdGVzdF9zZXRbbGFiZWxfY29sdW1uc10KCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4oCiAgICAgICAgbW9kZWw9bW9kZWwsCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICB0YWc9dGFnLAogICAgICAgIHNhbXBsZV9zZXQ9c2FtcGxlX3NldCwKICAgICAgICB5X2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICB0ZXN0X3NldD10ZXN0X3NldCwKICAgICAgICB4X3Rlc3Q9eF90ZXN0LAogICAgICAgIHlfdGVzdD15X3Rlc3QsCiAgICAgICAgYXJ0aWZhY3RzPWNvbnRleHQuYXJ0aWZhY3RzLAogICAgICAgIGxhYmVscz1sYWJlbHMsCiAgICApCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidHJhaW5pbmcgJ3ttb2RlbF9uYW1lfSciKQogICAgbW9kZWwuZml0KHhfdHJhaW4sIHlfdHJhaW4sICoqZml0X2t3YXJncykKCgpkZWYgZXZhbHVhdGUoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIEV2YWx1YXRpbmcgYSBtb2RlbC4gQXJ0aWZhY3RzIGdlbmVyYXRlZCBieSB0aGUgTUxIYW5kbGVyLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIGV2YWx1YXRlIHRoZSBtb2RlbCBvbi4gQ2FuIGJlIGVpdGhlciBhIFVSSSBvciBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICAgICAgICAgIHN0ciBvciBhIGxpc3Qgb2Ygc3RyaW5ncyB0aGF0IHJlcHJlc2VudCB0aGUgY29sdW1ucyB0byBkcm9wLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHRvIHRoZSBwcmVkaWN0IGZ1bmN0aW9uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChQUkVESUNUXyBwcmVmaXggaXMgbm90IHJlcXVpcmVkKS4KICAgICIiIgogICAgIyBHZXQgZGF0YXNldCBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgUGFyc2luZyBsYWJlbF9jb2x1bW5zOgogICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMgPSBbXQogICAgaWYgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gKAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zIGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgbGlzdCkgZWxzZSBbbGFiZWxfY29sdW1uc10KICAgICAgICApCiAgICAgICAgZm9yIGxjIGluIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGlmIGZzLmNvbW1vbi5mZWF0dXJlX3NlcGFyYXRvciBpbiBsYzoKICAgICAgICAgICAgICAgIGZlYXR1cmVfc2V0X25hbWUsIGxhYmVsX25hbWUsIGFsaWFzID0gZnMuY29tbW9uLnBhcnNlX2ZlYXR1cmVfc3RyaW5nKGxjKQogICAgICAgICAgICAgICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMuYXBwZW5kKGFsaWFzIG9yIGxhYmVsX25hbWUpCiAgICAgICAgaWYgcGFyc2VkX2xhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBwYXJzZWRfbGFiZWxfY29sdW1ucwoKICAgIHggPSBkYXRhc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKQogICAgeSA9IGRhdGFzZXRbbGFiZWxfY29sdW1uc10KCiAgICAjIExvYWRpbmcgdGhlIG1vZGVsIGFuZCBwcmVkaWN0aW5nOgogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWwsIGNvbnRleHQ9Y29udGV4dCwgbW9kZWxfbmFtZT0ibW9kZWxfTGluZWFyUmVncmVzc2lvbiIKICAgICkKICAgIEF1dG9NTFJ1bi5hcHBseV9tbHJ1bihtb2RlbF9oYW5kbGVyLm1vZGVsLCB5X3Rlc3Q9eSwgbW9kZWxfcGF0aD1tb2RlbCkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiZXZhbHVhdGluZyAne21vZGVsX2hhbmRsZXIubW9kZWxfbmFtZX0nIikKICAgIG1vZGVsX2hhbmRsZXIubW9kZWwucHJlZGljdCh4LCAqKmt3YXJncykKCgpkZWYgcHJlZGljdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWw6IHN0ciwKICAgIGRhdGFzZXQ6IG1scnVuLkRhdGFJdGVtLAogICAgZHJvcF9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXSwgaW50LCBMaXN0W2ludF1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgcmVzdWx0X3NldDogT3B0aW9uYWxbc3RyXSA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgUHJlZGljdGluZyBkYXRhc2V0IGJ5IGEgbW9kZWwuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwgU3RvcmUgcGF0aC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gcHJlZGljdCB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkksIGEgRmVhdHVyZVZlY3RvciBvciBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSBpbiBhIHNoYXBlIG9mIGEgbGlzdC9kaWN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHBhc3NpbmcgYSBzYW1wbGUsIHBhc3MgdGhlIGRhdGFzZXQgYXMgYSBmaWVsZCBpbiBgcGFyYW1zYCBpbnN0ZWFkIG9mIGBpbnB1dHNgLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIvaW50IG9yIGEgbGlzdCBvZiBzdHJpbmdzL2ludHMgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbiBuYW1lcy9pbmRpY2VzIHRvIGRyb3AuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFdoZW4gdGhlIGRhdGFzZXQgaXMgYSBsaXN0L2RpY3QgdGhpcyBwYXJhbWV0ZXIgc2hvdWxkIGJlIHJlcHJlc2VudGVkIGJ5IGludGVnZXJzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gcmVzdWx0X3NldDogICAgICAgICAgICAgIFRoZSBkYiBrZXkgdG8gc2V0IG5hbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0IGFuZCB0aGUgZmlsZW5hbWUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHQgdG8gJ3ByZWRpY3Rpb24nLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIGxvYWRpbmcgdGhlIG1vZGVsLCBhbmQgZ2V0dGluZyB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbChtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQpCgogICAgIyBGaXggZmVhdHVyZSBuYW1lcyBmb3IgbW9kZWxzIHRoYXQgcmVxdWlyZSB0aGVtIChlLmcuLCBYR0Jvb3N0KQogICAgIyBXaGVuIGRhdGFzZXQgY29tZXMgZnJvbSBhIGxpc3QsIHBhbmRhcyBhc3NpZ25zIGRlZmF1bHQgaW50ZWdlciBjb2x1bW4gbmFtZXMKICAgICMgYnV0IHNvbWUgbW9kZWxzIGV4cGVjdCBzcGVjaWZpYyBmZWF0dXJlIG5hbWVzIHRoZXkgd2VyZSB0cmFpbmVkIHdpdGgKICAgIGlmIGhhc2F0dHIobW9kZWxfaGFuZGxlci5tb2RlbCwgJ2ZlYXR1cmVfbmFtZXNfaW5fJyk6CiAgICAgICAgZXhwZWN0ZWRfZmVhdHVyZXMgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLmZlYXR1cmVfbmFtZXNfaW5fCiAgICAgICAgaWYgbGVuKGRhdGFzZXQuY29sdW1ucykgPT0gbGVuKGV4cGVjdGVkX2ZlYXR1cmVzKToKICAgICAgICAgICAgIyBPbmx5IHJlbmFtZSBpZiB0aGUgbnVtYmVyIG9mIGNvbHVtbnMgbWF0Y2hlcwogICAgICAgICAgICAjIFRoaXMgaGFuZGxlcyB0aGUgY2FzZSB3aGVyZSBhIGxpc3Qgd2FzIGNvbnZlcnRlZCB0byBEYXRhRnJhbWUgd2l0aCBkZWZhdWx0IGNvbHVtbiBuYW1lcwogICAgICAgICAgICBpZiBub3QgYWxsKGNvbCA9PSBmZWF0IGZvciBjb2wsIGZlYXQgaW4gemlwKGRhdGFzZXQuY29sdW1ucywgZXhwZWN0ZWRfZmVhdHVyZXMpKToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSZW5hbWluZyBkYXRhc2V0IGNvbHVtbnMgdG8gbWF0Y2ggbW9kZWwncyBleHBlY3RlZCBmZWF0dXJlIG5hbWVzIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZGF0YXNldC5jb2x1bW5zID0gZXhwZWN0ZWRfZmVhdHVyZXMKCiAgICAjIERyb3BwaW5nIGxhYmVsIGNvbHVtbnMgaWYgbmVjZXNzYXJ5OgogICAgaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtdCiAgICBlbGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgc3RyKToKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW2xhYmVsX2NvbHVtbnNdCgogICAgIyBQcmVkaWN0aW5nOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIm1ha2luZyBwcmVkaWN0aW9uIGJ5ICd7bW9kZWxfaGFuZGxlci5tb2RlbF9uYW1lfSciKQogICAgeV9wcmVkID0gbW9kZWxfaGFuZGxlci5tb2RlbC5wcmVkaWN0KGRhdGFzZXQsICoqa3dhcmdzKQoKICAgICMgUHJlcGFyaW5nIGFuZCB2YWxpZGF0aW5nIGxhYmVsIGNvbHVtbnMgZm9yIHRoZSBkYXRhZnJhbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0OgogICAgbnVtX3ByZWRpY3RlZCA9IDEgaWYgbGVuKHlfcHJlZC5zaGFwZSkgPT0gMSBlbHNlIHlfcHJlZC5zaGFwZVsxXQoKICAgIGlmIG51bV9wcmVkaWN0ZWQgPiBsZW4obGFiZWxfY29sdW1ucyk6CiAgICAgICAgaWYgbnVtX3ByZWRpY3RlZCA9PSAxOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWyJwcmVkaWN0ZWQgbGFiZWxzIl0KICAgICAgICBlbHNlOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zLmV4dGVuZCgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICBmInByZWRpY3RlZF9sYWJlbF97aSArIDEgKyBsZW4obGFiZWxfY29sdW1ucyl9IgogICAgICAgICAgICAgICAgICAgIGZvciBpIGluIHJhbmdlKG51bV9wcmVkaWN0ZWQgLSBsZW4obGFiZWxfY29sdW1ucykpCiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgICkKICAgIGVsaWYgbnVtX3ByZWRpY3RlZCA8IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJudW1iZXIgb2YgcHJlZGljdGVkIGxhYmVsczoge251bV9wcmVkaWN0ZWR9IGlzIHNtYWxsZXIgdGhhbiBudW1iZXIgb2YgbGFiZWwgY29sdW1uczoge2xlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBhcnRpZmFjdF9uYW1lID0gcmVzdWx0X3NldCBvciAicHJlZGljdGlvbiIKICAgIGxhYmVsc19pbnNpZGVfZGYgPSBzZXQobGFiZWxfY29sdW1ucykgJiBzZXQoZGF0YXNldC5jb2x1bW5zLnRvbGlzdCgpKQogICAgaWYgbGFiZWxzX2luc2lkZV9kZjoKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJUaGUgbGFiZWxzOiB7bGFiZWxzX2luc2lkZV9kZn0gYXJlIGFscmVhZHkgZXhpc3RlZCBpbiB0aGUgZGF0YWZyYW1lIgogICAgICAgICkKICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICBwcmVkX2RmID0gcGQuY29uY2F0KFtkYXRhc2V0LCBwZC5EYXRhRnJhbWUoeV9wcmVkLCBjb2x1bW5zPWxhYmVsX2NvbHVtbnMpXSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldChhcnRpZmFjdF9uYW1lLCBwcmVkX2RmLCBkYl9rZXk9cmVzdWx0X3NldCkK command: '' -kind: job -verbose: false + default_handler: train + image: mlrun/mlrun + disable_auto_mount: false + description: Automatic train, evaluate and predict functions for the ML frameworks + - Scikit-Learn, XGBoost and LightGBM. +metadata: + categories: + - machine-learning + - model-training + tag: '' + name: auto-trainer diff --git a/functions/src/auto_trainer/item.yaml b/functions/src/auto_trainer/item.yaml index ba33f6a08..d397a79d6 100755 --- a/functions/src/auto_trainer/item.yaml +++ b/functions/src/auto_trainer/item.yaml @@ -13,7 +13,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.0 +mlrunVersion: 1.10.0 name: auto_trainer platformVersion: 3.5.0 spec: @@ -23,4 +23,4 @@ spec: kind: job requirements: [] url: '' -version: 1.8.0 +version: 1.9.0 diff --git a/functions/src/auto_trainer/requirements.txt b/functions/src/auto_trainer/requirements.txt index b14a0293c..b23f9b9dd 100644 --- a/functions/src/auto_trainer/requirements.txt +++ b/functions/src/auto_trainer/requirements.txt @@ -1,4 +1,5 @@ pandas -scikit-learn<1.4.0 +scikit-learn~=1.5.2 +lightgbm xgboost<2.0.0 plotly diff --git a/functions/src/auto_trainer/test_auto_trainer.py b/functions/src/auto_trainer/test_auto_trainer.py index 9a1ff554c..ac95109f8 100644 --- a/functions/src/auto_trainer/test_auto_trainer.py +++ b/functions/src/auto_trainer/test_auto_trainer.py @@ -29,6 +29,9 @@ ("sklearn.linear_model.LinearRegression", "regression"), ("sklearn.ensemble.RandomForestClassifier", "classification"), ("xgboost.XGBRegressor", "regression"), + ("xgboost.XGBClassifier", "classification"), + ("lightgbm.LGBMRegressor", "regression"), + ("lightgbm.LGBMClassifier", "classification") ] REQUIRED_ENV_VARS = [ @@ -78,11 +81,15 @@ def _assert_train_handler(train_run): @pytest.mark.parametrize("model", MODELS) +@pytest.mark.skipif( + condition=not _validate_environment_variables(), + reason="Project's environment variables are not set", +) def test_train(model: Tuple[str, str]): dataset, label_columns = _get_dataset(model[1]) is_test_passed = True - project = mlrun.new_project("auto-trainer-test", context="./") + project = mlrun.get_or_create_project("auto-trainer-test", context="./") fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun") train_run = None @@ -119,7 +126,7 @@ def test_train_evaluate(model: Tuple[str, str]): dataset, label_columns = _get_dataset(model[1]) is_test_passed = True # Importing function: - project = mlrun.new_project("auto-trainer-test", context="./") + project = mlrun.get_or_create_project("auto-trainer-test", context="./") fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun") temp_dir = tempfile.mkdtemp() @@ -172,7 +179,7 @@ def test_train_predict(model: Tuple[str, str]): df = pd.read_csv(dataset) sample = df.head().drop("labels", axis=1).values.tolist() # Importing function: - project = mlrun.new_project("auto-trainer-test", context="./") + project = mlrun.get_or_create_project("auto-trainer-test", context="./") fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun") temp_dir = tempfile.mkdtemp() diff --git a/functions/src/describe/function.yaml b/functions/src/describe/function.yaml index a11461774..1c254c3c4 100644 --- a/functions/src/describe/function.yaml +++ b/functions/src/describe/function.yaml @@ -1,9 +1,44 @@ +metadata: + tag: '' + categories: + - data-analysis + name: describe +verbose: false +kind: job spec: + command: '' + image: mlrun/mlrun + description: describe and visualizes dataset stats + disable_auto_mount: false + default_handler: analyze entry_points: analyze: + doc: 'The function will output the following artifacts per + + column within the data frame (based on data types) + + If the data has more than 500,000 sample we + + sample randomly 500,000 samples: + + + describe csv + + histograms + + scatter-2d + + violin chart + + correlation-matrix chart + + correlation-matrix csv + + imbalance pie chart + + imbalance-weights-vec csv' + has_kwargs: false has_varargs: false - outputs: - - type: None parameters: - name: context type: MLClientCtx @@ -45,46 +80,11 @@ spec: - name: dask_client doc: Dask client object default: null - doc: 'The function will output the following artifacts per - - column within the data frame (based on data types) - - If the data has more than 500,000 sample we - - sample randomly 500,000 samples: - - - describe csv - - histograms - - scatter-2d - - violin chart - - correlation-matrix chart - - correlation-matrix csv - - imbalance pie chart - - imbalance-weights-vec csv' - has_kwargs: false + outputs: + - type: None name: analyze lineno: 46 - image: mlrun/mlrun - command: '' build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCmZyb20gdHlwaW5nIGltcG9ydCBVbmlvbgoKaW1wb3J0IG1scnVuCmltcG9ydCBudW1weSBhcyBucAoKd2FybmluZ3Muc2ltcGxlZmlsdGVyKGFjdGlvbj0iaWdub3JlIiwgY2F0ZWdvcnk9RnV0dXJlV2FybmluZykKCmltcG9ydCBtbHJ1bi5mZWF0dXJlX3N0b3JlIGFzIGZzdG9yZQppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCBwbG90bHkuZXhwcmVzcyBhcyBweAppbXBvcnQgcGxvdGx5LmZpZ3VyZV9mYWN0b3J5IGFzIGZmCmltcG9ydCBwbG90bHkuZ3JhcGhfb2JqZWN0cyBhcyBnbwpmcm9tIG1scnVuLmFydGlmYWN0cyBpbXBvcnQgKAogICAgQXJ0aWZhY3QsCiAgICBEYXRhc2V0QXJ0aWZhY3QsCiAgICBQbG90bHlBcnRpZmFjdCwKICAgIFRhYmxlQXJ0aWZhY3QsCiAgICB1cGRhdGVfZGF0YXNldF9tZXRhLAopCmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKZnJvbSBtbHJ1bi5mZWF0dXJlX3N0b3JlIGltcG9ydCBGZWF0dXJlU2V0CmZyb20gcGxvdGx5LnN1YnBsb3RzIGltcG9ydCBtYWtlX3N1YnBsb3RzCgpwZC5zZXRfb3B0aW9uKCJkaXNwbGF5LmZsb2F0X2Zvcm1hdCIsIGxhbWJkYSB4OiAiJS4yZiIgJSB4KQpNQVhfU0laRV9PRl9ERiA9IDUwMDAwMAoKCmRlZiBhbmFseXplKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBuYW1lOiBzdHIgPSAiZGF0YXNldCIsCiAgICB0YWJsZTogVW5pb25bRmVhdHVyZVNldCwgRGF0YUl0ZW1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gTm9uZSwKICAgIHBsb3RzX2Rlc3Q6IHN0ciA9ICJwbG90cyIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDEsCiAgICBwcm9ibGVtX3R5cGU6IHN0ciA9ICJjbGFzc2lmaWNhdGlvbiIsCiAgICBkYXNrX2tleTogc3RyID0gImRhc2tfa2V5IiwKICAgIGRhc2tfZnVuY3Rpb246IHN0ciA9IE5vbmUsCiAgICBkYXNrX2NsaWVudD1Ob25lLAopIC0+IE5vbmU6CiAgICAiIiIKICAgIFRoZSBmdW5jdGlvbiB3aWxsIG91dHB1dCB0aGUgZm9sbG93aW5nIGFydGlmYWN0cyBwZXIKICAgIGNvbHVtbiB3aXRoaW4gdGhlIGRhdGEgZnJhbWUgKGJhc2VkIG9uIGRhdGEgdHlwZXMpCiAgICBJZiB0aGUgZGF0YSBoYXMgbW9yZSB0aGFuIDUwMCwwMDAgc2FtcGxlIHdlCiAgICBzYW1wbGUgcmFuZG9tbHkgNTAwLDAwMCBzYW1wbGVzOgoKICAgIGRlc2NyaWJlIGNzdgogICAgaGlzdG9ncmFtcwogICAgc2NhdHRlci0yZAogICAgdmlvbGluIGNoYXJ0CiAgICBjb3JyZWxhdGlvbi1tYXRyaXggY2hhcnQKICAgIGNvcnJlbGF0aW9uLW1hdHJpeCBjc3YKICAgIGltYmFsYW5jZSBwaWUgY2hhcnQKICAgIGltYmFsYW5jZS13ZWlnaHRzLXZlYyBjc3YKCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIFRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbmFtZTogICAgICAgICAgICAgICAgICAgIEtleSBvZiBkYXRhc2V0IHRvIGRhdGFiYXNlICgiZGF0YXNldCIgZm9yIGRlZmF1bHQpCiAgICA6cGFyYW0gdGFibGU6ICAgICAgICAgICAgICAgICAgIE1MUnVuIGlucHV0IHBvaW50aW5nIHRvIHBhbmRhcyBkYXRhZnJhbWUgKGNzdi9wYXJxdWV0IGZpbGUgcGF0aCkgb3IgRmVhdHVyZVNldAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhcyBwYXJhbQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICAgICAgICBHcm91bmQgdHJ1dGggY29sdW1uIGxhYmVsCiAgICA6cGFyYW0gcGxvdHNfZGVzdDogICAgICAgICAgICAgIERlc3RpbmF0aW9uIGZvbGRlciBvZiBzdW1tYXJ5IHBsb3RzIChyZWxhdGl2ZSB0byBhcnRpZmFjdF9wYXRoKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoInBsb3RzIiBmb3IgZGVmYXVsdCkKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgV2hlbiB0aGUgdGFibGUgaGFzIG1vcmUgdGhhbiA1MDAsMDAwIHNhbXBsZXMsIHdlIHNhbXBsZSByYW5kb21seSA1MDAsMDAwIHNhbXBsZXMKICAgIDpwYXJhbSBwcm9ibGVtX3R5cGUgICAgICAgICAgICAgVGhlIHR5cGUgb2YgdGhlIE1MIHByb2JsZW0gdGhlIGRhdGEgZmFjaW5nIC0gcmVncmVzc2lvbiwgY2xhc3NpZmljYXRpb24gb3IgTm9uZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoY2xhc3NpZmljYXRpb24gZm9yIGRlZmF1bHQpCiAgICA6cGFyYW0gZGFza19rZXk6ICAgICAgICAgICAgICAgIEtleSBvZiBkYXRhZnJhbWUgaW4gZGFzayBjbGllbnQgImRhdGFzZXRzIiBhdHRyaWJ1dGUKICAgIDpwYXJhbSBkYXNrX2Z1bmN0aW9uOiAgICAgICAgICAgRGFzayBmdW5jdGlvbiB1cmwgKGRiOi8vLi4pCiAgICA6cGFyYW0gZGFza19jbGllbnQ6ICAgICAgICAgICAgIERhc2sgY2xpZW50IG9iamVjdAogICAgIiIiCiAgICBkYXRhX2l0ZW0sIGZlYXR1cmVzZXQsIGNyZWF0LCB1cGRhdGUgPSBGYWxzZSwgRmFsc2UsIEZhbHNlLCBGYWxzZQogICAgZ2V0X2Zyb21fdGFibGUgPSBUcnVlCiAgICBpZiBkYXNrX2Z1bmN0aW9uIG9yIGRhc2tfY2xpZW50OgogICAgICAgIGRhdGFfaXRlbSwgY3JlYXQgPSBUcnVlLCBUcnVlCiAgICAgICAgaWYgZGFza19mdW5jdGlvbjoKICAgICAgICAgICAgY2xpZW50ID0gbWxydW4uaW1wb3J0X2Z1bmN0aW9uKGRhc2tfZnVuY3Rpb24pLmNsaWVudAogICAgICAgIGVsaWYgZGFza19jbGllbnQ6CiAgICAgICAgICAgIGNsaWVudCA9IGRhc2tfY2xpZW50CiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigiZGFzayBjbGllbnQgd2FzIG5vdCBwcm92aWRlZCIpCgogICAgICAgIGlmIGRhc2tfa2V5IGluIGNsaWVudC5kYXRhc2V0czoKICAgICAgICAgICAgZGYgPSBjbGllbnQuZ2V0X2RhdGFzZXQoZGFza19rZXkpCiAgICAgICAgICAgIGRhdGFfaXRlbSwgY3JlYXQsIGdldF9mcm9tX3RhYmxlID0gVHJ1ZSwgVHJ1ZSwgRmFsc2UKICAgICAgICBlbGlmIHRhYmxlOgogICAgICAgICAgICBnZXRfZnJvbV90YWJsZSA9IFRydWUKICAgICAgICBlbHNlOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgZiJvbmx5IHRoZXNlIGRhdGFzZXRzIGFyZSBhdmFpbGFibGUge2NsaWVudC5kYXRhc2V0c30gaW4gY2xpZW50IHtjbGllbnR9IgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIEV4Y2VwdGlvbigiZGF0YXNldCBub3QgZm91bmQgb24gZGFzayBjbHVzdGVyIikKCiAgICBpZiBnZXRfZnJvbV90YWJsZToKICAgICAgICBpZiB0eXBlKHRhYmxlKSA9PSBEYXRhSXRlbToKICAgICAgICAgICAgaWYgdGFibGUubWV0YSBpcyBOb25lOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgVHJ1ZSwgRmFsc2UKICAgICAgICAgICAgZWxpZiB0YWJsZS5tZXRhLmtpbmQgPT0gImRhdGFzZXQiOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgRmFsc2UsIFRydWUKICAgICAgICAgICAgZWxpZiB0YWJsZS5tZXRhLmtpbmQgPT0gIkZlYXR1cmVWZWN0b3IiOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgRmFsc2UsIEZhbHNlCiAgICAgICAgICAgIGVsaWYgdGFibGUubWV0YS5raW5kID09ICJGZWF0dXJlU2V0IjoKICAgICAgICAgICAgICAgIGZlYXR1cmVzZXQsIGNyZWF0LCB1cGRhdGUgPSBUcnVlLCBGYWxzZSwgRmFsc2UKCiAgICAgICAgaWYgZGF0YV9pdGVtOgogICAgICAgICAgICBkZiA9IHRhYmxlLmFzX2RmKCkKICAgICAgICBlbGlmIGZlYXR1cmVzZXQ6CiAgICAgICAgICAgIHByb2plY3RfbmFtZSwgc2V0X25hbWUgPSAoCiAgICAgICAgICAgICAgICB0YWJsZS5fcGF0aC5zcGxpdCgiLyIpWzJdLAogICAgICAgICAgICAgICAgdGFibGUuX3BhdGguc3BsaXQoIi8iKVs0XSwKICAgICAgICAgICAgKQogICAgICAgICAgICBmZWF0dXJlX3NldCA9IGZzdG9yZS5nZXRfZmVhdHVyZV9zZXQoCiAgICAgICAgICAgICAgICBmInN0b3JlOi8vZmVhdHVyZS1zZXRzL3twcm9qZWN0X25hbWV9L3tzZXRfbmFtZX0iCiAgICAgICAgICAgICkKICAgICAgICAgICAgZGYgPSBmZWF0dXJlX3NldC50b19kYXRhZnJhbWUoKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKGYiV3JvbmcgdGFibGUgdHlwZS4iKQogICAgICAgICAgICByZXR1cm4KCiAgICBpZiBkZi5zaXplID4gTUFYX1NJWkVfT0ZfREY6CiAgICAgICAgZGYgPSBkZi5zYW1wbGUobj1pbnQoTUFYX1NJWkVfT0ZfREYgLyBkZi5zaGFwZVsxXSksIHJhbmRvbV9zdGF0ZT1yYW5kb21fc3RhdGUpCiAgICBleHRyYV9kYXRhID0ge30KCiAgICBpZiBsYWJlbF9jb2x1bW4gbm90IGluIGRmLmNvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1uID0gTm9uZQoKICAgIGV4dHJhX2RhdGFbImRlc2NyaWJlIGNzdiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgVGFibGVBcnRpZmFjdCgiZGVzY3JpYmUtY3N2IiwgZGY9ZGYuZGVzY3JpYmUoKSksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9kZXNjcmliZS5jc3YiLAogICAgKQoKICAgIHRyeToKICAgICAgICBfY3JlYXRlX2hpc3RvZ3JhbV9tYXRfYXJ0aWZhY3QoCiAgICAgICAgICAgIGNvbnRleHQsIGRmLCBleHRyYV9kYXRhLCBsYWJlbF9jb2x1bW4sIHBsb3RzX2Rlc3QKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgaGlzdG9ncmFtIG1hdHJpeCBhcnRpZmFjdCBkdWUgdG86IHtlfSIpCiAgICB0cnk6CiAgICAgICAgX2NyZWF0ZV9mZWF0dXJlc19oaXN0b2dyYW1fYXJ0aWZhY3RzKAogICAgICAgICAgICBjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0LCBwcm9ibGVtX3R5cGUKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgcGFpcnBsb3QgaGlzdG9ncmFtcyBkdWUgdG86IHtlfSIpCiAgICB0cnk6CiAgICAgICAgX2NyZWF0ZV9mZWF0dXJlc18yZF9zY2F0dGVyX2FydGlmYWN0cygKICAgICAgICAgICAgY29udGV4dCwgZGYsIGV4dHJhX2RhdGEsIGxhYmVsX2NvbHVtbiwgcGxvdHNfZGVzdCwgcHJvYmxlbV90eXBlCiAgICAgICAgKQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIHBhaXJwbG90IDJkX3NjYXR0ZXIgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfdmlvbGluX2FydGlmYWN0KGNvbnRleHQsIGRmLCBleHRyYV9kYXRhLCBwbG90c19kZXN0KQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIHZpb2xpbiBkaXN0cmlidXRpb24gcGxvdHMgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfaW1iYWxhbmNlX2FydGlmYWN0KAogICAgICAgICAgICBjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0LCBwcm9ibGVtX3R5cGUKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgY2xhc3MgaW1iYWxhbmNlIHBsb3QgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfY29ycl9hcnRpZmFjdChjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0KQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIGZlYXR1cmVzIGNvcnJlbGF0aW9uIHBsb3QgZHVlIHRvOiB7ZX0iKQoKICAgIGlmIG5vdCBkYXRhX2l0ZW06CiAgICAgICAgcmV0dXJuCgogICAgYXJ0aWZhY3QgPSB0YWJsZS5hcnRpZmFjdF91cmwKICAgIGlmIGNyZWF0OiAgIyBkYXRhc2V0IG5vdCBzdG9yZWQKICAgICAgICBhcnRpZmFjdCA9IERhdGFzZXRBcnRpZmFjdCgKICAgICAgICAgICAga2V5PSJkYXRhc2V0Iiwgc3RhdHM9VHJ1ZSwgZGY9ZGYsIGV4dHJhX2RhdGE9ZXh0cmFfZGF0YQogICAgICAgICkKICAgICAgICBhcnRpZmFjdCA9IGNvbnRleHQubG9nX2FydGlmYWN0KGFydGlmYWN0LCBkYl9rZXk9bmFtZSkKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVGhlIGRhdGEgc2V0IGlzIGxvZ2dlZCB0byB0aGUgcHJvamVjdCB1bmRlciB7bmFtZX0gbmFtZSIpCgogICAgaWYgdXBkYXRlOgogICAgICAgIHVwZGF0ZV9kYXRhc2V0X21ldGEoYXJ0aWZhY3QsIGV4dHJhX2RhdGE9ZXh0cmFfZGF0YSkKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVGhlIGRhdGEgc2V0IG5hbWVkIHtuYW1lfSBpcyB1cGRhdGVkIikKCiAgICAjIFRPRE8gOiAzLUQgcGxvdCBvbiBvbiBzZWxlY3RlZCBmZWF0dXJlcy4KICAgICMgVE9ETyA6IFJlaW50ZWdyYXRpb24gcGxvdCBvbiBvbiBzZWxlY3RlZCBmZWF0dXJlcy4KICAgICMgVE9ETyA6IFBDQSBwbG90ICh3aXRoIG9wdGlvbnMpCgoKZGVmIF9jcmVhdGVfaGlzdG9ncmFtX21hdF9hcnRpZmFjdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKKToKICAgICIiIgogICAgQ3JlYXRlIGFuZCBsb2cgYSBoaXN0b2dyYW0gbWF0cml4IGFydGlmYWN0CiAgICAiIiIKICAgIGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgIGl0ZW09QXJ0aWZhY3QoCiAgICAgICAgICAgIGtleT0iaGlzdCIsCiAgICAgICAgICAgIGJvZHk9YiI8Yj4gRGVwcmVjYXRlZCwgc2VlIHRoZSBhcnRpZmFjdHMgc2NhdHRlci0yZCAiCiAgICAgICAgICAgIGIiYW5kIGhpc3RvZ3JhbXMgaW5zdGVhZDxiPiIsCiAgICAgICAgKSwKICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2hpc3QuaHRtbCIsCiAgICApCgoKZGVmIF9jcmVhdGVfZmVhdHVyZXNfaGlzdG9ncmFtX2FydGlmYWN0cygKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhIGhpc3RvZ3JhbSBhcnRpZmFjdCBmb3IgZWFjaCBmZWF0dXJlCiAgICAiIiIKCiAgICBmaWdzID0gZGljdCgpCiAgICBmaXJzdF9mZWF0dXJlX25hbWUgPSAiIgogICAgaWYgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lIGFuZCBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICBhbGxfbGFiZWxzID0gZGZbbGFiZWxfY29sdW1uXS51bmlxdWUoKQogICAgdmlzaWJsZSA9IFRydWUKICAgIGZvciBjb2x1bW5fbmFtZSBpbiBkZi5jb2x1bW5zOgogICAgICAgIGlmIGNvbHVtbl9uYW1lID09IGxhYmVsX2NvbHVtbjoKICAgICAgICAgICAgY29udGludWUKCiAgICAgICAgaWYgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lIGFuZCBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICAgICAgZm9yIGxhYmVsIGluIGFsbF9sYWJlbHM6CiAgICAgICAgICAgICAgICBzdWJfZmlnID0gZ28uSGlzdG9ncmFtKAogICAgICAgICAgICAgICAgICAgIGhpc3RmdW5jPSJjb3VudCIsCiAgICAgICAgICAgICAgICAgICAgeD1kZi5sb2NbZGZbbGFiZWxfY29sdW1uXSA9PSBsYWJlbF1bY29sdW1uX25hbWVdLAogICAgICAgICAgICAgICAgICAgIG5hbWU9c3RyKGxhYmVsKSwKICAgICAgICAgICAgICAgICAgICB2aXNpYmxlPXZpc2libGUsCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBmaWdzW2Yie2NvbHVtbl9uYW1lfUA/QHtsYWJlbH0iXSA9IHN1Yl9maWcKICAgICAgICBlbHNlOgogICAgICAgICAgICBzdWJfZmlnID0gZ28uSGlzdG9ncmFtKGhpc3RmdW5jPSJjb3VudCIsIHg9ZGZbY29sdW1uX25hbWVdLCB2aXNpYmxlPXZpc2libGUpCiAgICAgICAgICAgIGZpZ3NbZiJ7Y29sdW1uX25hbWV9QD9AezF9Il0gPSBzdWJfZmlnCiAgICAgICAgaWYgdmlzaWJsZToKICAgICAgICAgICAgZmlyc3RfZmVhdHVyZV9uYW1lID0gY29sdW1uX25hbWUKICAgICAgICB2aXNpYmxlID0gRmFsc2UKCiAgICBmaWcgPSBnby5GaWd1cmUoKQogICAgZm9yIGsgaW4gZmlncy5rZXlzKCk6CiAgICAgICAgZmlnLmFkZF90cmFjZShmaWdzW2tdKQoKICAgIGZpZy51cGRhdGVfbGF5b3V0KAogICAgICAgIHVwZGF0ZW1lbnVzPVsKICAgICAgICAgICAgewogICAgICAgICAgICAgICAgImJ1dHRvbnMiOiBbCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAibGFiZWwiOiBjb2x1bW5fbmFtZSwKICAgICAgICAgICAgICAgICAgICAgICAgIm1ldGhvZCI6ICJ1cGRhdGUiLAogICAgICAgICAgICAgICAgICAgICAgICAiYXJncyI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidmlzaWJsZSI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAga2V5LnNwbGl0KCJAP0AiKVswXSA9PSBjb2x1bW5fbmFtZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb3Iga2V5IGluIGZpZ3Mua2V5cygpCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAieGF4aXMiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJyYW5nZSI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1pbihkZltjb2x1bW5fbmFtZV0pLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWF4KGRmW2NvbHVtbl9uYW1lXSksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHsidGl0bGUiOiBmIjxpPjxiPkhpc3RvZ3JhbSBvZiB7Y29sdW1uX25hbWV9PC9iPjwvaT4ifSwKICAgICAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICAgICAgZm9yIGNvbHVtbl9uYW1lIGluIGRmLmNvbHVtbnMKICAgICAgICAgICAgICAgICAgICBpZiBjb2x1bW5fbmFtZSAhPSBsYWJlbF9jb2x1bW4KICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICAiZGlyZWN0aW9uIjogImRvd24iLAogICAgICAgICAgICAgICAgInBhZCI6IHsiciI6IDEwLCAidCI6IDEwfSwKICAgICAgICAgICAgICAgICJzaG93YWN0aXZlIjogVHJ1ZSwKICAgICAgICAgICAgICAgICJ4IjogMC4yNSwKICAgICAgICAgICAgICAgICJ4YW5jaG9yIjogImxlZnQiLAogICAgICAgICAgICAgICAgInkiOiAxLjEsCiAgICAgICAgICAgICAgICAieWFuY2hvciI6ICJ0b3AiLAogICAgICAgICAgICB9CiAgICAgICAgXSwKICAgICAgICBhbm5vdGF0aW9ucz1bCiAgICAgICAgICAgIGRpY3QoCiAgICAgICAgICAgICAgICB0ZXh0PSJTZWxlY3QgRmVhdHVyZSBOYW1lICIsCiAgICAgICAgICAgICAgICBzaG93YXJyb3c9RmFsc2UsCiAgICAgICAgICAgICAgICB4PTAsCiAgICAgICAgICAgICAgICB5PTEuMDUsCiAgICAgICAgICAgICAgICB5cmVmPSJwYXBlciIsCiAgICAgICAgICAgICAgICB4cmVmPSJwYXBlciIsCiAgICAgICAgICAgICAgICBhbGlnbj0ibGVmdCIsCiAgICAgICAgICAgICAgICB4YW5jaG9yPSJsZWZ0IiwKICAgICAgICAgICAgICAgIHlhbmNob3I9InRvcCIsCiAgICAgICAgICAgICAgICBmb250PXsKICAgICAgICAgICAgICAgICAgICAiY29sb3IiOiAiYmx1ZSIsCiAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICApCiAgICAgICAgXSwKICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dCgKICAgICAgICB3aWR0aD02MDAsCiAgICAgICAgaGVpZ2h0PTQwMCwKICAgICAgICBhdXRvc2l6ZT1GYWxzZSwKICAgICAgICBtYXJnaW49ZGljdCh0PTEwMCwgYj0wLCBsPTAsIHI9MCksCiAgICAgICAgdGVtcGxhdGU9InBsb3RseV93aGl0ZSIsCiAgICApCgogICAgZmlnLnVwZGF0ZV9sYXlvdXQodGl0bGVfdGV4dD1mIjxpPjxiPkhpc3RvZ3JhbXMgb2Yge2ZpcnN0X2ZlYXR1cmVfbmFtZX08L2I+PC9pPiIpCiAgICBleHRyYV9kYXRhW2YiaGlzdG9ncmFtcyJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PWYiaGlzdG9ncmFtcyIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vaGlzdG9ncmFtcy5odG1sIiwKICAgICkKCgpkZWYgX2NyZWF0ZV9mZWF0dXJlc18yZF9zY2F0dGVyX2FydGlmYWN0cygKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhIHNjYXR0ZXItMmQgYXJ0aWZhY3QgZm9yIGVhY2ggY291cGxlIG9mIGZlYXR1cmVzCiAgICAiIiIKICAgIGZlYXR1cmVzID0gWwogICAgICAgIGNvbHVtbl9uYW1lIGZvciBjb2x1bW5fbmFtZSBpbiBkZi5jb2x1bW5zIGlmIGNvbHVtbl9uYW1lICE9IGxhYmVsX2NvbHVtbgogICAgXQogICAgbWF4X2ZlYXR1cmVfbGVuID0gZmxvYXQobWF4KGxlbihlbGVtKSBmb3IgZWxlbSBpbiBmZWF0dXJlcykpCiAgICBpZiBsYWJlbF9jb2x1bW4gaXMgbm90IE5vbmU6CiAgICAgICAgbGFiZWxzID0gc29ydGVkKGRmW2xhYmVsX2NvbHVtbl0udW5pcXVlKCkpCiAgICBlbHNlOgogICAgICAgIGxhYmVscyA9IFtOb25lXQogICAgZmlnID0gZ28uRmlndXJlKCkKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZSBhbmQgcHJvYmxlbV90eXBlID09ICJjbGFzc2lmaWNhdGlvbiI6CiAgICAgICAgZm9yIGwgaW4gbGFiZWxzOgogICAgICAgICAgICBmaWcuYWRkX3RyYWNlKAogICAgICAgICAgICAgICAgZ28uU2NhdHRlcigKICAgICAgICAgICAgICAgICAgICB4PWRmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgICAgICB5PWRmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgICAgICBtb2RlPSJtYXJrZXJzIiwKICAgICAgICAgICAgICAgICAgICB2aXNpYmxlPVRydWUsCiAgICAgICAgICAgICAgICAgICAgc2hvd2xlZ2VuZD1UcnVlLAogICAgICAgICAgICAgICAgICAgIG5hbWU9c3RyKGwpLAogICAgICAgICAgICAgICAgKQogICAgICAgICAgICApCiAgICBlbGlmIGxhYmVsX2NvbHVtbiBpcyBOb25lOgogICAgICAgIGZpZy5hZGRfdHJhY2UoCiAgICAgICAgICAgIGdvLlNjYXR0ZXIoCiAgICAgICAgICAgICAgICB4PWRmW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgIHk9ZGZbZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgbW9kZT0ibWFya2VycyIsCiAgICAgICAgICAgICAgICB2aXNpYmxlPVRydWUsCiAgICAgICAgICAgICkKICAgICAgICApCiAgICBlbGlmIHByb2JsZW1fdHlwZSA9PSAicmVncmVzc2lvbiI6CiAgICAgICAgZmlnLmFkZF90cmFjZSgKICAgICAgICAgICAgZ28uU2NhdHRlcigKICAgICAgICAgICAgICAgIHg9ZGZbZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgeT1kZltmZWF0dXJlc1swXV0sCiAgICAgICAgICAgICAgICBtb2RlPSJtYXJrZXJzIiwKICAgICAgICAgICAgICAgIG1hcmtlcj1kaWN0KAogICAgICAgICAgICAgICAgICAgIGNvbG9yPWRmW2xhYmVsX2NvbHVtbl0sIGNvbG9yc2NhbGU9IlZpcmlkaXMiLCBzaG93c2NhbGU9VHJ1ZQogICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgIHZpc2libGU9VHJ1ZSwKICAgICAgICAgICAgKQogICAgICAgICkKCiAgICB4X2J1dHRvbnMgPSBbXQogICAgeV9idXR0b25zID0gW10KCiAgICBmb3IgbmNvbCBpbiBmZWF0dXJlczoKICAgICAgICBpZiBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIiBhbmQgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lOgogICAgICAgICAgICB4X2J1dHRvbnMuYXBwZW5kKAogICAgICAgICAgICAgICAgZGljdCgKICAgICAgICAgICAgICAgICAgICBtZXRob2Q9InVwZGF0ZSIsCiAgICAgICAgICAgICAgICAgICAgbGFiZWw9bmNvbCwKICAgICAgICAgICAgICAgICAgICBhcmdzPVsKICAgICAgICAgICAgICAgICAgICAgICAgeyJ4IjogW2RmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW25jb2xdIGZvciBsIGluIGxhYmVsc119LAogICAgICAgICAgICAgICAgICAgICAgICBucC5hcmFuZ2UobGVuKGxhYmVscykpLnRvbGlzdCgpLAogICAgICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICkKCiAgICAgICAgICAgIHlfYnV0dG9ucy5hcHBlbmQoCiAgICAgICAgICAgICAgICBkaWN0KAogICAgICAgICAgICAgICAgICAgIG1ldGhvZD0idXBkYXRlIiwKICAgICAgICAgICAgICAgICAgICBsYWJlbD1uY29sLAogICAgICAgICAgICAgICAgICAgIGFyZ3M9WwogICAgICAgICAgICAgICAgICAgICAgICB7InkiOiBbZGYubG9jW2RmW2xhYmVsX2NvbHVtbl0gPT0gbF1bbmNvbF0gZm9yIGwgaW4gbGFiZWxzXX0sCiAgICAgICAgICAgICAgICAgICAgICAgIG5wLmFyYW5nZShsZW4obGFiZWxzKSkudG9saXN0KCksCiAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHhfYnV0dG9ucy5hcHBlbmQoCiAgICAgICAgICAgICAgICBkaWN0KG1ldGhvZD0idXBkYXRlIiwgbGFiZWw9bmNvbCwgYXJncz1beyJ4IjogW2RmW25jb2xdXX1dKQogICAgICAgICAgICApCgogICAgICAgICAgICB5X2J1dHRvbnMuYXBwZW5kKAogICAgICAgICAgICAgICAgZGljdChtZXRob2Q9InVwZGF0ZSIsIGxhYmVsPW5jb2wsIGFyZ3M9W3sieSI6IFtkZltuY29sXV19XSkKICAgICAgICAgICAgKQoKICAgICMgUGFzcyBidXR0b25zIHRvIHRoZSB1cGRhdGVtZW51cyBhcmd1bWVudAogICAgZmlnLnVwZGF0ZV9sYXlvdXQoCiAgICAgICAgdXBkYXRlbWVudXM9WwogICAgICAgICAgICBkaWN0KGJ1dHRvbnM9eF9idXR0b25zLCBkaXJlY3Rpb249InVwIiwgeD0wLjUsIHk9LTAuMSksCiAgICAgICAgICAgIGRpY3QoYnV0dG9ucz15X2J1dHRvbnMsIGRpcmVjdGlvbj0iZG93biIsIHg9LW1heF9mZWF0dXJlX2xlbiAvIDEwMCwgeT0wLjUpLAogICAgICAgIF0KICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dCgKICAgICAgICB3aWR0aD02MDAsCiAgICAgICAgaGVpZ2h0PTQwMCwKICAgICAgICBhdXRvc2l6ZT1GYWxzZSwKICAgICAgICBtYXJnaW49ZGljdCh0PTEwMCwgYj0wLCBsPTAsIHI9MCksCiAgICAgICAgdGVtcGxhdGU9InBsb3RseV93aGl0ZSIsCiAgICApCgogICAgZmlnLnVwZGF0ZV9sYXlvdXQodGl0bGVfdGV4dD1mIjxpPjxiPlNjYXR0ZXItMmQ8L2I+PC9pPiIpCiAgICBleHRyYV9kYXRhW2Yic2NhdHRlci0yZCJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PWYic2NhdHRlci0yZCIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vc2NhdHRlci0yZC5odG1sIiwKICAgICkKCgpkZWYgX2NyZWF0ZV92aW9saW5fYXJ0aWZhY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwgZGY6IHBkLkRhdGFGcmFtZSwgZXh0cmFfZGF0YTogZGljdCwgcGxvdHNfZGVzdDogc3RyCik6CiAgICAiIiIKICAgIENyZWF0ZSBhbmQgbG9nIGEgdmlvbGluIGFydGlmYWN0CiAgICAiIiIKICAgIGNvbHMgPSA1CiAgICByb3dzID0gKGRmLnNoYXBlWzFdIC8vIGNvbHMpICsgMQogICAgZmlnID0gbWFrZV9zdWJwbG90cyhyb3dzPXJvd3MsIGNvbHM9Y29scykKCiAgICBwbG90X251bSA9IDAKCiAgICBmb3IgY29sdW1uX25hbWUgaW4gZGYuY29sdW1uczoKICAgICAgICBjb2x1bW5fZGF0YSA9IGRmW2NvbHVtbl9uYW1lXQogICAgICAgIHZpb2xpbiA9IGdvLlZpb2xpbigKICAgICAgICAgICAgeD1bY29sdW1uX25hbWVdICogY29sdW1uX2RhdGEuc2hhcGVbMF0sCiAgICAgICAgICAgIHk9Y29sdW1uX2RhdGEsCiAgICAgICAgICAgIG5hbWU9Y29sdW1uX25hbWUsCiAgICAgICAgKQoKICAgICAgICBmaWcuYWRkX3RyYWNlKAogICAgICAgICAgICB2aW9saW4sCiAgICAgICAgICAgIHJvdz0ocGxvdF9udW0gLy8gY29scykgKyAxLAogICAgICAgICAgICBjb2w9KHBsb3RfbnVtICUgY29scykgKyAxLAogICAgICAgICkKCiAgICAgICAgcGxvdF9udW0gKz0gMQoKICAgIGZpZ1sibGF5b3V0Il0udXBkYXRlKAogICAgICAgIGhlaWdodD0ocm93cyArIDEpICogMjAwLAogICAgICAgIHdpZHRoPShjb2xzICsgMSkgKiAyMDAsCiAgICAgICAgdGl0bGU9IjxpPjxiPlZpb2xpbiBQbG90czwvYj48L2k+IiwKICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dChzaG93bGVnZW5kPUZhbHNlKQogICAgZXh0cmFfZGF0YVsidmlvbGluIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBQbG90bHlBcnRpZmFjdChrZXk9InZpb2xpbiIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vdmlvbGluLmh0bWwiLAogICAgKQoKCmRlZiBfY3JlYXRlX2ltYmFsYW5jZV9hcnRpZmFjdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhbiBpbWJhbGFuY2UgY2xhc3MgYXJ0aWZhY3QgKGNzdiArIHBsb3QpCiAgICAiIiIKICAgIGlmIGxhYmVsX2NvbHVtbjoKICAgICAgICBpZiBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICAgICAgdmFsdWVzX2NvbHVtbiA9ICJjb3VudCIKICAgICAgICAgICAgbGFiZWxzX2NvdW50ID0gZGZbbGFiZWxfY29sdW1uXS52YWx1ZV9jb3VudHMoKS5zb3J0X2luZGV4KCkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50ID0gcGQuRGF0YUZyYW1lKGxhYmVsc19jb3VudCkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W2xhYmVsX2NvbHVtbl0gPSBsYWJlbHNfY291bnQuaW5kZXgKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50LnJlbmFtZShjb2x1bW5zPXsiIjogdmFsdWVzX2NvbHVtbn0sIGlucGxhY2U9VHJ1ZSkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dID0gZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dIC8gc3VtKAogICAgICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dCiAgICAgICAgICAgICkKICAgICAgICAgICAgZmlnID0gcHgucGllKGRmX2xhYmVsc19jb3VudCwgbmFtZXM9bGFiZWxfY29sdW1uLCB2YWx1ZXM9dmFsdWVzX2NvbHVtbikKICAgICAgICBlbHNlOgogICAgICAgICAgICBmaWcgPSBweC5oaXN0b2dyYW0oCiAgICAgICAgICAgICAgICBoaXN0ZnVuYz0iY291bnQiLAogICAgICAgICAgICAgICAgeD1kZltsYWJlbF9jb2x1bW5dLAogICAgICAgICAgICApCiAgICAgICAgICAgIGhpc3QgPSBucC5oaXN0b2dyYW0oZGZbbGFiZWxfY29sdW1uXSkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50ID0gcGQuRGF0YUZyYW1lKAogICAgICAgICAgICAgICAgeyJtaW5fdmFsIjogaGlzdFsxXSwgImNvdW50IjogaGlzdFswXS50b2xpc3QoKSArIFswXX0KICAgICAgICAgICAgKQogICAgICAgIGZpZy51cGRhdGVfbGF5b3V0KHRpdGxlX3RleHQ9IjxpPjxiPkxhYmVscyBJbWJhbGFuY2U8L2I+PC9pPiIpCiAgICAgICAgZXh0cmFfZGF0YVsiaW1iYWxhbmNlIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJpbWJhbGFuY2UiLCBmaWd1cmU9ZmlnKSwKICAgICAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9pbWJhbGFuY2UuaHRtbCIsCiAgICAgICAgKQogICAgICAgIGV4dHJhX2RhdGFbImltYmFsYW5jZS1jc3YiXSA9IGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgICAgICBUYWJsZUFydGlmYWN0KCJpbWJhbGFuY2Utd2VpZ2h0cy12ZWMiLCBkZj1kZl9sYWJlbHNfY291bnQpLAogICAgICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2ltYmFsYW5jZS13ZWlnaHRzLXZlYy5jc3YiLAogICAgICAgICkKCgpkZWYgX2NyZWF0ZV9jb3JyX2FydGlmYWN0KAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkZjogcGQuRGF0YUZyYW1lLAogICAgZXh0cmFfZGF0YTogZGljdCwKICAgIGxhYmVsX2NvbHVtbjogc3RyLAogICAgcGxvdHNfZGVzdDogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhbiBjb3JyZWxhdGlvbi1tYXRyaXggYXJ0aWZhY3QgKGNzdiArIHBsb3QpCiAgICAiIiIKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZToKICAgICAgICBkZiA9IGRmLmRyb3AoW2xhYmVsX2NvbHVtbl0sIGF4aXM9MSkKICAgIHRibGNvcnIgPSBkZi5jb3JyKG51bWVyaWNfb25seT1UcnVlKQogICAgZXh0cmFfZGF0YVsiY29ycmVsYXRpb24tbWF0cml4LWNzdiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgVGFibGVBcnRpZmFjdCgiY29ycmVsYXRpb24tbWF0cml4LWNzdiIsIGRmPXRibGNvcnIsIHZpc2libGU9VHJ1ZSksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9jb3JyZWxhdGlvbi1tYXRyaXguY3N2IiwKICAgICkKCiAgICB6ID0gdGJsY29yci52YWx1ZXMudG9saXN0KCkKICAgIHpfdGV4dCA9IFtbIns6LjJmfSIuZm9ybWF0KHkpIGZvciB5IGluIHhdIGZvciB4IGluIHpdCiAgICBmaWcgPSBmZi5jcmVhdGVfYW5ub3RhdGVkX2hlYXRtYXAoCiAgICAgICAgeiwKICAgICAgICB4PWxpc3QodGJsY29yci5jb2x1bW5zKSwKICAgICAgICB5PWxpc3QodGJsY29yci5jb2x1bW5zKSwKICAgICAgICBhbm5vdGF0aW9uX3RleHQ9el90ZXh0LAogICAgICAgIGNvbG9yc2NhbGU9ImFnc3Vuc2V0IiwKICAgICkKICAgIGZpZ1sibGF5b3V0Il1bInlheGlzIl1bImF1dG9yYW5nZSJdID0gInJldmVyc2VkIiAgIyBsIC0+IHIKICAgIGZpZy51cGRhdGVfbGF5b3V0KHRpdGxlX3RleHQ9IjxpPjxiPkNvcnJlbGF0aW9uIG1hdHJpeDwvYj48L2k+IikKICAgIGZpZ1siZGF0YSJdWzBdWyJzaG93c2NhbGUiXSA9IFRydWUKCiAgICBleHRyYV9kYXRhWyJjb3JyZWxhdGlvbiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJjb3JyZWxhdGlvbiIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vY29ycmVsYXRpb24uaHRtbCIsCiAgICApCg== - code_origin: '' origin_filename: '' - description: describe and visualizes dataset stats - disable_auto_mount: false - default_handler: analyze -verbose: false -metadata: - tag: '' - name: describe - categories: - - data-analysis -kind: job + code_origin: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCmZyb20gdHlwaW5nIGltcG9ydCBVbmlvbgoKaW1wb3J0IG1scnVuCmltcG9ydCBudW1weSBhcyBucAoKd2FybmluZ3Muc2ltcGxlZmlsdGVyKGFjdGlvbj0iaWdub3JlIiwgY2F0ZWdvcnk9RnV0dXJlV2FybmluZykKCmltcG9ydCBtbHJ1bi5mZWF0dXJlX3N0b3JlIGFzIGZzdG9yZQppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCBwbG90bHkuZXhwcmVzcyBhcyBweAppbXBvcnQgcGxvdGx5LmZpZ3VyZV9mYWN0b3J5IGFzIGZmCmltcG9ydCBwbG90bHkuZ3JhcGhfb2JqZWN0cyBhcyBnbwpmcm9tIG1scnVuLmFydGlmYWN0cyBpbXBvcnQgKAogICAgQXJ0aWZhY3QsCiAgICBEYXRhc2V0QXJ0aWZhY3QsCiAgICBQbG90bHlBcnRpZmFjdCwKICAgIFRhYmxlQXJ0aWZhY3QsCiAgICB1cGRhdGVfZGF0YXNldF9tZXRhLAopCmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKZnJvbSBtbHJ1bi5mZWF0dXJlX3N0b3JlIGltcG9ydCBGZWF0dXJlU2V0CmZyb20gcGxvdGx5LnN1YnBsb3RzIGltcG9ydCBtYWtlX3N1YnBsb3RzCgpwZC5zZXRfb3B0aW9uKCJkaXNwbGF5LmZsb2F0X2Zvcm1hdCIsIGxhbWJkYSB4OiAiJS4yZiIgJSB4KQpNQVhfU0laRV9PRl9ERiA9IDUwMDAwMAoKCmRlZiBhbmFseXplKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBuYW1lOiBzdHIgPSAiZGF0YXNldCIsCiAgICB0YWJsZTogVW5pb25bRmVhdHVyZVNldCwgRGF0YUl0ZW1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gTm9uZSwKICAgIHBsb3RzX2Rlc3Q6IHN0ciA9ICJwbG90cyIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDEsCiAgICBwcm9ibGVtX3R5cGU6IHN0ciA9ICJjbGFzc2lmaWNhdGlvbiIsCiAgICBkYXNrX2tleTogc3RyID0gImRhc2tfa2V5IiwKICAgIGRhc2tfZnVuY3Rpb246IHN0ciA9IE5vbmUsCiAgICBkYXNrX2NsaWVudD1Ob25lLAopIC0+IE5vbmU6CiAgICAiIiIKICAgIFRoZSBmdW5jdGlvbiB3aWxsIG91dHB1dCB0aGUgZm9sbG93aW5nIGFydGlmYWN0cyBwZXIKICAgIGNvbHVtbiB3aXRoaW4gdGhlIGRhdGEgZnJhbWUgKGJhc2VkIG9uIGRhdGEgdHlwZXMpCiAgICBJZiB0aGUgZGF0YSBoYXMgbW9yZSB0aGFuIDUwMCwwMDAgc2FtcGxlIHdlCiAgICBzYW1wbGUgcmFuZG9tbHkgNTAwLDAwMCBzYW1wbGVzOgoKICAgIGRlc2NyaWJlIGNzdgogICAgaGlzdG9ncmFtcwogICAgc2NhdHRlci0yZAogICAgdmlvbGluIGNoYXJ0CiAgICBjb3JyZWxhdGlvbi1tYXRyaXggY2hhcnQKICAgIGNvcnJlbGF0aW9uLW1hdHJpeCBjc3YKICAgIGltYmFsYW5jZSBwaWUgY2hhcnQKICAgIGltYmFsYW5jZS13ZWlnaHRzLXZlYyBjc3YKCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIFRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbmFtZTogICAgICAgICAgICAgICAgICAgIEtleSBvZiBkYXRhc2V0IHRvIGRhdGFiYXNlICgiZGF0YXNldCIgZm9yIGRlZmF1bHQpCiAgICA6cGFyYW0gdGFibGU6ICAgICAgICAgICAgICAgICAgIE1MUnVuIGlucHV0IHBvaW50aW5nIHRvIHBhbmRhcyBkYXRhZnJhbWUgKGNzdi9wYXJxdWV0IGZpbGUgcGF0aCkgb3IgRmVhdHVyZVNldAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhcyBwYXJhbQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICAgICAgICBHcm91bmQgdHJ1dGggY29sdW1uIGxhYmVsCiAgICA6cGFyYW0gcGxvdHNfZGVzdDogICAgICAgICAgICAgIERlc3RpbmF0aW9uIGZvbGRlciBvZiBzdW1tYXJ5IHBsb3RzIChyZWxhdGl2ZSB0byBhcnRpZmFjdF9wYXRoKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoInBsb3RzIiBmb3IgZGVmYXVsdCkKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgV2hlbiB0aGUgdGFibGUgaGFzIG1vcmUgdGhhbiA1MDAsMDAwIHNhbXBsZXMsIHdlIHNhbXBsZSByYW5kb21seSA1MDAsMDAwIHNhbXBsZXMKICAgIDpwYXJhbSBwcm9ibGVtX3R5cGUgICAgICAgICAgICAgVGhlIHR5cGUgb2YgdGhlIE1MIHByb2JsZW0gdGhlIGRhdGEgZmFjaW5nIC0gcmVncmVzc2lvbiwgY2xhc3NpZmljYXRpb24gb3IgTm9uZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoY2xhc3NpZmljYXRpb24gZm9yIGRlZmF1bHQpCiAgICA6cGFyYW0gZGFza19rZXk6ICAgICAgICAgICAgICAgIEtleSBvZiBkYXRhZnJhbWUgaW4gZGFzayBjbGllbnQgImRhdGFzZXRzIiBhdHRyaWJ1dGUKICAgIDpwYXJhbSBkYXNrX2Z1bmN0aW9uOiAgICAgICAgICAgRGFzayBmdW5jdGlvbiB1cmwgKGRiOi8vLi4pCiAgICA6cGFyYW0gZGFza19jbGllbnQ6ICAgICAgICAgICAgIERhc2sgY2xpZW50IG9iamVjdAogICAgIiIiCiAgICBkYXRhX2l0ZW0sIGZlYXR1cmVzZXQsIGNyZWF0LCB1cGRhdGUgPSBGYWxzZSwgRmFsc2UsIEZhbHNlLCBGYWxzZQogICAgZ2V0X2Zyb21fdGFibGUgPSBUcnVlCiAgICBpZiBkYXNrX2Z1bmN0aW9uIG9yIGRhc2tfY2xpZW50OgogICAgICAgIGRhdGFfaXRlbSwgY3JlYXQgPSBUcnVlLCBUcnVlCiAgICAgICAgaWYgZGFza19mdW5jdGlvbjoKICAgICAgICAgICAgY2xpZW50ID0gbWxydW4uaW1wb3J0X2Z1bmN0aW9uKGRhc2tfZnVuY3Rpb24pLmNsaWVudAogICAgICAgIGVsaWYgZGFza19jbGllbnQ6CiAgICAgICAgICAgIGNsaWVudCA9IGRhc2tfY2xpZW50CiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigiZGFzayBjbGllbnQgd2FzIG5vdCBwcm92aWRlZCIpCgogICAgICAgIGlmIGRhc2tfa2V5IGluIGNsaWVudC5kYXRhc2V0czoKICAgICAgICAgICAgZGYgPSBjbGllbnQuZ2V0X2RhdGFzZXQoZGFza19rZXkpCiAgICAgICAgICAgIGRhdGFfaXRlbSwgY3JlYXQsIGdldF9mcm9tX3RhYmxlID0gVHJ1ZSwgVHJ1ZSwgRmFsc2UKICAgICAgICBlbGlmIHRhYmxlOgogICAgICAgICAgICBnZXRfZnJvbV90YWJsZSA9IFRydWUKICAgICAgICBlbHNlOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgZiJvbmx5IHRoZXNlIGRhdGFzZXRzIGFyZSBhdmFpbGFibGUge2NsaWVudC5kYXRhc2V0c30gaW4gY2xpZW50IHtjbGllbnR9IgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIEV4Y2VwdGlvbigiZGF0YXNldCBub3QgZm91bmQgb24gZGFzayBjbHVzdGVyIikKCiAgICBpZiBnZXRfZnJvbV90YWJsZToKICAgICAgICBpZiB0eXBlKHRhYmxlKSA9PSBEYXRhSXRlbToKICAgICAgICAgICAgaWYgdGFibGUubWV0YSBpcyBOb25lOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgVHJ1ZSwgRmFsc2UKICAgICAgICAgICAgZWxpZiB0YWJsZS5tZXRhLmtpbmQgPT0gImRhdGFzZXQiOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgRmFsc2UsIFRydWUKICAgICAgICAgICAgZWxpZiB0YWJsZS5tZXRhLmtpbmQgPT0gIkZlYXR1cmVWZWN0b3IiOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgRmFsc2UsIEZhbHNlCiAgICAgICAgICAgIGVsaWYgdGFibGUubWV0YS5raW5kID09ICJGZWF0dXJlU2V0IjoKICAgICAgICAgICAgICAgIGZlYXR1cmVzZXQsIGNyZWF0LCB1cGRhdGUgPSBUcnVlLCBGYWxzZSwgRmFsc2UKCiAgICAgICAgaWYgZGF0YV9pdGVtOgogICAgICAgICAgICBkZiA9IHRhYmxlLmFzX2RmKCkKICAgICAgICBlbGlmIGZlYXR1cmVzZXQ6CiAgICAgICAgICAgIHByb2plY3RfbmFtZSwgc2V0X25hbWUgPSAoCiAgICAgICAgICAgICAgICB0YWJsZS5fcGF0aC5zcGxpdCgiLyIpWzJdLAogICAgICAgICAgICAgICAgdGFibGUuX3BhdGguc3BsaXQoIi8iKVs0XSwKICAgICAgICAgICAgKQogICAgICAgICAgICBmZWF0dXJlX3NldCA9IGZzdG9yZS5nZXRfZmVhdHVyZV9zZXQoCiAgICAgICAgICAgICAgICBmInN0b3JlOi8vZmVhdHVyZS1zZXRzL3twcm9qZWN0X25hbWV9L3tzZXRfbmFtZX0iCiAgICAgICAgICAgICkKICAgICAgICAgICAgZGYgPSBmZWF0dXJlX3NldC50b19kYXRhZnJhbWUoKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKGYiV3JvbmcgdGFibGUgdHlwZS4iKQogICAgICAgICAgICByZXR1cm4KCiAgICBpZiBkZi5zaXplID4gTUFYX1NJWkVfT0ZfREY6CiAgICAgICAgZGYgPSBkZi5zYW1wbGUobj1pbnQoTUFYX1NJWkVfT0ZfREYgLyBkZi5zaGFwZVsxXSksIHJhbmRvbV9zdGF0ZT1yYW5kb21fc3RhdGUpCiAgICBleHRyYV9kYXRhID0ge30KCiAgICBpZiBsYWJlbF9jb2x1bW4gbm90IGluIGRmLmNvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1uID0gTm9uZQoKICAgIGV4dHJhX2RhdGFbImRlc2NyaWJlIGNzdiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgVGFibGVBcnRpZmFjdCgiZGVzY3JpYmUtY3N2IiwgZGY9ZGYuZGVzY3JpYmUoKSksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9kZXNjcmliZS5jc3YiLAogICAgKQoKICAgIHRyeToKICAgICAgICBfY3JlYXRlX2hpc3RvZ3JhbV9tYXRfYXJ0aWZhY3QoCiAgICAgICAgICAgIGNvbnRleHQsIGRmLCBleHRyYV9kYXRhLCBsYWJlbF9jb2x1bW4sIHBsb3RzX2Rlc3QKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgaGlzdG9ncmFtIG1hdHJpeCBhcnRpZmFjdCBkdWUgdG86IHtlfSIpCiAgICB0cnk6CiAgICAgICAgX2NyZWF0ZV9mZWF0dXJlc19oaXN0b2dyYW1fYXJ0aWZhY3RzKAogICAgICAgICAgICBjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0LCBwcm9ibGVtX3R5cGUKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgcGFpcnBsb3QgaGlzdG9ncmFtcyBkdWUgdG86IHtlfSIpCiAgICB0cnk6CiAgICAgICAgX2NyZWF0ZV9mZWF0dXJlc18yZF9zY2F0dGVyX2FydGlmYWN0cygKICAgICAgICAgICAgY29udGV4dCwgZGYsIGV4dHJhX2RhdGEsIGxhYmVsX2NvbHVtbiwgcGxvdHNfZGVzdCwgcHJvYmxlbV90eXBlCiAgICAgICAgKQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIHBhaXJwbG90IDJkX3NjYXR0ZXIgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfdmlvbGluX2FydGlmYWN0KGNvbnRleHQsIGRmLCBleHRyYV9kYXRhLCBwbG90c19kZXN0KQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIHZpb2xpbiBkaXN0cmlidXRpb24gcGxvdHMgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfaW1iYWxhbmNlX2FydGlmYWN0KAogICAgICAgICAgICBjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0LCBwcm9ibGVtX3R5cGUKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgY2xhc3MgaW1iYWxhbmNlIHBsb3QgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfY29ycl9hcnRpZmFjdChjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0KQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIGZlYXR1cmVzIGNvcnJlbGF0aW9uIHBsb3QgZHVlIHRvOiB7ZX0iKQoKICAgIGlmIG5vdCBkYXRhX2l0ZW06CiAgICAgICAgcmV0dXJuCgogICAgYXJ0aWZhY3QgPSB0YWJsZS5hcnRpZmFjdF91cmwKICAgIGlmIGNyZWF0OiAgIyBkYXRhc2V0IG5vdCBzdG9yZWQKICAgICAgICBhcnRpZmFjdCA9IERhdGFzZXRBcnRpZmFjdCgKICAgICAgICAgICAga2V5PSJkYXRhc2V0Iiwgc3RhdHM9VHJ1ZSwgZGY9ZGYsIGV4dHJhX2RhdGE9ZXh0cmFfZGF0YQogICAgICAgICkKICAgICAgICBhcnRpZmFjdCA9IGNvbnRleHQubG9nX2FydGlmYWN0KGFydGlmYWN0LCBkYl9rZXk9bmFtZSkKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVGhlIGRhdGEgc2V0IGlzIGxvZ2dlZCB0byB0aGUgcHJvamVjdCB1bmRlciB7bmFtZX0gbmFtZSIpCgogICAgaWYgdXBkYXRlOgogICAgICAgIHVwZGF0ZV9kYXRhc2V0X21ldGEoYXJ0aWZhY3QsIGV4dHJhX2RhdGE9ZXh0cmFfZGF0YSkKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVGhlIGRhdGEgc2V0IG5hbWVkIHtuYW1lfSBpcyB1cGRhdGVkIikKCiAgICAjIFRPRE8gOiAzLUQgcGxvdCBvbiBvbiBzZWxlY3RlZCBmZWF0dXJlcy4KICAgICMgVE9ETyA6IFJlaW50ZWdyYXRpb24gcGxvdCBvbiBvbiBzZWxlY3RlZCBmZWF0dXJlcy4KICAgICMgVE9ETyA6IFBDQSBwbG90ICh3aXRoIG9wdGlvbnMpCgoKZGVmIF9jcmVhdGVfaGlzdG9ncmFtX21hdF9hcnRpZmFjdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKKToKICAgICIiIgogICAgQ3JlYXRlIGFuZCBsb2cgYSBoaXN0b2dyYW0gbWF0cml4IGFydGlmYWN0CiAgICAiIiIKICAgIGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgIGl0ZW09QXJ0aWZhY3QoCiAgICAgICAgICAgIGtleT0iaGlzdCIsCiAgICAgICAgICAgIGJvZHk9YiI8Yj4gRGVwcmVjYXRlZCwgc2VlIHRoZSBhcnRpZmFjdHMgc2NhdHRlci0yZCAiCiAgICAgICAgICAgIGIiYW5kIGhpc3RvZ3JhbXMgaW5zdGVhZDxiPiIsCiAgICAgICAgKSwKICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2hpc3QuaHRtbCIsCiAgICApCgoKZGVmIF9jcmVhdGVfZmVhdHVyZXNfaGlzdG9ncmFtX2FydGlmYWN0cygKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhIGhpc3RvZ3JhbSBhcnRpZmFjdCBmb3IgZWFjaCBmZWF0dXJlCiAgICAiIiIKCiAgICBmaWdzID0gZGljdCgpCiAgICBmaXJzdF9mZWF0dXJlX25hbWUgPSAiIgogICAgaWYgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lIGFuZCBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICBhbGxfbGFiZWxzID0gZGZbbGFiZWxfY29sdW1uXS51bmlxdWUoKQogICAgdmlzaWJsZSA9IFRydWUKICAgIGZvciBjb2x1bW5fbmFtZSBpbiBkZi5jb2x1bW5zOgogICAgICAgIGlmIGNvbHVtbl9uYW1lID09IGxhYmVsX2NvbHVtbjoKICAgICAgICAgICAgY29udGludWUKCiAgICAgICAgaWYgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lIGFuZCBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICAgICAgZm9yIGxhYmVsIGluIGFsbF9sYWJlbHM6CiAgICAgICAgICAgICAgICBzdWJfZmlnID0gZ28uSGlzdG9ncmFtKAogICAgICAgICAgICAgICAgICAgIGhpc3RmdW5jPSJjb3VudCIsCiAgICAgICAgICAgICAgICAgICAgeD1kZi5sb2NbZGZbbGFiZWxfY29sdW1uXSA9PSBsYWJlbF1bY29sdW1uX25hbWVdLAogICAgICAgICAgICAgICAgICAgIG5hbWU9c3RyKGxhYmVsKSwKICAgICAgICAgICAgICAgICAgICB2aXNpYmxlPXZpc2libGUsCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBmaWdzW2Yie2NvbHVtbl9uYW1lfUA/QHtsYWJlbH0iXSA9IHN1Yl9maWcKICAgICAgICBlbHNlOgogICAgICAgICAgICBzdWJfZmlnID0gZ28uSGlzdG9ncmFtKGhpc3RmdW5jPSJjb3VudCIsIHg9ZGZbY29sdW1uX25hbWVdLCB2aXNpYmxlPXZpc2libGUpCiAgICAgICAgICAgIGZpZ3NbZiJ7Y29sdW1uX25hbWV9QD9AezF9Il0gPSBzdWJfZmlnCiAgICAgICAgaWYgdmlzaWJsZToKICAgICAgICAgICAgZmlyc3RfZmVhdHVyZV9uYW1lID0gY29sdW1uX25hbWUKICAgICAgICB2aXNpYmxlID0gRmFsc2UKCiAgICBmaWcgPSBnby5GaWd1cmUoKQogICAgZm9yIGsgaW4gZmlncy5rZXlzKCk6CiAgICAgICAgZmlnLmFkZF90cmFjZShmaWdzW2tdKQoKICAgIGZpZy51cGRhdGVfbGF5b3V0KAogICAgICAgIHVwZGF0ZW1lbnVzPVsKICAgICAgICAgICAgewogICAgICAgICAgICAgICAgImJ1dHRvbnMiOiBbCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAibGFiZWwiOiBjb2x1bW5fbmFtZSwKICAgICAgICAgICAgICAgICAgICAgICAgIm1ldGhvZCI6ICJ1cGRhdGUiLAogICAgICAgICAgICAgICAgICAgICAgICAiYXJncyI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidmlzaWJsZSI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAga2V5LnNwbGl0KCJAP0AiKVswXSA9PSBjb2x1bW5fbmFtZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb3Iga2V5IGluIGZpZ3Mua2V5cygpCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAieGF4aXMiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJyYW5nZSI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1pbihkZltjb2x1bW5fbmFtZV0pLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWF4KGRmW2NvbHVtbl9uYW1lXSksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHsidGl0bGUiOiBmIjxpPjxiPkhpc3RvZ3JhbSBvZiB7Y29sdW1uX25hbWV9PC9iPjwvaT4ifSwKICAgICAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICAgICAgZm9yIGNvbHVtbl9uYW1lIGluIGRmLmNvbHVtbnMKICAgICAgICAgICAgICAgICAgICBpZiBjb2x1bW5fbmFtZSAhPSBsYWJlbF9jb2x1bW4KICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICAiZGlyZWN0aW9uIjogImRvd24iLAogICAgICAgICAgICAgICAgInBhZCI6IHsiciI6IDEwLCAidCI6IDEwfSwKICAgICAgICAgICAgICAgICJzaG93YWN0aXZlIjogVHJ1ZSwKICAgICAgICAgICAgICAgICJ4IjogMC4yNSwKICAgICAgICAgICAgICAgICJ4YW5jaG9yIjogImxlZnQiLAogICAgICAgICAgICAgICAgInkiOiAxLjEsCiAgICAgICAgICAgICAgICAieWFuY2hvciI6ICJ0b3AiLAogICAgICAgICAgICB9CiAgICAgICAgXSwKICAgICAgICBhbm5vdGF0aW9ucz1bCiAgICAgICAgICAgIGRpY3QoCiAgICAgICAgICAgICAgICB0ZXh0PSJTZWxlY3QgRmVhdHVyZSBOYW1lICIsCiAgICAgICAgICAgICAgICBzaG93YXJyb3c9RmFsc2UsCiAgICAgICAgICAgICAgICB4PTAsCiAgICAgICAgICAgICAgICB5PTEuMDUsCiAgICAgICAgICAgICAgICB5cmVmPSJwYXBlciIsCiAgICAgICAgICAgICAgICB4cmVmPSJwYXBlciIsCiAgICAgICAgICAgICAgICBhbGlnbj0ibGVmdCIsCiAgICAgICAgICAgICAgICB4YW5jaG9yPSJsZWZ0IiwKICAgICAgICAgICAgICAgIHlhbmNob3I9InRvcCIsCiAgICAgICAgICAgICAgICBmb250PXsKICAgICAgICAgICAgICAgICAgICAiY29sb3IiOiAiYmx1ZSIsCiAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICApCiAgICAgICAgXSwKICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dCgKICAgICAgICB3aWR0aD02MDAsCiAgICAgICAgaGVpZ2h0PTQwMCwKICAgICAgICBhdXRvc2l6ZT1GYWxzZSwKICAgICAgICBtYXJnaW49ZGljdCh0PTEwMCwgYj0wLCBsPTAsIHI9MCksCiAgICAgICAgdGVtcGxhdGU9InBsb3RseV93aGl0ZSIsCiAgICApCgogICAgZmlnLnVwZGF0ZV9sYXlvdXQodGl0bGVfdGV4dD1mIjxpPjxiPkhpc3RvZ3JhbXMgb2Yge2ZpcnN0X2ZlYXR1cmVfbmFtZX08L2I+PC9pPiIpCiAgICBleHRyYV9kYXRhW2YiaGlzdG9ncmFtcyJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PWYiaGlzdG9ncmFtcyIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vaGlzdG9ncmFtcy5odG1sIiwKICAgICkKCgpkZWYgX2NyZWF0ZV9mZWF0dXJlc18yZF9zY2F0dGVyX2FydGlmYWN0cygKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhIHNjYXR0ZXItMmQgYXJ0aWZhY3QgZm9yIGVhY2ggY291cGxlIG9mIGZlYXR1cmVzCiAgICAiIiIKICAgIGZlYXR1cmVzID0gWwogICAgICAgIGNvbHVtbl9uYW1lIGZvciBjb2x1bW5fbmFtZSBpbiBkZi5jb2x1bW5zIGlmIGNvbHVtbl9uYW1lICE9IGxhYmVsX2NvbHVtbgogICAgXQogICAgbWF4X2ZlYXR1cmVfbGVuID0gZmxvYXQobWF4KGxlbihlbGVtKSBmb3IgZWxlbSBpbiBmZWF0dXJlcykpCiAgICBpZiBsYWJlbF9jb2x1bW4gaXMgbm90IE5vbmU6CiAgICAgICAgbGFiZWxzID0gc29ydGVkKGRmW2xhYmVsX2NvbHVtbl0udW5pcXVlKCkpCiAgICBlbHNlOgogICAgICAgIGxhYmVscyA9IFtOb25lXQogICAgZmlnID0gZ28uRmlndXJlKCkKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZSBhbmQgcHJvYmxlbV90eXBlID09ICJjbGFzc2lmaWNhdGlvbiI6CiAgICAgICAgZm9yIGwgaW4gbGFiZWxzOgogICAgICAgICAgICBmaWcuYWRkX3RyYWNlKAogICAgICAgICAgICAgICAgZ28uU2NhdHRlcigKICAgICAgICAgICAgICAgICAgICB4PWRmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgICAgICB5PWRmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgICAgICBtb2RlPSJtYXJrZXJzIiwKICAgICAgICAgICAgICAgICAgICB2aXNpYmxlPVRydWUsCiAgICAgICAgICAgICAgICAgICAgc2hvd2xlZ2VuZD1UcnVlLAogICAgICAgICAgICAgICAgICAgIG5hbWU9c3RyKGwpLAogICAgICAgICAgICAgICAgKQogICAgICAgICAgICApCiAgICBlbGlmIGxhYmVsX2NvbHVtbiBpcyBOb25lOgogICAgICAgIGZpZy5hZGRfdHJhY2UoCiAgICAgICAgICAgIGdvLlNjYXR0ZXIoCiAgICAgICAgICAgICAgICB4PWRmW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgIHk9ZGZbZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgbW9kZT0ibWFya2VycyIsCiAgICAgICAgICAgICAgICB2aXNpYmxlPVRydWUsCiAgICAgICAgICAgICkKICAgICAgICApCiAgICBlbGlmIHByb2JsZW1fdHlwZSA9PSAicmVncmVzc2lvbiI6CiAgICAgICAgZmlnLmFkZF90cmFjZSgKICAgICAgICAgICAgZ28uU2NhdHRlcigKICAgICAgICAgICAgICAgIHg9ZGZbZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgeT1kZltmZWF0dXJlc1swXV0sCiAgICAgICAgICAgICAgICBtb2RlPSJtYXJrZXJzIiwKICAgICAgICAgICAgICAgIG1hcmtlcj1kaWN0KAogICAgICAgICAgICAgICAgICAgIGNvbG9yPWRmW2xhYmVsX2NvbHVtbl0sIGNvbG9yc2NhbGU9IlZpcmlkaXMiLCBzaG93c2NhbGU9VHJ1ZQogICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgIHZpc2libGU9VHJ1ZSwKICAgICAgICAgICAgKQogICAgICAgICkKCiAgICB4X2J1dHRvbnMgPSBbXQogICAgeV9idXR0b25zID0gW10KCiAgICBmb3IgbmNvbCBpbiBmZWF0dXJlczoKICAgICAgICBpZiBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIiBhbmQgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lOgogICAgICAgICAgICB4X2J1dHRvbnMuYXBwZW5kKAogICAgICAgICAgICAgICAgZGljdCgKICAgICAgICAgICAgICAgICAgICBtZXRob2Q9InVwZGF0ZSIsCiAgICAgICAgICAgICAgICAgICAgbGFiZWw9bmNvbCwKICAgICAgICAgICAgICAgICAgICBhcmdzPVsKICAgICAgICAgICAgICAgICAgICAgICAgeyJ4IjogW2RmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW25jb2xdIGZvciBsIGluIGxhYmVsc119LAogICAgICAgICAgICAgICAgICAgICAgICBucC5hcmFuZ2UobGVuKGxhYmVscykpLnRvbGlzdCgpLAogICAgICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICkKCiAgICAgICAgICAgIHlfYnV0dG9ucy5hcHBlbmQoCiAgICAgICAgICAgICAgICBkaWN0KAogICAgICAgICAgICAgICAgICAgIG1ldGhvZD0idXBkYXRlIiwKICAgICAgICAgICAgICAgICAgICBsYWJlbD1uY29sLAogICAgICAgICAgICAgICAgICAgIGFyZ3M9WwogICAgICAgICAgICAgICAgICAgICAgICB7InkiOiBbZGYubG9jW2RmW2xhYmVsX2NvbHVtbl0gPT0gbF1bbmNvbF0gZm9yIGwgaW4gbGFiZWxzXX0sCiAgICAgICAgICAgICAgICAgICAgICAgIG5wLmFyYW5nZShsZW4obGFiZWxzKSkudG9saXN0KCksCiAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHhfYnV0dG9ucy5hcHBlbmQoCiAgICAgICAgICAgICAgICBkaWN0KG1ldGhvZD0idXBkYXRlIiwgbGFiZWw9bmNvbCwgYXJncz1beyJ4IjogW2RmW25jb2xdXX1dKQogICAgICAgICAgICApCgogICAgICAgICAgICB5X2J1dHRvbnMuYXBwZW5kKAogICAgICAgICAgICAgICAgZGljdChtZXRob2Q9InVwZGF0ZSIsIGxhYmVsPW5jb2wsIGFyZ3M9W3sieSI6IFtkZltuY29sXV19XSkKICAgICAgICAgICAgKQoKICAgICMgUGFzcyBidXR0b25zIHRvIHRoZSB1cGRhdGVtZW51cyBhcmd1bWVudAogICAgZmlnLnVwZGF0ZV9sYXlvdXQoCiAgICAgICAgdXBkYXRlbWVudXM9WwogICAgICAgICAgICBkaWN0KGJ1dHRvbnM9eF9idXR0b25zLCBkaXJlY3Rpb249InVwIiwgeD0wLjUsIHk9LTAuMSksCiAgICAgICAgICAgIGRpY3QoYnV0dG9ucz15X2J1dHRvbnMsIGRpcmVjdGlvbj0iZG93biIsIHg9LW1heF9mZWF0dXJlX2xlbiAvIDEwMCwgeT0wLjUpLAogICAgICAgIF0KICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dCgKICAgICAgICB3aWR0aD02MDAsCiAgICAgICAgaGVpZ2h0PTQwMCwKICAgICAgICBhdXRvc2l6ZT1GYWxzZSwKICAgICAgICBtYXJnaW49ZGljdCh0PTEwMCwgYj0wLCBsPTAsIHI9MCksCiAgICAgICAgdGVtcGxhdGU9InBsb3RseV93aGl0ZSIsCiAgICApCgogICAgZmlnLnVwZGF0ZV9sYXlvdXQodGl0bGVfdGV4dD1mIjxpPjxiPlNjYXR0ZXItMmQ8L2I+PC9pPiIpCiAgICBleHRyYV9kYXRhW2Yic2NhdHRlci0yZCJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PWYic2NhdHRlci0yZCIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vc2NhdHRlci0yZC5odG1sIiwKICAgICkKCgpkZWYgX2NyZWF0ZV92aW9saW5fYXJ0aWZhY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwgZGY6IHBkLkRhdGFGcmFtZSwgZXh0cmFfZGF0YTogZGljdCwgcGxvdHNfZGVzdDogc3RyCik6CiAgICAiIiIKICAgIENyZWF0ZSBhbmQgbG9nIGEgdmlvbGluIGFydGlmYWN0CiAgICAiIiIKICAgIGNvbHMgPSA1CiAgICByb3dzID0gKGRmLnNoYXBlWzFdIC8vIGNvbHMpICsgMQogICAgZmlnID0gbWFrZV9zdWJwbG90cyhyb3dzPXJvd3MsIGNvbHM9Y29scykKCiAgICBwbG90X251bSA9IDAKCiAgICBmb3IgY29sdW1uX25hbWUgaW4gZGYuY29sdW1uczoKICAgICAgICBjb2x1bW5fZGF0YSA9IGRmW2NvbHVtbl9uYW1lXQogICAgICAgIHZpb2xpbiA9IGdvLlZpb2xpbigKICAgICAgICAgICAgeD1bY29sdW1uX25hbWVdICogY29sdW1uX2RhdGEuc2hhcGVbMF0sCiAgICAgICAgICAgIHk9Y29sdW1uX2RhdGEsCiAgICAgICAgICAgIG5hbWU9Y29sdW1uX25hbWUsCiAgICAgICAgKQoKICAgICAgICBmaWcuYWRkX3RyYWNlKAogICAgICAgICAgICB2aW9saW4sCiAgICAgICAgICAgIHJvdz0ocGxvdF9udW0gLy8gY29scykgKyAxLAogICAgICAgICAgICBjb2w9KHBsb3RfbnVtICUgY29scykgKyAxLAogICAgICAgICkKCiAgICAgICAgcGxvdF9udW0gKz0gMQoKICAgIGZpZ1sibGF5b3V0Il0udXBkYXRlKAogICAgICAgIGhlaWdodD0ocm93cyArIDEpICogMjAwLAogICAgICAgIHdpZHRoPShjb2xzICsgMSkgKiAyMDAsCiAgICAgICAgdGl0bGU9IjxpPjxiPlZpb2xpbiBQbG90czwvYj48L2k+IiwKICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dChzaG93bGVnZW5kPUZhbHNlKQogICAgZXh0cmFfZGF0YVsidmlvbGluIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBQbG90bHlBcnRpZmFjdChrZXk9InZpb2xpbiIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vdmlvbGluLmh0bWwiLAogICAgKQoKCmRlZiBfY3JlYXRlX2ltYmFsYW5jZV9hcnRpZmFjdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhbiBpbWJhbGFuY2UgY2xhc3MgYXJ0aWZhY3QgKGNzdiArIHBsb3QpCiAgICAiIiIKICAgIGlmIGxhYmVsX2NvbHVtbjoKICAgICAgICBpZiBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICAgICAgdmFsdWVzX2NvbHVtbiA9ICJjb3VudCIKICAgICAgICAgICAgbGFiZWxzX2NvdW50ID0gZGZbbGFiZWxfY29sdW1uXS52YWx1ZV9jb3VudHMoKS5zb3J0X2luZGV4KCkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50ID0gcGQuRGF0YUZyYW1lKGxhYmVsc19jb3VudCkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W2xhYmVsX2NvbHVtbl0gPSBsYWJlbHNfY291bnQuaW5kZXgKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50LnJlbmFtZShjb2x1bW5zPXsiIjogdmFsdWVzX2NvbHVtbn0sIGlucGxhY2U9VHJ1ZSkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dID0gZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dIC8gc3VtKAogICAgICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dCiAgICAgICAgICAgICkKICAgICAgICAgICAgZmlnID0gcHgucGllKGRmX2xhYmVsc19jb3VudCwgbmFtZXM9bGFiZWxfY29sdW1uLCB2YWx1ZXM9dmFsdWVzX2NvbHVtbikKICAgICAgICBlbHNlOgogICAgICAgICAgICBmaWcgPSBweC5oaXN0b2dyYW0oCiAgICAgICAgICAgICAgICBoaXN0ZnVuYz0iY291bnQiLAogICAgICAgICAgICAgICAgeD1kZltsYWJlbF9jb2x1bW5dLAogICAgICAgICAgICApCiAgICAgICAgICAgIGhpc3QgPSBucC5oaXN0b2dyYW0oZGZbbGFiZWxfY29sdW1uXSkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50ID0gcGQuRGF0YUZyYW1lKAogICAgICAgICAgICAgICAgeyJtaW5fdmFsIjogaGlzdFsxXSwgImNvdW50IjogaGlzdFswXS50b2xpc3QoKSArIFswXX0KICAgICAgICAgICAgKQogICAgICAgIGZpZy51cGRhdGVfbGF5b3V0KHRpdGxlX3RleHQ9IjxpPjxiPkxhYmVscyBJbWJhbGFuY2U8L2I+PC9pPiIpCiAgICAgICAgZXh0cmFfZGF0YVsiaW1iYWxhbmNlIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJpbWJhbGFuY2UiLCBmaWd1cmU9ZmlnKSwKICAgICAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9pbWJhbGFuY2UuaHRtbCIsCiAgICAgICAgKQogICAgICAgIGV4dHJhX2RhdGFbImltYmFsYW5jZS1jc3YiXSA9IGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgICAgICBUYWJsZUFydGlmYWN0KCJpbWJhbGFuY2Utd2VpZ2h0cy12ZWMiLCBkZj1kZl9sYWJlbHNfY291bnQpLAogICAgICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2ltYmFsYW5jZS13ZWlnaHRzLXZlYy5jc3YiLAogICAgICAgICkKCgpkZWYgX2NyZWF0ZV9jb3JyX2FydGlmYWN0KAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkZjogcGQuRGF0YUZyYW1lLAogICAgZXh0cmFfZGF0YTogZGljdCwKICAgIGxhYmVsX2NvbHVtbjogc3RyLAogICAgcGxvdHNfZGVzdDogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhbiBjb3JyZWxhdGlvbi1tYXRyaXggYXJ0aWZhY3QgKGNzdiArIHBsb3QpCiAgICAiIiIKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZToKICAgICAgICBkZiA9IGRmLmRyb3AoW2xhYmVsX2NvbHVtbl0sIGF4aXM9MSkKICAgIHRibGNvcnIgPSBkZi5jb3JyKG51bWVyaWNfb25seT1UcnVlKQogICAgZXh0cmFfZGF0YVsiY29ycmVsYXRpb24tbWF0cml4LWNzdiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgVGFibGVBcnRpZmFjdCgiY29ycmVsYXRpb24tbWF0cml4LWNzdiIsIGRmPXRibGNvcnIsIHZpc2libGU9VHJ1ZSksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9jb3JyZWxhdGlvbi1tYXRyaXguY3N2IiwKICAgICkKCiAgICB6ID0gdGJsY29yci52YWx1ZXMudG9saXN0KCkKICAgIHpfdGV4dCA9IFtbIns6LjJmfSIuZm9ybWF0KHkpIGZvciB5IGluIHhdIGZvciB4IGluIHpdCiAgICBmaWcgPSBmZi5jcmVhdGVfYW5ub3RhdGVkX2hlYXRtYXAoCiAgICAgICAgeiwKICAgICAgICB4PWxpc3QodGJsY29yci5jb2x1bW5zKSwKICAgICAgICB5PWxpc3QodGJsY29yci5jb2x1bW5zKSwKICAgICAgICBhbm5vdGF0aW9uX3RleHQ9el90ZXh0LAogICAgICAgIGNvbG9yc2NhbGU9ImFnc3Vuc2V0IiwKICAgICkKICAgIGZpZ1sibGF5b3V0Il1bInlheGlzIl1bImF1dG9yYW5nZSJdID0gInJldmVyc2VkIiAgIyBsIC0+IHIKICAgIGZpZy51cGRhdGVfbGF5b3V0KHRpdGxlX3RleHQ9IjxpPjxiPkNvcnJlbGF0aW9uIG1hdHJpeDwvYj48L2k+IikKICAgIGZpZ1siZGF0YSJdWzBdWyJzaG93c2NhbGUiXSA9IFRydWUKCiAgICBleHRyYV9kYXRhWyJjb3JyZWxhdGlvbiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJjb3JyZWxhdGlvbiIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vY29ycmVsYXRpb24uaHRtbCIsCiAgICApCg== diff --git a/functions/src/describe/item.yaml b/functions/src/describe/item.yaml index da26f1501..a1aa47372 100644 --- a/functions/src/describe/item.yaml +++ b/functions/src/describe/item.yaml @@ -11,7 +11,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.0 +mlrunVersion: 1.10.0 name: describe platformVersion: 3.5.3 spec: @@ -21,4 +21,4 @@ spec: kind: job requirements: [] url: '' -version: 1.4.0 +version: 1.5.0 diff --git a/functions/src/describe/requirements.txt b/functions/src/describe/requirements.txt index 15492b176..ac445e6d6 100644 --- a/functions/src/describe/requirements.txt +++ b/functions/src/describe/requirements.txt @@ -1,4 +1,4 @@ -scikit-learn~=1.0.2 +scikit-learn~=1.5.2 plotly~=5.23 pytest~=7.0.1 matplotlib~=3.5.1 diff --git a/functions/src/gen_class_data/function.yaml b/functions/src/gen_class_data/function.yaml index 1769bec07..fa802964e 100644 --- a/functions/src/gen_class_data/function.yaml +++ b/functions/src/gen_class_data/function.yaml @@ -1,13 +1,15 @@ metadata: - categories: - - data-generation tag: '' name: gen-class-data + categories: + - data-generation +verbose: false spec: description: Create a binary classification sample dataset and save. - default_handler: gen_class_data entry_points: gen_class_data: + lineno: 22 + has_varargs: false has_kwargs: false parameters: - name: context @@ -48,7 +50,6 @@ spec: - name: sk_params doc: additional parameters for `sklearn.datasets.make_classification` default: {} - lineno: 22 doc: 'Create a binary classification sample dataset and save. If no filename is given it will default to: @@ -59,14 +60,13 @@ spec: Additional scikit-learn parameters can be set using **sk_params, please see https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_classification.html for more details.' - has_varargs: false name: gen_class_data - command: '' - disable_auto_mount: false - image: mlrun/mlrun build: origin_filename: '' functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IHBhbmRhcyBhcyBwZApmcm9tIHR5cGluZyBpbXBvcnQgT3B0aW9uYWwsIExpc3QKZnJvbSBza2xlYXJuLmRhdGFzZXRzIGltcG9ydCBtYWtlX2NsYXNzaWZpY2F0aW9uCgpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKCgpkZWYgZ2VuX2NsYXNzX2RhdGEoCiAgICAgICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICAgICAgbl9zYW1wbGVzOiBpbnQsCiAgICAgICAgbV9mZWF0dXJlczogaW50LAogICAgICAgIGtfY2xhc3NlczogaW50LAogICAgICAgIGhlYWRlcjogT3B0aW9uYWxbTGlzdFtzdHJdXSwKICAgICAgICBsYWJlbF9jb2x1bW46IE9wdGlvbmFsW3N0cl0gPSAibGFiZWxzIiwKICAgICAgICB3ZWlnaHQ6IGZsb2F0ID0gMC41LAogICAgICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gMSwKICAgICAgICBrZXk6IHN0ciA9ICJjbGFzc2lmaWVyLWRhdGEiLAogICAgICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICAgICAgc2tfcGFyYW1zPXt9Cik6CiAgICAiIiJDcmVhdGUgYSBiaW5hcnkgY2xhc3NpZmljYXRpb24gc2FtcGxlIGRhdGFzZXQgYW5kIHNhdmUuCiAgICBJZiBubyBmaWxlbmFtZSBpcyBnaXZlbiBpdCB3aWxsIGRlZmF1bHQgdG86CiAgICAic2ltZGF0YS17bl9zYW1wbGVzfVh7bV9mZWF0dXJlc30ucGFycXVldCIuCgogICAgQWRkaXRpb25hbCBzY2lraXQtbGVhcm4gcGFyYW1ldGVycyBjYW4gYmUgc2V0IHVzaW5nICoqc2tfcGFyYW1zLCBwbGVhc2Ugc2VlIGh0dHBzOi8vc2Npa2l0LWxlYXJuLm9yZy9zdGFibGUvbW9kdWxlcy9nZW5lcmF0ZWQvc2tsZWFybi5kYXRhc2V0cy5tYWtlX2NsYXNzaWZpY2F0aW9uLmh0bWwgZm9yIG1vcmUgZGV0YWlscy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgZnVuY3Rpb24gY29udGV4dAogICAgOnBhcmFtIG5fc2FtcGxlczogICAgIG51bWJlciBvZiByb3dzL3NhbXBsZXMKICAgIDpwYXJhbSBtX2ZlYXR1cmVzOiAgICBudW1iZXIgb2YgY29scy9mZWF0dXJlcwogICAgOnBhcmFtIGtfY2xhc3NlczogICAgIG51bWJlciBvZiBjbGFzc2VzCiAgICA6cGFyYW0gaGVhZGVyOiAgICAgICAgaGVhZGVyIGZvciBmZWF0dXJlcyBhcnJheQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogIGNvbHVtbiBuYW1lIG9mIGdyb3VuZC10cnV0aCBzZXJpZXMKICAgIDpwYXJhbSB3ZWlnaHQ6ICAgICAgICBmcmFjdGlvbiBvZiBzYW1wbGUgbmVnYXRpdmUgdmFsdWUgKGdyb3VuZC10cnV0aD0wKQogICAgOnBhcmFtIHJhbmRvbV9zdGF0ZTogIHJuZyBzZWVkIChzZWUgaHR0cHM6Ly9zY2lraXQtbGVhcm4ub3JnL3N0YWJsZS9nbG9zc2FyeS5odG1sI3Rlcm0tcmFuZG9tLXN0YXRlKQogICAgOnBhcmFtIGtleTogICAgICAgICAgIGtleSBvZiBkYXRhIGluIGFydGlmYWN0IHN0b3JlCiAgICA6cGFyYW0gZmlsZV9leHQ6ICAgICAgKHBxdCkgZXh0ZW5zaW9uIGZvciBwYXJxdWV0IGZpbGUKICAgIDpwYXJhbSBza19wYXJhbXM6ICAgICBhZGRpdGlvbmFsIHBhcmFtZXRlcnMgZm9yIGBza2xlYXJuLmRhdGFzZXRzLm1ha2VfY2xhc3NpZmljYXRpb25gCiAgICAiIiIKICAgIGZlYXR1cmVzLCBsYWJlbHMgPSBtYWtlX2NsYXNzaWZpY2F0aW9uKAogICAgICAgIG5fc2FtcGxlcz1uX3NhbXBsZXMsCiAgICAgICAgbl9mZWF0dXJlcz1tX2ZlYXR1cmVzLAogICAgICAgIHdlaWdodHM9d2VpZ2h0LAogICAgICAgIG5fY2xhc3Nlcz1rX2NsYXNzZXMsCiAgICAgICAgcmFuZG9tX3N0YXRlPXJhbmRvbV9zdGF0ZSwKICAgICAgICAqKnNrX3BhcmFtcykKCiAgICAjIG1ha2UgZGF0YWZyYW1lcywgYWRkIGNvbHVtbiBuYW1lcywgY29uY2F0ZW5hdGUgKFgsIHkpCiAgICBYID0gcGQuRGF0YUZyYW1lKGZlYXR1cmVzKQogICAgaWYgbm90IGhlYWRlcjoKICAgICAgICBYLmNvbHVtbnMgPSBbImZlYXRfIiArIHN0cih4KSBmb3IgeCBpbiByYW5nZShtX2ZlYXR1cmVzKV0KICAgIGVsc2U6CiAgICAgICAgWC5jb2x1bW5zID0gaGVhZGVyCgogICAgeSA9IHBkLkRhdGFGcmFtZShsYWJlbHMsIGNvbHVtbnM9W2xhYmVsX2NvbHVtbl0pCiAgICBkYXRhID0gcGQuY29uY2F0KFtYLCB5XSwgYXhpcz0xKQoKICAgIGNvbnRleHQubG9nX2RhdGFzZXQoa2V5LCBkZj1kYXRhLCBmb3JtYXQ9ZmlsZV9leHQsIGluZGV4PUZhbHNlKQo= code_origin: '' + command: '' + image: mlrun/mlrun + default_handler: gen_class_data + disable_auto_mount: false kind: job -verbose: false diff --git a/functions/src/gen_class_data/item.yaml b/functions/src/gen_class_data/item.yaml index 30f5cd21c..082b00305 100644 --- a/functions/src/gen_class_data/item.yaml +++ b/functions/src/gen_class_data/item.yaml @@ -11,7 +11,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.0 +mlrunVersion: 1.10.0 name: gen_class_data platformVersion: 3.5.3 spec: @@ -21,4 +21,4 @@ spec: kind: job requirements: [] url: '' -version: 1.3.0 +version: 1.4.0 diff --git a/functions/src/gen_class_data/requirements.txt b/functions/src/gen_class_data/requirements.txt index d7dbe376b..e265290f6 100644 --- a/functions/src/gen_class_data/requirements.txt +++ b/functions/src/gen_class_data/requirements.txt @@ -1,2 +1,2 @@ pandas -scikit-learn==1.0.2 \ No newline at end of file +scikit-learn~=1.5.2 \ No newline at end of file diff --git a/functions/src/gen_class_data/test_gen_class_data.py b/functions/src/gen_class_data/test_gen_class_data.py index e06eeb16b..990075dec 100644 --- a/functions/src/gen_class_data/test_gen_class_data.py +++ b/functions/src/gen_class_data/test_gen_class_data.py @@ -36,4 +36,7 @@ def test_gen_class_data(): local=True, artifact_path="./artifacts", ) - assert os.path.isfile(run.status.artifacts[0]['spec']['target_path']), 'dataset is not available' + # In local mode, artifacts are in function-name/iteration subdirectory + # Default key is "classifier-data" (can be overridden in params) + dataset_path = "./artifacts/test-gen-class-data-gen-class-data/0/classifier-data.csv" + assert os.path.isfile(dataset_path), f'dataset is not available at {dataset_path}'