From 3ab150c4791d82fdb113b4c7cfffbb959dd15de4 Mon Sep 17 00:00:00 2001 From: tomerbv Date: Mon, 19 Jan 2026 16:39:04 +0200 Subject: [PATCH 01/15] updated scikit-learn~=1.5 fixes and patches for new scikit-learn version changes in item.yaml and regenerate function.yaml --- functions/src/auto_trainer/function.yaml | 59 ++++----- functions/src/auto_trainer/item.yaml | 4 +- functions/src/auto_trainer/requirements.txt | 2 +- .../src/auto_trainer/test_auto_trainer.py | 37 +++++- functions/src/describe/function.yaml | 83 ++++++------ functions/src/describe/item.yaml | 4 +- functions/src/describe/requirements.txt | 2 +- functions/src/gen_class_data/function.yaml | 19 +-- functions/src/gen_class_data/item.yaml | 4 +- functions/src/gen_class_data/requirements.txt | 2 +- .../src/gen_class_data/test_gen_class_data.py | 5 +- .../src/sklearn_classifier/function.yaml | 110 ++++++++++++++-- functions/src/sklearn_classifier/item.yaml | 4 +- .../src/sklearn_classifier/requirements.txt | 2 +- .../sklearn_classifier/sklearn_classifier.py | 122 +++++++++++++++++- .../test_sklearn_classifier.py | 28 ++-- 16 files changed, 368 insertions(+), 119 deletions(-) diff --git a/functions/src/auto_trainer/function.yaml b/functions/src/auto_trainer/function.yaml index 0920b1033..3020b6521 100644 --- a/functions/src/auto_trainer/function.yaml +++ b/functions/src/auto_trainer/function.yaml @@ -1,22 +1,31 @@ -metadata: - categories: - - machine-learning - - model-training - tag: '' - name: auto-trainer +kind: job spec: - image: mlrun/mlrun build: - origin_filename: '' functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICBzdG9yZV91cmlfcHJlZml4LCBfID0gbWxydW4uZGF0YXN0b3JlLnBhcnNlX3N0b3JlX3VyaShkYXRhc2V0LmFydGlmYWN0X3VybCkKCiAgICAjIEdldHRpbmcgdGhlIGRhdGFzZXQ6CiAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IGxhYmVsX2NvbHVtbnMgb3IgZGF0YXNldC5tZXRhLnN0YXR1cy5sYWJlbF9jb2x1bW4KICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYibGFiZWwgY29sdW1uczoge2xhYmVsX2NvbHVtbnN9IikKICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICB0cnk6CiAgICAgICAgICAgIGZ2ID0gbWxydW4uZGF0YXN0b3JlLmdldF9zdG9yZV9yZXNvdXJjZShkYXRhc2V0LmFydGlmYWN0X3VybCkKICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZXhjZXB0IEF0dHJpYnV0ZUVycm9yOgogICAgICAgICAgICAjIExlYXZlIGhlcmUgZm9yIGJhY2t3YXJkcyBjb21wYXRpYmlsaXR5CiAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgIGRhdGFzZXQubWV0YS51cmksIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMKICAgICAgICAgICAgKS50b19kYXRhZnJhbWUoKQoKICAgIGVsaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgImxhYmVsX2NvbHVtbnMgbm90IHByb3ZpZGVkLCBtYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBlbGlmIGlzaW5zdGFuY2UoZGF0YXNldCwgKGxpc3QsIGRpY3QpKToKICAgICAgICAjIGxpc3QvZGljdCBjYXNlOgogICAgICAgIGRhdGFzZXQgPSBwZC5EYXRhRnJhbWUoZGF0YXNldCkKICAgICAgICAjIENoZWNraW5nIGlmIGRyb3BfY29sdW1ucyBwcm92aWRlZCBieSBpbnRlZ2VyIHR5cGU6CiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICBpZiBpc2luc3RhbmNlKGRyb3BfY29sdW1ucywgc3RyKSBvciAoCiAgICAgICAgICAgICAgICBpc2luc3RhbmNlKGRyb3BfY29sdW1ucywgbGlzdCkKICAgICAgICAgICAgICAgIGFuZCBhbnkoaXNpbnN0YW5jZShjb2wsIHN0cikgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpCiAgICAgICAgICAgICk6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIG11c3QgYmUgYW4gaW50ZWdlci9saXN0IG9mIGludGVnZXJzIGlmIG5vdCBwcm92aWRlZCB3aXRoIGEgVVJJL0ZlYXR1cmVWZWN0b3IgZGF0YXNldCIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xLCBpbnBsYWNlPVRydWUpCgogICAgZWxzZToKICAgICAgICAjIHNpbXBsZSBVUkwgY2FzZToKICAgICAgICBkYXRhc2V0ID0gZGF0YXNldC5hc19kZigpCiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSkKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgIm5vdCBhbGwgb2YgdGhlIGNvbHVtbnMgdG8gZHJvcCBpbiB0aGUgZGF0YXNldCwgZHJvcCBjb2x1bW5zIHByb2Nlc3Mgc2tpcHBlZCIKICAgICAgICAgICAgICAgICkKCiAgICByZXR1cm4gZGF0YXNldCwgbGFiZWxfY29sdW1ucwoKCmRlZiB0cmFpbigKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBtb2RlbF9jbGFzczogc3RyLAogICAgbGFiZWxfY29sdW1uczogT3B0aW9uYWxbVW5pb25bc3RyLCBMaXN0W3N0cl1dXSA9IE5vbmUsCiAgICBkcm9wX2NvbHVtbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSAibW9kZWwiLAogICAgdGFnOiBzdHIgPSAiIiwKICAgIHNhbXBsZV9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRlc3Rfc2V0OiBEYXRhSXRlbSA9IE5vbmUsCiAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemU6IGZsb2F0ID0gTm9uZSwKICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gTm9uZSwKICAgIGxhYmVsczogZGljdCA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgVHJhaW5pbmcgYSBtb2RlbCB3aXRoIHRoZSBnaXZlbiBkYXRhc2V0LgoKICAgIGV4YW1wbGU6OgoKICAgICAgICBpbXBvcnQgbWxydW4KICAgICAgICBwcm9qZWN0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9wcm9qZWN0KCJteS1wcm9qZWN0IikKICAgICAgICBwcm9qZWN0LnNldF9mdW5jdGlvbigiaHViOi8vYXV0b190cmFpbmVyIiwgInRyYWluIikKICAgICAgICB0cmFpbmVyX3J1biA9IHByb2plY3QucnVuKAogICAgICAgICAgICBuYW1lPSJ0cmFpbiIsCiAgICAgICAgICAgIGhhbmRsZXI9InRyYWluIiwKICAgICAgICAgICAgaW5wdXRzPXsiZGF0YXNldCI6ICIuL3BhdGgvdG8vZGF0YXNldC5jc3YifSwKICAgICAgICAgICAgcGFyYW1zPXsKICAgICAgICAgICAgICAgICJtb2RlbF9jbGFzcyI6ICJza2xlYXJuLmxpbmVhcl9tb2RlbC5Mb2dpc3RpY1JlZ3Jlc3Npb24iLAogICAgICAgICAgICAgICAgImxhYmVsX2NvbHVtbnMiOiAibGFiZWwiLAogICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyI6ICJpZCIsCiAgICAgICAgICAgICAgICAibW9kZWxfbmFtZSI6ICJteS1tb2RlbCIsCiAgICAgICAgICAgICAgICAidGFnIjogInYxLjAuMCIsCiAgICAgICAgICAgICAgICAic2FtcGxlX3NldCI6ICIuL3BhdGgvdG8vc2FtcGxlX3NldC5jc3YiLAogICAgICAgICAgICAgICAgInRlc3Rfc2V0IjogIi4vcGF0aC90by90ZXN0X3NldC5jc3YiLAogICAgICAgICAgICAgICAgIkNMQVNTX3NvbHZlciI6ICJsaWJsaW5lYXIiLAogICAgICAgICAgICB9LAogICAgICAgICkKCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQKICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3RvcgogICAgOnBhcmFtIG1vZGVsX2NsYXNzOiAgICAgICAgICAgICBUaGUgY2xhc3Mgb2YgdGhlIG1vZGVsLCBlLmcuIGBza2xlYXJuLmxpbmVhcl9tb2RlbC5Mb2dpc3RpY1JlZ3Jlc3Npb25gCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uczogICAgICAgICAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2xhc3NpZmljYXRpb24gdGFza3MuIE1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyIG9yIGEgbGlzdCBvZiBzdHJpbmdzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW5zIHRvIGRyb3AKICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgICAgICAgICAgICAgVGhlIG1vZGVsJ3MgbmFtZSB0byB1c2UgZm9yIHN0b3JpbmcgdGhlIG1vZGVsIGFydGlmYWN0LCBkZWZhdWx0IHRvICdtb2RlbCcKICAgIDpwYXJhbSB0YWc6ICAgICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsJ3MgdGFnIHRvIGxvZyB3aXRoCiAgICA6cGFyYW0gc2FtcGxlX3NldDogICAgICAgICAgICAgIEEgc2FtcGxlIHNldCBvZiBpbnB1dHMgZm9yIHRoZSBtb2RlbCBmb3IgbG9nZ2luZyBpdHMgc3RhdHMgYWxvbmcgdGhlIG1vZGVsIGluIGZhdm91cgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvZiBtb2RlbCBtb25pdG9yaW5nLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3RvcgogICAgOnBhcmFtIHRlc3Rfc2V0OiAgICAgICAgICAgICAgICBUaGUgdGVzdCBzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIHdpdGguCiAgICA6cGFyYW0gdHJhaW5fdGVzdF9zcGxpdF9zaXplOiAgIGlmIHRlc3Rfc2V0IHdhcyBwcm92aWRlZCB0aGVuIHRoaXMgYXJndW1lbnQgaXMgaWdub3JlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgU2hvdWxkIGJlIGJldHdlZW4gMC4wIGFuZCAxLjAgYW5kIHJlcHJlc2VudCB0aGUgcHJvcG9ydGlvbiBvZiB0aGUgZGF0YXNldCB0byBpbmNsdWRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGluIHRoZSB0ZXN0IHNwbGl0LiBUaGUgc2l6ZSBvZiB0aGUgVHJhaW5pbmcgc2V0IGlzIHNldCB0byB0aGUgY29tcGxlbWVudCBvZiB0aGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHZhbHVlLiBEZWZhdWx0ID0gMC4yCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICAgICAgIFJlbGV2YW50IG9ubHkgd2hlbiB1c2luZyB0cmFpbl90ZXN0X3NwbGl0X3NpemUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIEEgcmFuZG9tIHN0YXRlIHNlZWQgdG8gc2h1ZmZsZSB0aGUgZGF0YS4gRm9yIG1vcmUgaW5mb3JtYXRpb24sIHNlZToKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaHR0cHM6Ly9zY2lraXQtbGVhcm4ub3JnL3N0YWJsZS9nbG9zc2FyeS5odG1sI3Rlcm0tcmFuZG9tX3N0YXRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE5vdGljZSB0aGF0IGhlcmUgd2Ugb25seSBwYXNzIGludGVnZXIgdmFsdWVzLgogICAgOnBhcmFtIGxhYmVsczogICAgICAgICAgICAgICAgICBMYWJlbHMgdG8gbG9nIHdpdGggdGhlIG1vZGVsCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHdpdGggcHJlZml4ZXMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoYXQgd2lsbCBiZSBwYXJzZWQgYW5kIHBhc3NlZCB0byB0aGUgcmVsZXZhbnQgZnVuY3Rpb24sIGJ5IHRoZSBmb2xsb3dpbmcgcHJlZml4ZXM6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYENMQVNTX2AgLSBmb3IgdGhlIG1vZGVsIGNsYXNzIGFyZ3VtZW50cwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAtIGBGSVRfYCAtIGZvciB0aGUgYGZpdGAgZnVuY3Rpb24gYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYFRSQUlOX2AgLSBmb3IgdGhlIGB0cmFpbmAgZnVuY3Rpb24gKGluIHhnYiBvciBsZ2JtIHRyYWluIGZ1bmN0aW9uIC0gZnV0dXJlKQoKICAgICIiIgogICAgIyBWYWxpZGF0ZSBpbnB1dHM6CiAgICAjIENoZWNrIGlmIGV4YWN0bHkgb25lIG9mIHRoZW0gaXMgc3VwcGxpZWQ6CiAgICBpZiB0ZXN0X3NldCBpcyBOb25lOgogICAgICAgIGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSBpcyBOb25lOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgInRlc3Rfc2V0IG9yIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSBhcmUgbm90IHByb3ZpZGVkLCBzZXR0aW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB0byAwLjIiCiAgICAgICAgICAgICkKICAgICAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gMC4yCgogICAgZWxpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgInRlc3Rfc2V0IHByb3ZpZGVkLCBpZ25vcmluZyBnaXZlbiB0cmFpbl90ZXN0X3NwbGl0X3NpemUgdmFsdWUiCiAgICAgICAgKQogICAgICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSA9IE5vbmUKCiAgICAjIEdldCBEYXRhRnJhbWUgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIEdldHRpbmcgdGhlIHNhbXBsZSBzZXQ6CiAgICBpZiBzYW1wbGVfc2V0IGlzIE5vbmU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgZiJTYW1wbGUgc2V0IG5vdCBnaXZlbiwgdXNpbmcgdGhlIHdob2xlIHRyYWluaW5nIHNldCBhcyB0aGUgc2FtcGxlIHNldCIKICAgICAgICApCiAgICAgICAgc2FtcGxlX3NldCA9IGRhdGFzZXQKICAgIGVsc2U6CiAgICAgICAgc2FtcGxlX3NldCwgXyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgICAgIGRhdGFzZXQ9c2FtcGxlX3NldCwKICAgICAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgICAgICkKCiAgICAjIFBhcnNpbmcga3dhcmdzOgogICAgIyBUT0RPOiBVc2UgaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24uCiAgICB0cmFpbl9rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeChzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLlRSQUlOKQogICAgZml0X2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuRklUKQogICAgbW9kZWxfY2xhc3Nfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoCiAgICAgICAgc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5NT0RFTF9DTEFTUwogICAgKQoKICAgICMgQ2hlY2sgaWYgbW9kZWwgb3IgZnVuY3Rpb246CiAgICBpZiBoYXNhdHRyKG1vZGVsX2NsYXNzLCAidHJhaW4iKToKICAgICAgICAjIFRPRE86IE5lZWQgdG8gY2FsbDogbW9kZWwoKSwgYWZ0ZXJ3YXJkcyB0byBzdGFydCB0aGUgdHJhaW4gZnVuY3Rpb24uCiAgICAgICAgIyBtb2RlbCA9IGNyZWF0ZV9mdW5jdGlvbihmInttb2RlbF9jbGFzc30udHJhaW4iKQogICAgICAgIHJhaXNlIE5vdEltcGxlbWVudGVkRXJyb3IKICAgIGVsc2U6CiAgICAgICAgIyBDcmVhdGluZyBtb2RlbCBpbnN0YW5jZToKICAgICAgICBtb2RlbCA9IGNyZWF0ZV9jbGFzcyhtb2RlbF9jbGFzcykoKiptb2RlbF9jbGFzc19rd2FyZ3MpCgogICAgeCA9IGRhdGFzZXQuZHJvcChsYWJlbF9jb2x1bW5zLCBheGlzPTEpCiAgICB5ID0gZGF0YXNldFtsYWJlbF9jb2x1bW5zXQogICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplOgogICAgICAgIHhfdHJhaW4sIHhfdGVzdCwgeV90cmFpbiwgeV90ZXN0ID0gdHJhaW5fdGVzdF9zcGxpdCgKICAgICAgICAgICAgeCwgeSwgdGVzdF9zaXplPXRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSwgcmFuZG9tX3N0YXRlPXJhbmRvbV9zdGF0ZQogICAgICAgICkKICAgIGVsc2U6CiAgICAgICAgeF90cmFpbiwgeV90cmFpbiA9IHgsIHkKCiAgICAgICAgdGVzdF9zZXQgPSB0ZXN0X3NldC5hc19kZigpCiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICB0ZXN0X3NldCA9IGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSkKCiAgICAgICAgeF90ZXN0LCB5X3Rlc3QgPSB0ZXN0X3NldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSksIHRlc3Rfc2V0W2xhYmVsX2NvbHVtbnNdCgogICAgQXV0b01MUnVuLmFwcGx5X21scnVuKAogICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgIG1vZGVsX25hbWU9bW9kZWxfbmFtZSwKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgdGFnPXRhZywKICAgICAgICBzYW1wbGVfc2V0PXNhbXBsZV9zZXQsCiAgICAgICAgeV9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgdGVzdF9zZXQ9dGVzdF9zZXQsCiAgICAgICAgeF90ZXN0PXhfdGVzdCwKICAgICAgICB5X3Rlc3Q9eV90ZXN0LAogICAgICAgIGFydGlmYWN0cz1jb250ZXh0LmFydGlmYWN0cywKICAgICAgICBsYWJlbHM9bGFiZWxzLAogICAgKQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInRyYWluaW5nICd7bW9kZWxfbmFtZX0nIikKICAgIG1vZGVsLmZpdCh4X3RyYWluLCB5X3RyYWluLCAqKmZpdF9rd2FyZ3MpCgoKZGVmIGV2YWx1YXRlKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBtb2RlbDogc3RyLAogICAgZGF0YXNldDogbWxydW4uRGF0YUl0ZW0sCiAgICBkcm9wX2NvbHVtbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBFdmFsdWF0aW5nIGEgbW9kZWwuIEFydGlmYWN0cyBnZW5lcmF0ZWQgYnkgdGhlIE1MSGFuZGxlci4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gbW9kZWw6ICAgICAgICAgICAgICAgICAgIFRoZSBtb2RlbCBTdG9yZSBwYXRoLgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byBldmFsdWF0ZSB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcC4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIFBhcnNpbmcgbGFiZWxfY29sdW1uczoKICAgIHBhcnNlZF9sYWJlbF9jb2x1bW5zID0gW10KICAgIGlmIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9ICgKICAgICAgICAgICAgbGFiZWxfY29sdW1ucyBpZiBpc2luc3RhbmNlKGxhYmVsX2NvbHVtbnMsIGxpc3QpIGVsc2UgW2xhYmVsX2NvbHVtbnNdCiAgICAgICAgKQogICAgICAgIGZvciBsYyBpbiBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICBpZiBmcy5jb21tb24uZmVhdHVyZV9zZXBhcmF0b3IgaW4gbGM6CiAgICAgICAgICAgICAgICBmZWF0dXJlX3NldF9uYW1lLCBsYWJlbF9uYW1lLCBhbGlhcyA9IGZzLmNvbW1vbi5wYXJzZV9mZWF0dXJlX3N0cmluZyhsYykKICAgICAgICAgICAgICAgIHBhcnNlZF9sYWJlbF9jb2x1bW5zLmFwcGVuZChhbGlhcyBvciBsYWJlbF9uYW1lKQogICAgICAgIGlmIHBhcnNlZF9sYWJlbF9jb2x1bW5zOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gcGFyc2VkX2xhYmVsX2NvbHVtbnMKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCgogICAgIyBMb2FkaW5nIHRoZSBtb2RlbCBhbmQgcHJlZGljdGluZzoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbCgKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQsIG1vZGVsX25hbWU9Im1vZGVsX0xpbmVhclJlZ3Jlc3Npb24iCiAgICApCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4obW9kZWxfaGFuZGxlci5tb2RlbCwgeV90ZXN0PXksIG1vZGVsX3BhdGg9bW9kZWwpCgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmImV2YWx1YXRpbmcgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoeCwgKiprd2FyZ3MpCgoKZGVmIHByZWRpY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIHJlc3VsdF9zZXQ6IE9wdGlvbmFsW3N0cl0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIFByZWRpY3RpbmcgZGF0YXNldCBieSBhIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIHByZWRpY3QgdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJLCBhIEZlYXR1cmVWZWN0b3Igb3IgYQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgaW4gYSBzaGFwZSBvZiBhIGxpc3QvZGljdC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV2hlbiBwYXNzaW5nIGEgc2FtcGxlLCBwYXNzIHRoZSBkYXRhc2V0IGFzIGEgZmllbGQgaW4gYHBhcmFtc2AgaW5zdGVhZCBvZiBgaW5wdXRzYC4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHRoZSBkYXRhc2V0IGlzIGEgbGlzdC9kaWN0IHRoaXMgcGFyYW1ldGVyIHNob3VsZCBiZSByZXByZXNlbnRlZCBieSBpbnRlZ2Vycy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIHJlc3VsdF9zZXQ6ICAgICAgICAgICAgICBUaGUgZGIga2V5IHRvIHNldCBuYW1lIG9mIHRoZSBwcmVkaWN0aW9uIHJlc3VsdCBhbmQgdGhlIGZpbGVuYW1lLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0IHRvICdwcmVkaWN0aW9uJy4KICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgdG8gdGhlIHByZWRpY3QgZnVuY3Rpb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKFBSRURJQ1RfIHByZWZpeCBpcyBub3QgcmVxdWlyZWQpLgogICAgIiIiCiAgICAjIEdldCBkYXRhc2V0IGJ5IFVSTCBvciBieSBGZWF0dXJlVmVjdG9yOgogICAgZGF0YXNldCwgbGFiZWxfY29sdW1ucyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICBkYXRhc2V0PWRhdGFzZXQsCiAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICApCgogICAgIyBsb2FkaW5nIHRoZSBtb2RlbCwgYW5kIGdldHRpbmcgdGhlIG1vZGVsIGhhbmRsZXI6CiAgICBtb2RlbF9oYW5kbGVyID0gQXV0b01MUnVuLmxvYWRfbW9kZWwobW9kZWxfcGF0aD1tb2RlbCwgY29udGV4dD1jb250ZXh0KQoKICAgICMgRHJvcHBpbmcgbGFiZWwgY29sdW1ucyBpZiBuZWNlc3Nhcnk6CiAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW10KICAgIGVsaWYgaXNpbnN0YW5jZShsYWJlbF9jb2x1bW5zLCBzdHIpOgogICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbbGFiZWxfY29sdW1uc10KCiAgICAjIFByZWRpY3Rpbmc6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYibWFraW5nIHByZWRpY3Rpb24gYnkgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICB5X3ByZWQgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoZGF0YXNldCwgKiprd2FyZ3MpCgogICAgIyBQcmVwYXJpbmcgYW5kIHZhbGlkYXRpbmcgbGFiZWwgY29sdW1ucyBmb3IgdGhlIGRhdGFmcmFtZSBvZiB0aGUgcHJlZGljdGlvbiByZXN1bHQ6CiAgICBudW1fcHJlZGljdGVkID0gMSBpZiBsZW4oeV9wcmVkLnNoYXBlKSA9PSAxIGVsc2UgeV9wcmVkLnNoYXBlWzFdCgogICAgaWYgbnVtX3ByZWRpY3RlZCA+IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBpZiBudW1fcHJlZGljdGVkID09IDE6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbInByZWRpY3RlZCBsYWJlbHMiXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMuZXh0ZW5kKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIGYicHJlZGljdGVkX2xhYmVsX3tpICsgMSArIGxlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgICAgICAgICAgICAgZm9yIGkgaW4gcmFuZ2UobnVtX3ByZWRpY3RlZCAtIGxlbihsYWJlbF9jb2x1bW5zKSkKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgZWxpZiBudW1fcHJlZGljdGVkIDwgbGVuKGxhYmVsX2NvbHVtbnMpOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIm51bWJlciBvZiBwcmVkaWN0ZWQgbGFiZWxzOiB7bnVtX3ByZWRpY3RlZH0gaXMgc21hbGxlciB0aGFuIG51bWJlciBvZiBsYWJlbCBjb2x1bW5zOiB7bGVuKGxhYmVsX2NvbHVtbnMpfSIKICAgICAgICApCiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcgoKICAgIGFydGlmYWN0X25hbWUgPSByZXN1bHRfc2V0IG9yICJwcmVkaWN0aW9uIgogICAgbGFiZWxzX2luc2lkZV9kZiA9IHNldChsYWJlbF9jb2x1bW5zKSAmIHNldChkYXRhc2V0LmNvbHVtbnMudG9saXN0KCkpCiAgICBpZiBsYWJlbHNfaW5zaWRlX2RmOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIlRoZSBsYWJlbHM6IHtsYWJlbHNfaW5zaWRlX2RmfSBhcmUgYWxyZWFkeSBleGlzdGVkIGluIHRoZSBkYXRhZnJhbWUiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgIHByZWRfZGYgPSBwZC5jb25jYXQoW2RhdGFzZXQsIHBkLkRhdGFGcmFtZSh5X3ByZWQsIGNvbHVtbnM9bGFiZWxfY29sdW1ucyldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KGFydGlmYWN0X25hbWUsIHByZWRfZGYsIGRiX2tleT1yZXN1bHRfc2V0KQo= code_origin: '' + origin_filename: '' + image: mlrun/mlrun + default_handler: train description: Automatic train, evaluate and predict functions for the ML frameworks - Scikit-Learn, XGBoost and LightGBM. + filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/auto_trainer/auto_trainer.py + command: '' disable_auto_mount: false - default_handler: train entry_points: train: lineno: 121 + has_varargs: false + name: train + doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ + \ project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ + hub://auto_trainer\", \"train\")\n trainer_run = project.run(\n \ + \ name=\"train\",\n handler=\"train\",\n inputs={\"dataset\"\ + : \"./path/to/dataset.csv\"},\n params={\n \"model_class\"\ + : \"sklearn.linear_model.LogisticRegression\",\n \"label_columns\"\ + : \"label\",\n \"drop_columns\": \"id\",\n \"model_name\"\ + : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ + : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ + ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" parameters: - name: context type: MLClientCtx @@ -70,21 +79,12 @@ spec: type: dict doc: Labels to log with the model default: null - has_varargs: false - name: train has_kwargs: true - doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ - \ project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ - hub://auto_trainer\", \"train\")\n trainer_run = project.run(\n \ - \ name=\"train\",\n handler=\"train\",\n inputs={\"dataset\"\ - : \"./path/to/dataset.csv\"},\n params={\n \"model_class\"\ - : \"sklearn.linear_model.LogisticRegression\",\n \"label_columns\"\ - : \"label\",\n \"drop_columns\": \"id\",\n \"model_name\"\ - : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ - : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ - ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" evaluate: lineno: 273 + has_varargs: false + name: evaluate + doc: Evaluating a model. Artifacts generated by the MLHandler. parameters: - name: context type: MLClientCtx @@ -104,12 +104,12 @@ spec: doc: The target label(s) of the column(s) in the dataset. for Regression or Classification tasks. Mandatory when dataset is not a FeatureVector. default: null - has_varargs: false - name: evaluate has_kwargs: true - doc: Evaluating a model. Artifacts generated by the MLHandler. predict: lineno: 327 + has_varargs: false + name: predict + doc: Predicting dataset by a model. parameters: - name: context type: MLClientCtx @@ -138,10 +138,11 @@ spec: doc: The db key to set name of the prediction result and the filename. Default to 'prediction'. default: null - has_varargs: false - name: predict has_kwargs: true - doc: Predicting dataset by a model. - command: '' -kind: job verbose: false +metadata: + name: auto-trainer + categories: + - machine-learning + - model-training + tag: '' diff --git a/functions/src/auto_trainer/item.yaml b/functions/src/auto_trainer/item.yaml index ba33f6a08..d397a79d6 100755 --- a/functions/src/auto_trainer/item.yaml +++ b/functions/src/auto_trainer/item.yaml @@ -13,7 +13,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.0 +mlrunVersion: 1.10.0 name: auto_trainer platformVersion: 3.5.0 spec: @@ -23,4 +23,4 @@ spec: kind: job requirements: [] url: '' -version: 1.8.0 +version: 1.9.0 diff --git a/functions/src/auto_trainer/requirements.txt b/functions/src/auto_trainer/requirements.txt index b14a0293c..4854d84fd 100644 --- a/functions/src/auto_trainer/requirements.txt +++ b/functions/src/auto_trainer/requirements.txt @@ -1,4 +1,4 @@ pandas -scikit-learn<1.4.0 +scikit-learn~=1.5 xgboost<2.0.0 plotly diff --git a/functions/src/auto_trainer/test_auto_trainer.py b/functions/src/auto_trainer/test_auto_trainer.py index 9a1ff554c..4a517f112 100644 --- a/functions/src/auto_trainer/test_auto_trainer.py +++ b/functions/src/auto_trainer/test_auto_trainer.py @@ -25,6 +25,37 @@ make_regression, ) +# Monkey-patch sklearn metrics to fix MLRun compatibility with sklearn 1.5+ +# MLRun 1.10.0 calls metrics with the deprecated 'squared' parameter +import sklearn.metrics +from sklearn.metrics import ( + mean_squared_error as _original_mse, + mean_absolute_error as _original_mae, + median_absolute_error as _original_medae, +) + + +def _patched_mean_squared_error(y_true, y_pred, sample_weight=None, multioutput='uniform_average', squared=None): + """Wrapper for mean_squared_error that ignores the deprecated 'squared' parameter.""" + # In sklearn 1.4+, 'squared' parameter was removed. Always return MSE (not RMSE) + return _original_mse(y_true, y_pred, sample_weight=sample_weight, multioutput=multioutput) + + +def _patched_mean_absolute_error(y_true, y_pred, sample_weight=None, multioutput='uniform_average', squared=None): + """Wrapper for mean_absolute_error that ignores any 'squared' parameter.""" + return _original_mae(y_true, y_pred, sample_weight=sample_weight, multioutput=multioutput) + + +def _patched_median_absolute_error(y_true, y_pred, multioutput='uniform_average', sample_weight=None, squared=None): + """Wrapper for median_absolute_error that ignores any 'squared' parameter.""" + return _original_medae(y_true, y_pred, multioutput=multioutput, sample_weight=sample_weight) + + +# Apply the patches +sklearn.metrics.mean_squared_error = _patched_mean_squared_error +sklearn.metrics.mean_absolute_error = _patched_mean_absolute_error +sklearn.metrics.median_absolute_error = _patched_median_absolute_error + MODELS = [ ("sklearn.linear_model.LinearRegression", "regression"), ("sklearn.ensemble.RandomForestClassifier", "classification"), @@ -82,7 +113,7 @@ def test_train(model: Tuple[str, str]): dataset, label_columns = _get_dataset(model[1]) is_test_passed = True - project = mlrun.new_project("auto-trainer-test", context="./") + project = mlrun.get_or_create_project("auto-trainer-test", context="./") fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun") train_run = None @@ -119,7 +150,7 @@ def test_train_evaluate(model: Tuple[str, str]): dataset, label_columns = _get_dataset(model[1]) is_test_passed = True # Importing function: - project = mlrun.new_project("auto-trainer-test", context="./") + project = mlrun.get_or_create_project("auto-trainer-test", context="./") fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun") temp_dir = tempfile.mkdtemp() @@ -172,7 +203,7 @@ def test_train_predict(model: Tuple[str, str]): df = pd.read_csv(dataset) sample = df.head().drop("labels", axis=1).values.tolist() # Importing function: - project = mlrun.new_project("auto-trainer-test", context="./") + project = mlrun.get_or_create_project("auto-trainer-test", context="./") fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun") temp_dir = tempfile.mkdtemp() diff --git a/functions/src/describe/function.yaml b/functions/src/describe/function.yaml index a11461774..7116fae92 100644 --- a/functions/src/describe/function.yaml +++ b/functions/src/describe/function.yaml @@ -1,9 +1,44 @@ +metadata: + tag: '' + categories: + - data-analysis + name: describe +verbose: false +kind: job spec: + command: '' + image: mlrun/mlrun + description: describe and visualizes dataset stats + disable_auto_mount: false + default_handler: analyze entry_points: analyze: + doc: 'The function will output the following artifacts per + + column within the data frame (based on data types) + + If the data has more than 500,000 sample we + + sample randomly 500,000 samples: + + + describe csv + + histograms + + scatter-2d + + violin chart + + correlation-matrix chart + + correlation-matrix csv + + imbalance pie chart + + imbalance-weights-vec csv' + has_kwargs: false has_varargs: false - outputs: - - type: None parameters: - name: context type: MLClientCtx @@ -45,46 +80,12 @@ spec: - name: dask_client doc: Dask client object default: null - doc: 'The function will output the following artifacts per - - column within the data frame (based on data types) - - If the data has more than 500,000 sample we - - sample randomly 500,000 samples: - - - describe csv - - histograms - - scatter-2d - - violin chart - - correlation-matrix chart - - correlation-matrix csv - - imbalance pie chart - - imbalance-weights-vec csv' - has_kwargs: false + outputs: + - type: None name: analyze lineno: 46 - image: mlrun/mlrun - command: '' + filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/describe/describe.py build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCmZyb20gdHlwaW5nIGltcG9ydCBVbmlvbgoKaW1wb3J0IG1scnVuCmltcG9ydCBudW1weSBhcyBucAoKd2FybmluZ3Muc2ltcGxlZmlsdGVyKGFjdGlvbj0iaWdub3JlIiwgY2F0ZWdvcnk9RnV0dXJlV2FybmluZykKCmltcG9ydCBtbHJ1bi5mZWF0dXJlX3N0b3JlIGFzIGZzdG9yZQppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCBwbG90bHkuZXhwcmVzcyBhcyBweAppbXBvcnQgcGxvdGx5LmZpZ3VyZV9mYWN0b3J5IGFzIGZmCmltcG9ydCBwbG90bHkuZ3JhcGhfb2JqZWN0cyBhcyBnbwpmcm9tIG1scnVuLmFydGlmYWN0cyBpbXBvcnQgKAogICAgQXJ0aWZhY3QsCiAgICBEYXRhc2V0QXJ0aWZhY3QsCiAgICBQbG90bHlBcnRpZmFjdCwKICAgIFRhYmxlQXJ0aWZhY3QsCiAgICB1cGRhdGVfZGF0YXNldF9tZXRhLAopCmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKZnJvbSBtbHJ1bi5mZWF0dXJlX3N0b3JlIGltcG9ydCBGZWF0dXJlU2V0CmZyb20gcGxvdGx5LnN1YnBsb3RzIGltcG9ydCBtYWtlX3N1YnBsb3RzCgpwZC5zZXRfb3B0aW9uKCJkaXNwbGF5LmZsb2F0X2Zvcm1hdCIsIGxhbWJkYSB4OiAiJS4yZiIgJSB4KQpNQVhfU0laRV9PRl9ERiA9IDUwMDAwMAoKCmRlZiBhbmFseXplKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBuYW1lOiBzdHIgPSAiZGF0YXNldCIsCiAgICB0YWJsZTogVW5pb25bRmVhdHVyZVNldCwgRGF0YUl0ZW1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gTm9uZSwKICAgIHBsb3RzX2Rlc3Q6IHN0ciA9ICJwbG90cyIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDEsCiAgICBwcm9ibGVtX3R5cGU6IHN0ciA9ICJjbGFzc2lmaWNhdGlvbiIsCiAgICBkYXNrX2tleTogc3RyID0gImRhc2tfa2V5IiwKICAgIGRhc2tfZnVuY3Rpb246IHN0ciA9IE5vbmUsCiAgICBkYXNrX2NsaWVudD1Ob25lLAopIC0+IE5vbmU6CiAgICAiIiIKICAgIFRoZSBmdW5jdGlvbiB3aWxsIG91dHB1dCB0aGUgZm9sbG93aW5nIGFydGlmYWN0cyBwZXIKICAgIGNvbHVtbiB3aXRoaW4gdGhlIGRhdGEgZnJhbWUgKGJhc2VkIG9uIGRhdGEgdHlwZXMpCiAgICBJZiB0aGUgZGF0YSBoYXMgbW9yZSB0aGFuIDUwMCwwMDAgc2FtcGxlIHdlCiAgICBzYW1wbGUgcmFuZG9tbHkgNTAwLDAwMCBzYW1wbGVzOgoKICAgIGRlc2NyaWJlIGNzdgogICAgaGlzdG9ncmFtcwogICAgc2NhdHRlci0yZAogICAgdmlvbGluIGNoYXJ0CiAgICBjb3JyZWxhdGlvbi1tYXRyaXggY2hhcnQKICAgIGNvcnJlbGF0aW9uLW1hdHJpeCBjc3YKICAgIGltYmFsYW5jZSBwaWUgY2hhcnQKICAgIGltYmFsYW5jZS13ZWlnaHRzLXZlYyBjc3YKCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIFRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbmFtZTogICAgICAgICAgICAgICAgICAgIEtleSBvZiBkYXRhc2V0IHRvIGRhdGFiYXNlICgiZGF0YXNldCIgZm9yIGRlZmF1bHQpCiAgICA6cGFyYW0gdGFibGU6ICAgICAgICAgICAgICAgICAgIE1MUnVuIGlucHV0IHBvaW50aW5nIHRvIHBhbmRhcyBkYXRhZnJhbWUgKGNzdi9wYXJxdWV0IGZpbGUgcGF0aCkgb3IgRmVhdHVyZVNldAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhcyBwYXJhbQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICAgICAgICBHcm91bmQgdHJ1dGggY29sdW1uIGxhYmVsCiAgICA6cGFyYW0gcGxvdHNfZGVzdDogICAgICAgICAgICAgIERlc3RpbmF0aW9uIGZvbGRlciBvZiBzdW1tYXJ5IHBsb3RzIChyZWxhdGl2ZSB0byBhcnRpZmFjdF9wYXRoKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoInBsb3RzIiBmb3IgZGVmYXVsdCkKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgV2hlbiB0aGUgdGFibGUgaGFzIG1vcmUgdGhhbiA1MDAsMDAwIHNhbXBsZXMsIHdlIHNhbXBsZSByYW5kb21seSA1MDAsMDAwIHNhbXBsZXMKICAgIDpwYXJhbSBwcm9ibGVtX3R5cGUgICAgICAgICAgICAgVGhlIHR5cGUgb2YgdGhlIE1MIHByb2JsZW0gdGhlIGRhdGEgZmFjaW5nIC0gcmVncmVzc2lvbiwgY2xhc3NpZmljYXRpb24gb3IgTm9uZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoY2xhc3NpZmljYXRpb24gZm9yIGRlZmF1bHQpCiAgICA6cGFyYW0gZGFza19rZXk6ICAgICAgICAgICAgICAgIEtleSBvZiBkYXRhZnJhbWUgaW4gZGFzayBjbGllbnQgImRhdGFzZXRzIiBhdHRyaWJ1dGUKICAgIDpwYXJhbSBkYXNrX2Z1bmN0aW9uOiAgICAgICAgICAgRGFzayBmdW5jdGlvbiB1cmwgKGRiOi8vLi4pCiAgICA6cGFyYW0gZGFza19jbGllbnQ6ICAgICAgICAgICAgIERhc2sgY2xpZW50IG9iamVjdAogICAgIiIiCiAgICBkYXRhX2l0ZW0sIGZlYXR1cmVzZXQsIGNyZWF0LCB1cGRhdGUgPSBGYWxzZSwgRmFsc2UsIEZhbHNlLCBGYWxzZQogICAgZ2V0X2Zyb21fdGFibGUgPSBUcnVlCiAgICBpZiBkYXNrX2Z1bmN0aW9uIG9yIGRhc2tfY2xpZW50OgogICAgICAgIGRhdGFfaXRlbSwgY3JlYXQgPSBUcnVlLCBUcnVlCiAgICAgICAgaWYgZGFza19mdW5jdGlvbjoKICAgICAgICAgICAgY2xpZW50ID0gbWxydW4uaW1wb3J0X2Z1bmN0aW9uKGRhc2tfZnVuY3Rpb24pLmNsaWVudAogICAgICAgIGVsaWYgZGFza19jbGllbnQ6CiAgICAgICAgICAgIGNsaWVudCA9IGRhc2tfY2xpZW50CiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigiZGFzayBjbGllbnQgd2FzIG5vdCBwcm92aWRlZCIpCgogICAgICAgIGlmIGRhc2tfa2V5IGluIGNsaWVudC5kYXRhc2V0czoKICAgICAgICAgICAgZGYgPSBjbGllbnQuZ2V0X2RhdGFzZXQoZGFza19rZXkpCiAgICAgICAgICAgIGRhdGFfaXRlbSwgY3JlYXQsIGdldF9mcm9tX3RhYmxlID0gVHJ1ZSwgVHJ1ZSwgRmFsc2UKICAgICAgICBlbGlmIHRhYmxlOgogICAgICAgICAgICBnZXRfZnJvbV90YWJsZSA9IFRydWUKICAgICAgICBlbHNlOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgZiJvbmx5IHRoZXNlIGRhdGFzZXRzIGFyZSBhdmFpbGFibGUge2NsaWVudC5kYXRhc2V0c30gaW4gY2xpZW50IHtjbGllbnR9IgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIEV4Y2VwdGlvbigiZGF0YXNldCBub3QgZm91bmQgb24gZGFzayBjbHVzdGVyIikKCiAgICBpZiBnZXRfZnJvbV90YWJsZToKICAgICAgICBpZiB0eXBlKHRhYmxlKSA9PSBEYXRhSXRlbToKICAgICAgICAgICAgaWYgdGFibGUubWV0YSBpcyBOb25lOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgVHJ1ZSwgRmFsc2UKICAgICAgICAgICAgZWxpZiB0YWJsZS5tZXRhLmtpbmQgPT0gImRhdGFzZXQiOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgRmFsc2UsIFRydWUKICAgICAgICAgICAgZWxpZiB0YWJsZS5tZXRhLmtpbmQgPT0gIkZlYXR1cmVWZWN0b3IiOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgRmFsc2UsIEZhbHNlCiAgICAgICAgICAgIGVsaWYgdGFibGUubWV0YS5raW5kID09ICJGZWF0dXJlU2V0IjoKICAgICAgICAgICAgICAgIGZlYXR1cmVzZXQsIGNyZWF0LCB1cGRhdGUgPSBUcnVlLCBGYWxzZSwgRmFsc2UKCiAgICAgICAgaWYgZGF0YV9pdGVtOgogICAgICAgICAgICBkZiA9IHRhYmxlLmFzX2RmKCkKICAgICAgICBlbGlmIGZlYXR1cmVzZXQ6CiAgICAgICAgICAgIHByb2plY3RfbmFtZSwgc2V0X25hbWUgPSAoCiAgICAgICAgICAgICAgICB0YWJsZS5fcGF0aC5zcGxpdCgiLyIpWzJdLAogICAgICAgICAgICAgICAgdGFibGUuX3BhdGguc3BsaXQoIi8iKVs0XSwKICAgICAgICAgICAgKQogICAgICAgICAgICBmZWF0dXJlX3NldCA9IGZzdG9yZS5nZXRfZmVhdHVyZV9zZXQoCiAgICAgICAgICAgICAgICBmInN0b3JlOi8vZmVhdHVyZS1zZXRzL3twcm9qZWN0X25hbWV9L3tzZXRfbmFtZX0iCiAgICAgICAgICAgICkKICAgICAgICAgICAgZGYgPSBmZWF0dXJlX3NldC50b19kYXRhZnJhbWUoKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKGYiV3JvbmcgdGFibGUgdHlwZS4iKQogICAgICAgICAgICByZXR1cm4KCiAgICBpZiBkZi5zaXplID4gTUFYX1NJWkVfT0ZfREY6CiAgICAgICAgZGYgPSBkZi5zYW1wbGUobj1pbnQoTUFYX1NJWkVfT0ZfREYgLyBkZi5zaGFwZVsxXSksIHJhbmRvbV9zdGF0ZT1yYW5kb21fc3RhdGUpCiAgICBleHRyYV9kYXRhID0ge30KCiAgICBpZiBsYWJlbF9jb2x1bW4gbm90IGluIGRmLmNvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1uID0gTm9uZQoKICAgIGV4dHJhX2RhdGFbImRlc2NyaWJlIGNzdiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgVGFibGVBcnRpZmFjdCgiZGVzY3JpYmUtY3N2IiwgZGY9ZGYuZGVzY3JpYmUoKSksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9kZXNjcmliZS5jc3YiLAogICAgKQoKICAgIHRyeToKICAgICAgICBfY3JlYXRlX2hpc3RvZ3JhbV9tYXRfYXJ0aWZhY3QoCiAgICAgICAgICAgIGNvbnRleHQsIGRmLCBleHRyYV9kYXRhLCBsYWJlbF9jb2x1bW4sIHBsb3RzX2Rlc3QKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgaGlzdG9ncmFtIG1hdHJpeCBhcnRpZmFjdCBkdWUgdG86IHtlfSIpCiAgICB0cnk6CiAgICAgICAgX2NyZWF0ZV9mZWF0dXJlc19oaXN0b2dyYW1fYXJ0aWZhY3RzKAogICAgICAgICAgICBjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0LCBwcm9ibGVtX3R5cGUKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgcGFpcnBsb3QgaGlzdG9ncmFtcyBkdWUgdG86IHtlfSIpCiAgICB0cnk6CiAgICAgICAgX2NyZWF0ZV9mZWF0dXJlc18yZF9zY2F0dGVyX2FydGlmYWN0cygKICAgICAgICAgICAgY29udGV4dCwgZGYsIGV4dHJhX2RhdGEsIGxhYmVsX2NvbHVtbiwgcGxvdHNfZGVzdCwgcHJvYmxlbV90eXBlCiAgICAgICAgKQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIHBhaXJwbG90IDJkX3NjYXR0ZXIgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfdmlvbGluX2FydGlmYWN0KGNvbnRleHQsIGRmLCBleHRyYV9kYXRhLCBwbG90c19kZXN0KQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIHZpb2xpbiBkaXN0cmlidXRpb24gcGxvdHMgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfaW1iYWxhbmNlX2FydGlmYWN0KAogICAgICAgICAgICBjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0LCBwcm9ibGVtX3R5cGUKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgY2xhc3MgaW1iYWxhbmNlIHBsb3QgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfY29ycl9hcnRpZmFjdChjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0KQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIGZlYXR1cmVzIGNvcnJlbGF0aW9uIHBsb3QgZHVlIHRvOiB7ZX0iKQoKICAgIGlmIG5vdCBkYXRhX2l0ZW06CiAgICAgICAgcmV0dXJuCgogICAgYXJ0aWZhY3QgPSB0YWJsZS5hcnRpZmFjdF91cmwKICAgIGlmIGNyZWF0OiAgIyBkYXRhc2V0IG5vdCBzdG9yZWQKICAgICAgICBhcnRpZmFjdCA9IERhdGFzZXRBcnRpZmFjdCgKICAgICAgICAgICAga2V5PSJkYXRhc2V0Iiwgc3RhdHM9VHJ1ZSwgZGY9ZGYsIGV4dHJhX2RhdGE9ZXh0cmFfZGF0YQogICAgICAgICkKICAgICAgICBhcnRpZmFjdCA9IGNvbnRleHQubG9nX2FydGlmYWN0KGFydGlmYWN0LCBkYl9rZXk9bmFtZSkKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVGhlIGRhdGEgc2V0IGlzIGxvZ2dlZCB0byB0aGUgcHJvamVjdCB1bmRlciB7bmFtZX0gbmFtZSIpCgogICAgaWYgdXBkYXRlOgogICAgICAgIHVwZGF0ZV9kYXRhc2V0X21ldGEoYXJ0aWZhY3QsIGV4dHJhX2RhdGE9ZXh0cmFfZGF0YSkKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVGhlIGRhdGEgc2V0IG5hbWVkIHtuYW1lfSBpcyB1cGRhdGVkIikKCiAgICAjIFRPRE8gOiAzLUQgcGxvdCBvbiBvbiBzZWxlY3RlZCBmZWF0dXJlcy4KICAgICMgVE9ETyA6IFJlaW50ZWdyYXRpb24gcGxvdCBvbiBvbiBzZWxlY3RlZCBmZWF0dXJlcy4KICAgICMgVE9ETyA6IFBDQSBwbG90ICh3aXRoIG9wdGlvbnMpCgoKZGVmIF9jcmVhdGVfaGlzdG9ncmFtX21hdF9hcnRpZmFjdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKKToKICAgICIiIgogICAgQ3JlYXRlIGFuZCBsb2cgYSBoaXN0b2dyYW0gbWF0cml4IGFydGlmYWN0CiAgICAiIiIKICAgIGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgIGl0ZW09QXJ0aWZhY3QoCiAgICAgICAgICAgIGtleT0iaGlzdCIsCiAgICAgICAgICAgIGJvZHk9YiI8Yj4gRGVwcmVjYXRlZCwgc2VlIHRoZSBhcnRpZmFjdHMgc2NhdHRlci0yZCAiCiAgICAgICAgICAgIGIiYW5kIGhpc3RvZ3JhbXMgaW5zdGVhZDxiPiIsCiAgICAgICAgKSwKICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2hpc3QuaHRtbCIsCiAgICApCgoKZGVmIF9jcmVhdGVfZmVhdHVyZXNfaGlzdG9ncmFtX2FydGlmYWN0cygKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhIGhpc3RvZ3JhbSBhcnRpZmFjdCBmb3IgZWFjaCBmZWF0dXJlCiAgICAiIiIKCiAgICBmaWdzID0gZGljdCgpCiAgICBmaXJzdF9mZWF0dXJlX25hbWUgPSAiIgogICAgaWYgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lIGFuZCBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICBhbGxfbGFiZWxzID0gZGZbbGFiZWxfY29sdW1uXS51bmlxdWUoKQogICAgdmlzaWJsZSA9IFRydWUKICAgIGZvciBjb2x1bW5fbmFtZSBpbiBkZi5jb2x1bW5zOgogICAgICAgIGlmIGNvbHVtbl9uYW1lID09IGxhYmVsX2NvbHVtbjoKICAgICAgICAgICAgY29udGludWUKCiAgICAgICAgaWYgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lIGFuZCBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICAgICAgZm9yIGxhYmVsIGluIGFsbF9sYWJlbHM6CiAgICAgICAgICAgICAgICBzdWJfZmlnID0gZ28uSGlzdG9ncmFtKAogICAgICAgICAgICAgICAgICAgIGhpc3RmdW5jPSJjb3VudCIsCiAgICAgICAgICAgICAgICAgICAgeD1kZi5sb2NbZGZbbGFiZWxfY29sdW1uXSA9PSBsYWJlbF1bY29sdW1uX25hbWVdLAogICAgICAgICAgICAgICAgICAgIG5hbWU9c3RyKGxhYmVsKSwKICAgICAgICAgICAgICAgICAgICB2aXNpYmxlPXZpc2libGUsCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBmaWdzW2Yie2NvbHVtbl9uYW1lfUA/QHtsYWJlbH0iXSA9IHN1Yl9maWcKICAgICAgICBlbHNlOgogICAgICAgICAgICBzdWJfZmlnID0gZ28uSGlzdG9ncmFtKGhpc3RmdW5jPSJjb3VudCIsIHg9ZGZbY29sdW1uX25hbWVdLCB2aXNpYmxlPXZpc2libGUpCiAgICAgICAgICAgIGZpZ3NbZiJ7Y29sdW1uX25hbWV9QD9AezF9Il0gPSBzdWJfZmlnCiAgICAgICAgaWYgdmlzaWJsZToKICAgICAgICAgICAgZmlyc3RfZmVhdHVyZV9uYW1lID0gY29sdW1uX25hbWUKICAgICAgICB2aXNpYmxlID0gRmFsc2UKCiAgICBmaWcgPSBnby5GaWd1cmUoKQogICAgZm9yIGsgaW4gZmlncy5rZXlzKCk6CiAgICAgICAgZmlnLmFkZF90cmFjZShmaWdzW2tdKQoKICAgIGZpZy51cGRhdGVfbGF5b3V0KAogICAgICAgIHVwZGF0ZW1lbnVzPVsKICAgICAgICAgICAgewogICAgICAgICAgICAgICAgImJ1dHRvbnMiOiBbCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAibGFiZWwiOiBjb2x1bW5fbmFtZSwKICAgICAgICAgICAgICAgICAgICAgICAgIm1ldGhvZCI6ICJ1cGRhdGUiLAogICAgICAgICAgICAgICAgICAgICAgICAiYXJncyI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidmlzaWJsZSI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAga2V5LnNwbGl0KCJAP0AiKVswXSA9PSBjb2x1bW5fbmFtZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb3Iga2V5IGluIGZpZ3Mua2V5cygpCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAieGF4aXMiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJyYW5nZSI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1pbihkZltjb2x1bW5fbmFtZV0pLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWF4KGRmW2NvbHVtbl9uYW1lXSksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHsidGl0bGUiOiBmIjxpPjxiPkhpc3RvZ3JhbSBvZiB7Y29sdW1uX25hbWV9PC9iPjwvaT4ifSwKICAgICAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICAgICAgZm9yIGNvbHVtbl9uYW1lIGluIGRmLmNvbHVtbnMKICAgICAgICAgICAgICAgICAgICBpZiBjb2x1bW5fbmFtZSAhPSBsYWJlbF9jb2x1bW4KICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICAiZGlyZWN0aW9uIjogImRvd24iLAogICAgICAgICAgICAgICAgInBhZCI6IHsiciI6IDEwLCAidCI6IDEwfSwKICAgICAgICAgICAgICAgICJzaG93YWN0aXZlIjogVHJ1ZSwKICAgICAgICAgICAgICAgICJ4IjogMC4yNSwKICAgICAgICAgICAgICAgICJ4YW5jaG9yIjogImxlZnQiLAogICAgICAgICAgICAgICAgInkiOiAxLjEsCiAgICAgICAgICAgICAgICAieWFuY2hvciI6ICJ0b3AiLAogICAgICAgICAgICB9CiAgICAgICAgXSwKICAgICAgICBhbm5vdGF0aW9ucz1bCiAgICAgICAgICAgIGRpY3QoCiAgICAgICAgICAgICAgICB0ZXh0PSJTZWxlY3QgRmVhdHVyZSBOYW1lICIsCiAgICAgICAgICAgICAgICBzaG93YXJyb3c9RmFsc2UsCiAgICAgICAgICAgICAgICB4PTAsCiAgICAgICAgICAgICAgICB5PTEuMDUsCiAgICAgICAgICAgICAgICB5cmVmPSJwYXBlciIsCiAgICAgICAgICAgICAgICB4cmVmPSJwYXBlciIsCiAgICAgICAgICAgICAgICBhbGlnbj0ibGVmdCIsCiAgICAgICAgICAgICAgICB4YW5jaG9yPSJsZWZ0IiwKICAgICAgICAgICAgICAgIHlhbmNob3I9InRvcCIsCiAgICAgICAgICAgICAgICBmb250PXsKICAgICAgICAgICAgICAgICAgICAiY29sb3IiOiAiYmx1ZSIsCiAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICApCiAgICAgICAgXSwKICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dCgKICAgICAgICB3aWR0aD02MDAsCiAgICAgICAgaGVpZ2h0PTQwMCwKICAgICAgICBhdXRvc2l6ZT1GYWxzZSwKICAgICAgICBtYXJnaW49ZGljdCh0PTEwMCwgYj0wLCBsPTAsIHI9MCksCiAgICAgICAgdGVtcGxhdGU9InBsb3RseV93aGl0ZSIsCiAgICApCgogICAgZmlnLnVwZGF0ZV9sYXlvdXQodGl0bGVfdGV4dD1mIjxpPjxiPkhpc3RvZ3JhbXMgb2Yge2ZpcnN0X2ZlYXR1cmVfbmFtZX08L2I+PC9pPiIpCiAgICBleHRyYV9kYXRhW2YiaGlzdG9ncmFtcyJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PWYiaGlzdG9ncmFtcyIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vaGlzdG9ncmFtcy5odG1sIiwKICAgICkKCgpkZWYgX2NyZWF0ZV9mZWF0dXJlc18yZF9zY2F0dGVyX2FydGlmYWN0cygKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhIHNjYXR0ZXItMmQgYXJ0aWZhY3QgZm9yIGVhY2ggY291cGxlIG9mIGZlYXR1cmVzCiAgICAiIiIKICAgIGZlYXR1cmVzID0gWwogICAgICAgIGNvbHVtbl9uYW1lIGZvciBjb2x1bW5fbmFtZSBpbiBkZi5jb2x1bW5zIGlmIGNvbHVtbl9uYW1lICE9IGxhYmVsX2NvbHVtbgogICAgXQogICAgbWF4X2ZlYXR1cmVfbGVuID0gZmxvYXQobWF4KGxlbihlbGVtKSBmb3IgZWxlbSBpbiBmZWF0dXJlcykpCiAgICBpZiBsYWJlbF9jb2x1bW4gaXMgbm90IE5vbmU6CiAgICAgICAgbGFiZWxzID0gc29ydGVkKGRmW2xhYmVsX2NvbHVtbl0udW5pcXVlKCkpCiAgICBlbHNlOgogICAgICAgIGxhYmVscyA9IFtOb25lXQogICAgZmlnID0gZ28uRmlndXJlKCkKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZSBhbmQgcHJvYmxlbV90eXBlID09ICJjbGFzc2lmaWNhdGlvbiI6CiAgICAgICAgZm9yIGwgaW4gbGFiZWxzOgogICAgICAgICAgICBmaWcuYWRkX3RyYWNlKAogICAgICAgICAgICAgICAgZ28uU2NhdHRlcigKICAgICAgICAgICAgICAgICAgICB4PWRmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgICAgICB5PWRmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgICAgICBtb2RlPSJtYXJrZXJzIiwKICAgICAgICAgICAgICAgICAgICB2aXNpYmxlPVRydWUsCiAgICAgICAgICAgICAgICAgICAgc2hvd2xlZ2VuZD1UcnVlLAogICAgICAgICAgICAgICAgICAgIG5hbWU9c3RyKGwpLAogICAgICAgICAgICAgICAgKQogICAgICAgICAgICApCiAgICBlbGlmIGxhYmVsX2NvbHVtbiBpcyBOb25lOgogICAgICAgIGZpZy5hZGRfdHJhY2UoCiAgICAgICAgICAgIGdvLlNjYXR0ZXIoCiAgICAgICAgICAgICAgICB4PWRmW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgIHk9ZGZbZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgbW9kZT0ibWFya2VycyIsCiAgICAgICAgICAgICAgICB2aXNpYmxlPVRydWUsCiAgICAgICAgICAgICkKICAgICAgICApCiAgICBlbGlmIHByb2JsZW1fdHlwZSA9PSAicmVncmVzc2lvbiI6CiAgICAgICAgZmlnLmFkZF90cmFjZSgKICAgICAgICAgICAgZ28uU2NhdHRlcigKICAgICAgICAgICAgICAgIHg9ZGZbZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgeT1kZltmZWF0dXJlc1swXV0sCiAgICAgICAgICAgICAgICBtb2RlPSJtYXJrZXJzIiwKICAgICAgICAgICAgICAgIG1hcmtlcj1kaWN0KAogICAgICAgICAgICAgICAgICAgIGNvbG9yPWRmW2xhYmVsX2NvbHVtbl0sIGNvbG9yc2NhbGU9IlZpcmlkaXMiLCBzaG93c2NhbGU9VHJ1ZQogICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgIHZpc2libGU9VHJ1ZSwKICAgICAgICAgICAgKQogICAgICAgICkKCiAgICB4X2J1dHRvbnMgPSBbXQogICAgeV9idXR0b25zID0gW10KCiAgICBmb3IgbmNvbCBpbiBmZWF0dXJlczoKICAgICAgICBpZiBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIiBhbmQgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lOgogICAgICAgICAgICB4X2J1dHRvbnMuYXBwZW5kKAogICAgICAgICAgICAgICAgZGljdCgKICAgICAgICAgICAgICAgICAgICBtZXRob2Q9InVwZGF0ZSIsCiAgICAgICAgICAgICAgICAgICAgbGFiZWw9bmNvbCwKICAgICAgICAgICAgICAgICAgICBhcmdzPVsKICAgICAgICAgICAgICAgICAgICAgICAgeyJ4IjogW2RmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW25jb2xdIGZvciBsIGluIGxhYmVsc119LAogICAgICAgICAgICAgICAgICAgICAgICBucC5hcmFuZ2UobGVuKGxhYmVscykpLnRvbGlzdCgpLAogICAgICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICkKCiAgICAgICAgICAgIHlfYnV0dG9ucy5hcHBlbmQoCiAgICAgICAgICAgICAgICBkaWN0KAogICAgICAgICAgICAgICAgICAgIG1ldGhvZD0idXBkYXRlIiwKICAgICAgICAgICAgICAgICAgICBsYWJlbD1uY29sLAogICAgICAgICAgICAgICAgICAgIGFyZ3M9WwogICAgICAgICAgICAgICAgICAgICAgICB7InkiOiBbZGYubG9jW2RmW2xhYmVsX2NvbHVtbl0gPT0gbF1bbmNvbF0gZm9yIGwgaW4gbGFiZWxzXX0sCiAgICAgICAgICAgICAgICAgICAgICAgIG5wLmFyYW5nZShsZW4obGFiZWxzKSkudG9saXN0KCksCiAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHhfYnV0dG9ucy5hcHBlbmQoCiAgICAgICAgICAgICAgICBkaWN0KG1ldGhvZD0idXBkYXRlIiwgbGFiZWw9bmNvbCwgYXJncz1beyJ4IjogW2RmW25jb2xdXX1dKQogICAgICAgICAgICApCgogICAgICAgICAgICB5X2J1dHRvbnMuYXBwZW5kKAogICAgICAgICAgICAgICAgZGljdChtZXRob2Q9InVwZGF0ZSIsIGxhYmVsPW5jb2wsIGFyZ3M9W3sieSI6IFtkZltuY29sXV19XSkKICAgICAgICAgICAgKQoKICAgICMgUGFzcyBidXR0b25zIHRvIHRoZSB1cGRhdGVtZW51cyBhcmd1bWVudAogICAgZmlnLnVwZGF0ZV9sYXlvdXQoCiAgICAgICAgdXBkYXRlbWVudXM9WwogICAgICAgICAgICBkaWN0KGJ1dHRvbnM9eF9idXR0b25zLCBkaXJlY3Rpb249InVwIiwgeD0wLjUsIHk9LTAuMSksCiAgICAgICAgICAgIGRpY3QoYnV0dG9ucz15X2J1dHRvbnMsIGRpcmVjdGlvbj0iZG93biIsIHg9LW1heF9mZWF0dXJlX2xlbiAvIDEwMCwgeT0wLjUpLAogICAgICAgIF0KICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dCgKICAgICAgICB3aWR0aD02MDAsCiAgICAgICAgaGVpZ2h0PTQwMCwKICAgICAgICBhdXRvc2l6ZT1GYWxzZSwKICAgICAgICBtYXJnaW49ZGljdCh0PTEwMCwgYj0wLCBsPTAsIHI9MCksCiAgICAgICAgdGVtcGxhdGU9InBsb3RseV93aGl0ZSIsCiAgICApCgogICAgZmlnLnVwZGF0ZV9sYXlvdXQodGl0bGVfdGV4dD1mIjxpPjxiPlNjYXR0ZXItMmQ8L2I+PC9pPiIpCiAgICBleHRyYV9kYXRhW2Yic2NhdHRlci0yZCJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PWYic2NhdHRlci0yZCIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vc2NhdHRlci0yZC5odG1sIiwKICAgICkKCgpkZWYgX2NyZWF0ZV92aW9saW5fYXJ0aWZhY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwgZGY6IHBkLkRhdGFGcmFtZSwgZXh0cmFfZGF0YTogZGljdCwgcGxvdHNfZGVzdDogc3RyCik6CiAgICAiIiIKICAgIENyZWF0ZSBhbmQgbG9nIGEgdmlvbGluIGFydGlmYWN0CiAgICAiIiIKICAgIGNvbHMgPSA1CiAgICByb3dzID0gKGRmLnNoYXBlWzFdIC8vIGNvbHMpICsgMQogICAgZmlnID0gbWFrZV9zdWJwbG90cyhyb3dzPXJvd3MsIGNvbHM9Y29scykKCiAgICBwbG90X251bSA9IDAKCiAgICBmb3IgY29sdW1uX25hbWUgaW4gZGYuY29sdW1uczoKICAgICAgICBjb2x1bW5fZGF0YSA9IGRmW2NvbHVtbl9uYW1lXQogICAgICAgIHZpb2xpbiA9IGdvLlZpb2xpbigKICAgICAgICAgICAgeD1bY29sdW1uX25hbWVdICogY29sdW1uX2RhdGEuc2hhcGVbMF0sCiAgICAgICAgICAgIHk9Y29sdW1uX2RhdGEsCiAgICAgICAgICAgIG5hbWU9Y29sdW1uX25hbWUsCiAgICAgICAgKQoKICAgICAgICBmaWcuYWRkX3RyYWNlKAogICAgICAgICAgICB2aW9saW4sCiAgICAgICAgICAgIHJvdz0ocGxvdF9udW0gLy8gY29scykgKyAxLAogICAgICAgICAgICBjb2w9KHBsb3RfbnVtICUgY29scykgKyAxLAogICAgICAgICkKCiAgICAgICAgcGxvdF9udW0gKz0gMQoKICAgIGZpZ1sibGF5b3V0Il0udXBkYXRlKAogICAgICAgIGhlaWdodD0ocm93cyArIDEpICogMjAwLAogICAgICAgIHdpZHRoPShjb2xzICsgMSkgKiAyMDAsCiAgICAgICAgdGl0bGU9IjxpPjxiPlZpb2xpbiBQbG90czwvYj48L2k+IiwKICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dChzaG93bGVnZW5kPUZhbHNlKQogICAgZXh0cmFfZGF0YVsidmlvbGluIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBQbG90bHlBcnRpZmFjdChrZXk9InZpb2xpbiIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vdmlvbGluLmh0bWwiLAogICAgKQoKCmRlZiBfY3JlYXRlX2ltYmFsYW5jZV9hcnRpZmFjdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhbiBpbWJhbGFuY2UgY2xhc3MgYXJ0aWZhY3QgKGNzdiArIHBsb3QpCiAgICAiIiIKICAgIGlmIGxhYmVsX2NvbHVtbjoKICAgICAgICBpZiBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICAgICAgdmFsdWVzX2NvbHVtbiA9ICJjb3VudCIKICAgICAgICAgICAgbGFiZWxzX2NvdW50ID0gZGZbbGFiZWxfY29sdW1uXS52YWx1ZV9jb3VudHMoKS5zb3J0X2luZGV4KCkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50ID0gcGQuRGF0YUZyYW1lKGxhYmVsc19jb3VudCkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W2xhYmVsX2NvbHVtbl0gPSBsYWJlbHNfY291bnQuaW5kZXgKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50LnJlbmFtZShjb2x1bW5zPXsiIjogdmFsdWVzX2NvbHVtbn0sIGlucGxhY2U9VHJ1ZSkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dID0gZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dIC8gc3VtKAogICAgICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dCiAgICAgICAgICAgICkKICAgICAgICAgICAgZmlnID0gcHgucGllKGRmX2xhYmVsc19jb3VudCwgbmFtZXM9bGFiZWxfY29sdW1uLCB2YWx1ZXM9dmFsdWVzX2NvbHVtbikKICAgICAgICBlbHNlOgogICAgICAgICAgICBmaWcgPSBweC5oaXN0b2dyYW0oCiAgICAgICAgICAgICAgICBoaXN0ZnVuYz0iY291bnQiLAogICAgICAgICAgICAgICAgeD1kZltsYWJlbF9jb2x1bW5dLAogICAgICAgICAgICApCiAgICAgICAgICAgIGhpc3QgPSBucC5oaXN0b2dyYW0oZGZbbGFiZWxfY29sdW1uXSkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50ID0gcGQuRGF0YUZyYW1lKAogICAgICAgICAgICAgICAgeyJtaW5fdmFsIjogaGlzdFsxXSwgImNvdW50IjogaGlzdFswXS50b2xpc3QoKSArIFswXX0KICAgICAgICAgICAgKQogICAgICAgIGZpZy51cGRhdGVfbGF5b3V0KHRpdGxlX3RleHQ9IjxpPjxiPkxhYmVscyBJbWJhbGFuY2U8L2I+PC9pPiIpCiAgICAgICAgZXh0cmFfZGF0YVsiaW1iYWxhbmNlIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJpbWJhbGFuY2UiLCBmaWd1cmU9ZmlnKSwKICAgICAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9pbWJhbGFuY2UuaHRtbCIsCiAgICAgICAgKQogICAgICAgIGV4dHJhX2RhdGFbImltYmFsYW5jZS1jc3YiXSA9IGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgICAgICBUYWJsZUFydGlmYWN0KCJpbWJhbGFuY2Utd2VpZ2h0cy12ZWMiLCBkZj1kZl9sYWJlbHNfY291bnQpLAogICAgICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2ltYmFsYW5jZS13ZWlnaHRzLXZlYy5jc3YiLAogICAgICAgICkKCgpkZWYgX2NyZWF0ZV9jb3JyX2FydGlmYWN0KAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkZjogcGQuRGF0YUZyYW1lLAogICAgZXh0cmFfZGF0YTogZGljdCwKICAgIGxhYmVsX2NvbHVtbjogc3RyLAogICAgcGxvdHNfZGVzdDogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhbiBjb3JyZWxhdGlvbi1tYXRyaXggYXJ0aWZhY3QgKGNzdiArIHBsb3QpCiAgICAiIiIKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZToKICAgICAgICBkZiA9IGRmLmRyb3AoW2xhYmVsX2NvbHVtbl0sIGF4aXM9MSkKICAgIHRibGNvcnIgPSBkZi5jb3JyKG51bWVyaWNfb25seT1UcnVlKQogICAgZXh0cmFfZGF0YVsiY29ycmVsYXRpb24tbWF0cml4LWNzdiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgVGFibGVBcnRpZmFjdCgiY29ycmVsYXRpb24tbWF0cml4LWNzdiIsIGRmPXRibGNvcnIsIHZpc2libGU9VHJ1ZSksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9jb3JyZWxhdGlvbi1tYXRyaXguY3N2IiwKICAgICkKCiAgICB6ID0gdGJsY29yci52YWx1ZXMudG9saXN0KCkKICAgIHpfdGV4dCA9IFtbIns6LjJmfSIuZm9ybWF0KHkpIGZvciB5IGluIHhdIGZvciB4IGluIHpdCiAgICBmaWcgPSBmZi5jcmVhdGVfYW5ub3RhdGVkX2hlYXRtYXAoCiAgICAgICAgeiwKICAgICAgICB4PWxpc3QodGJsY29yci5jb2x1bW5zKSwKICAgICAgICB5PWxpc3QodGJsY29yci5jb2x1bW5zKSwKICAgICAgICBhbm5vdGF0aW9uX3RleHQ9el90ZXh0LAogICAgICAgIGNvbG9yc2NhbGU9ImFnc3Vuc2V0IiwKICAgICkKICAgIGZpZ1sibGF5b3V0Il1bInlheGlzIl1bImF1dG9yYW5nZSJdID0gInJldmVyc2VkIiAgIyBsIC0+IHIKICAgIGZpZy51cGRhdGVfbGF5b3V0KHRpdGxlX3RleHQ9IjxpPjxiPkNvcnJlbGF0aW9uIG1hdHJpeDwvYj48L2k+IikKICAgIGZpZ1siZGF0YSJdWzBdWyJzaG93c2NhbGUiXSA9IFRydWUKCiAgICBleHRyYV9kYXRhWyJjb3JyZWxhdGlvbiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJjb3JyZWxhdGlvbiIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vY29ycmVsYXRpb24uaHRtbCIsCiAgICApCg== - code_origin: '' origin_filename: '' - description: describe and visualizes dataset stats - disable_auto_mount: false - default_handler: analyze -verbose: false -metadata: - tag: '' - name: describe - categories: - - data-analysis -kind: job + code_origin: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCmZyb20gdHlwaW5nIGltcG9ydCBVbmlvbgoKaW1wb3J0IG1scnVuCmltcG9ydCBudW1weSBhcyBucAoKd2FybmluZ3Muc2ltcGxlZmlsdGVyKGFjdGlvbj0iaWdub3JlIiwgY2F0ZWdvcnk9RnV0dXJlV2FybmluZykKCmltcG9ydCBtbHJ1bi5mZWF0dXJlX3N0b3JlIGFzIGZzdG9yZQppbXBvcnQgcGFuZGFzIGFzIHBkCmltcG9ydCBwbG90bHkuZXhwcmVzcyBhcyBweAppbXBvcnQgcGxvdGx5LmZpZ3VyZV9mYWN0b3J5IGFzIGZmCmltcG9ydCBwbG90bHkuZ3JhcGhfb2JqZWN0cyBhcyBnbwpmcm9tIG1scnVuLmFydGlmYWN0cyBpbXBvcnQgKAogICAgQXJ0aWZhY3QsCiAgICBEYXRhc2V0QXJ0aWZhY3QsCiAgICBQbG90bHlBcnRpZmFjdCwKICAgIFRhYmxlQXJ0aWZhY3QsCiAgICB1cGRhdGVfZGF0YXNldF9tZXRhLAopCmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKZnJvbSBtbHJ1bi5mZWF0dXJlX3N0b3JlIGltcG9ydCBGZWF0dXJlU2V0CmZyb20gcGxvdGx5LnN1YnBsb3RzIGltcG9ydCBtYWtlX3N1YnBsb3RzCgpwZC5zZXRfb3B0aW9uKCJkaXNwbGF5LmZsb2F0X2Zvcm1hdCIsIGxhbWJkYSB4OiAiJS4yZiIgJSB4KQpNQVhfU0laRV9PRl9ERiA9IDUwMDAwMAoKCmRlZiBhbmFseXplKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBuYW1lOiBzdHIgPSAiZGF0YXNldCIsCiAgICB0YWJsZTogVW5pb25bRmVhdHVyZVNldCwgRGF0YUl0ZW1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gTm9uZSwKICAgIHBsb3RzX2Rlc3Q6IHN0ciA9ICJwbG90cyIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDEsCiAgICBwcm9ibGVtX3R5cGU6IHN0ciA9ICJjbGFzc2lmaWNhdGlvbiIsCiAgICBkYXNrX2tleTogc3RyID0gImRhc2tfa2V5IiwKICAgIGRhc2tfZnVuY3Rpb246IHN0ciA9IE5vbmUsCiAgICBkYXNrX2NsaWVudD1Ob25lLAopIC0+IE5vbmU6CiAgICAiIiIKICAgIFRoZSBmdW5jdGlvbiB3aWxsIG91dHB1dCB0aGUgZm9sbG93aW5nIGFydGlmYWN0cyBwZXIKICAgIGNvbHVtbiB3aXRoaW4gdGhlIGRhdGEgZnJhbWUgKGJhc2VkIG9uIGRhdGEgdHlwZXMpCiAgICBJZiB0aGUgZGF0YSBoYXMgbW9yZSB0aGFuIDUwMCwwMDAgc2FtcGxlIHdlCiAgICBzYW1wbGUgcmFuZG9tbHkgNTAwLDAwMCBzYW1wbGVzOgoKICAgIGRlc2NyaWJlIGNzdgogICAgaGlzdG9ncmFtcwogICAgc2NhdHRlci0yZAogICAgdmlvbGluIGNoYXJ0CiAgICBjb3JyZWxhdGlvbi1tYXRyaXggY2hhcnQKICAgIGNvcnJlbGF0aW9uLW1hdHJpeCBjc3YKICAgIGltYmFsYW5jZSBwaWUgY2hhcnQKICAgIGltYmFsYW5jZS13ZWlnaHRzLXZlYyBjc3YKCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIFRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbmFtZTogICAgICAgICAgICAgICAgICAgIEtleSBvZiBkYXRhc2V0IHRvIGRhdGFiYXNlICgiZGF0YXNldCIgZm9yIGRlZmF1bHQpCiAgICA6cGFyYW0gdGFibGU6ICAgICAgICAgICAgICAgICAgIE1MUnVuIGlucHV0IHBvaW50aW5nIHRvIHBhbmRhcyBkYXRhZnJhbWUgKGNzdi9wYXJxdWV0IGZpbGUgcGF0aCkgb3IgRmVhdHVyZVNldAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBhcyBwYXJhbQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICAgICAgICBHcm91bmQgdHJ1dGggY29sdW1uIGxhYmVsCiAgICA6cGFyYW0gcGxvdHNfZGVzdDogICAgICAgICAgICAgIERlc3RpbmF0aW9uIGZvbGRlciBvZiBzdW1tYXJ5IHBsb3RzIChyZWxhdGl2ZSB0byBhcnRpZmFjdF9wYXRoKQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoInBsb3RzIiBmb3IgZGVmYXVsdCkKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgV2hlbiB0aGUgdGFibGUgaGFzIG1vcmUgdGhhbiA1MDAsMDAwIHNhbXBsZXMsIHdlIHNhbXBsZSByYW5kb21seSA1MDAsMDAwIHNhbXBsZXMKICAgIDpwYXJhbSBwcm9ibGVtX3R5cGUgICAgICAgICAgICAgVGhlIHR5cGUgb2YgdGhlIE1MIHByb2JsZW0gdGhlIGRhdGEgZmFjaW5nIC0gcmVncmVzc2lvbiwgY2xhc3NpZmljYXRpb24gb3IgTm9uZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoY2xhc3NpZmljYXRpb24gZm9yIGRlZmF1bHQpCiAgICA6cGFyYW0gZGFza19rZXk6ICAgICAgICAgICAgICAgIEtleSBvZiBkYXRhZnJhbWUgaW4gZGFzayBjbGllbnQgImRhdGFzZXRzIiBhdHRyaWJ1dGUKICAgIDpwYXJhbSBkYXNrX2Z1bmN0aW9uOiAgICAgICAgICAgRGFzayBmdW5jdGlvbiB1cmwgKGRiOi8vLi4pCiAgICA6cGFyYW0gZGFza19jbGllbnQ6ICAgICAgICAgICAgIERhc2sgY2xpZW50IG9iamVjdAogICAgIiIiCiAgICBkYXRhX2l0ZW0sIGZlYXR1cmVzZXQsIGNyZWF0LCB1cGRhdGUgPSBGYWxzZSwgRmFsc2UsIEZhbHNlLCBGYWxzZQogICAgZ2V0X2Zyb21fdGFibGUgPSBUcnVlCiAgICBpZiBkYXNrX2Z1bmN0aW9uIG9yIGRhc2tfY2xpZW50OgogICAgICAgIGRhdGFfaXRlbSwgY3JlYXQgPSBUcnVlLCBUcnVlCiAgICAgICAgaWYgZGFza19mdW5jdGlvbjoKICAgICAgICAgICAgY2xpZW50ID0gbWxydW4uaW1wb3J0X2Z1bmN0aW9uKGRhc2tfZnVuY3Rpb24pLmNsaWVudAogICAgICAgIGVsaWYgZGFza19jbGllbnQ6CiAgICAgICAgICAgIGNsaWVudCA9IGRhc2tfY2xpZW50CiAgICAgICAgZWxzZToKICAgICAgICAgICAgcmFpc2UgVmFsdWVFcnJvcigiZGFzayBjbGllbnQgd2FzIG5vdCBwcm92aWRlZCIpCgogICAgICAgIGlmIGRhc2tfa2V5IGluIGNsaWVudC5kYXRhc2V0czoKICAgICAgICAgICAgZGYgPSBjbGllbnQuZ2V0X2RhdGFzZXQoZGFza19rZXkpCiAgICAgICAgICAgIGRhdGFfaXRlbSwgY3JlYXQsIGdldF9mcm9tX3RhYmxlID0gVHJ1ZSwgVHJ1ZSwgRmFsc2UKICAgICAgICBlbGlmIHRhYmxlOgogICAgICAgICAgICBnZXRfZnJvbV90YWJsZSA9IFRydWUKICAgICAgICBlbHNlOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgZiJvbmx5IHRoZXNlIGRhdGFzZXRzIGFyZSBhdmFpbGFibGUge2NsaWVudC5kYXRhc2V0c30gaW4gY2xpZW50IHtjbGllbnR9IgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIEV4Y2VwdGlvbigiZGF0YXNldCBub3QgZm91bmQgb24gZGFzayBjbHVzdGVyIikKCiAgICBpZiBnZXRfZnJvbV90YWJsZToKICAgICAgICBpZiB0eXBlKHRhYmxlKSA9PSBEYXRhSXRlbToKICAgICAgICAgICAgaWYgdGFibGUubWV0YSBpcyBOb25lOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgVHJ1ZSwgRmFsc2UKICAgICAgICAgICAgZWxpZiB0YWJsZS5tZXRhLmtpbmQgPT0gImRhdGFzZXQiOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgRmFsc2UsIFRydWUKICAgICAgICAgICAgZWxpZiB0YWJsZS5tZXRhLmtpbmQgPT0gIkZlYXR1cmVWZWN0b3IiOgogICAgICAgICAgICAgICAgZGF0YV9pdGVtLCBjcmVhdCwgdXBkYXRlID0gVHJ1ZSwgRmFsc2UsIEZhbHNlCiAgICAgICAgICAgIGVsaWYgdGFibGUubWV0YS5raW5kID09ICJGZWF0dXJlU2V0IjoKICAgICAgICAgICAgICAgIGZlYXR1cmVzZXQsIGNyZWF0LCB1cGRhdGUgPSBUcnVlLCBGYWxzZSwgRmFsc2UKCiAgICAgICAgaWYgZGF0YV9pdGVtOgogICAgICAgICAgICBkZiA9IHRhYmxlLmFzX2RmKCkKICAgICAgICBlbGlmIGZlYXR1cmVzZXQ6CiAgICAgICAgICAgIHByb2plY3RfbmFtZSwgc2V0X25hbWUgPSAoCiAgICAgICAgICAgICAgICB0YWJsZS5fcGF0aC5zcGxpdCgiLyIpWzJdLAogICAgICAgICAgICAgICAgdGFibGUuX3BhdGguc3BsaXQoIi8iKVs0XSwKICAgICAgICAgICAgKQogICAgICAgICAgICBmZWF0dXJlX3NldCA9IGZzdG9yZS5nZXRfZmVhdHVyZV9zZXQoCiAgICAgICAgICAgICAgICBmInN0b3JlOi8vZmVhdHVyZS1zZXRzL3twcm9qZWN0X25hbWV9L3tzZXRfbmFtZX0iCiAgICAgICAgICAgICkKICAgICAgICAgICAgZGYgPSBmZWF0dXJlX3NldC50b19kYXRhZnJhbWUoKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKGYiV3JvbmcgdGFibGUgdHlwZS4iKQogICAgICAgICAgICByZXR1cm4KCiAgICBpZiBkZi5zaXplID4gTUFYX1NJWkVfT0ZfREY6CiAgICAgICAgZGYgPSBkZi5zYW1wbGUobj1pbnQoTUFYX1NJWkVfT0ZfREYgLyBkZi5zaGFwZVsxXSksIHJhbmRvbV9zdGF0ZT1yYW5kb21fc3RhdGUpCiAgICBleHRyYV9kYXRhID0ge30KCiAgICBpZiBsYWJlbF9jb2x1bW4gbm90IGluIGRmLmNvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1uID0gTm9uZQoKICAgIGV4dHJhX2RhdGFbImRlc2NyaWJlIGNzdiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgVGFibGVBcnRpZmFjdCgiZGVzY3JpYmUtY3N2IiwgZGY9ZGYuZGVzY3JpYmUoKSksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9kZXNjcmliZS5jc3YiLAogICAgKQoKICAgIHRyeToKICAgICAgICBfY3JlYXRlX2hpc3RvZ3JhbV9tYXRfYXJ0aWZhY3QoCiAgICAgICAgICAgIGNvbnRleHQsIGRmLCBleHRyYV9kYXRhLCBsYWJlbF9jb2x1bW4sIHBsb3RzX2Rlc3QKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgaGlzdG9ncmFtIG1hdHJpeCBhcnRpZmFjdCBkdWUgdG86IHtlfSIpCiAgICB0cnk6CiAgICAgICAgX2NyZWF0ZV9mZWF0dXJlc19oaXN0b2dyYW1fYXJ0aWZhY3RzKAogICAgICAgICAgICBjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0LCBwcm9ibGVtX3R5cGUKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgcGFpcnBsb3QgaGlzdG9ncmFtcyBkdWUgdG86IHtlfSIpCiAgICB0cnk6CiAgICAgICAgX2NyZWF0ZV9mZWF0dXJlc18yZF9zY2F0dGVyX2FydGlmYWN0cygKICAgICAgICAgICAgY29udGV4dCwgZGYsIGV4dHJhX2RhdGEsIGxhYmVsX2NvbHVtbiwgcGxvdHNfZGVzdCwgcHJvYmxlbV90eXBlCiAgICAgICAgKQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIHBhaXJwbG90IDJkX3NjYXR0ZXIgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfdmlvbGluX2FydGlmYWN0KGNvbnRleHQsIGRmLCBleHRyYV9kYXRhLCBwbG90c19kZXN0KQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIHZpb2xpbiBkaXN0cmlidXRpb24gcGxvdHMgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfaW1iYWxhbmNlX2FydGlmYWN0KAogICAgICAgICAgICBjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0LCBwcm9ibGVtX3R5cGUKICAgICAgICApCiAgICBleGNlcHQgRXhjZXB0aW9uIGFzIGU6CiAgICAgICAgY29udGV4dC5sb2dnZXIud2FybihmIkZhaWxlZCB0byBjcmVhdGUgY2xhc3MgaW1iYWxhbmNlIHBsb3QgZHVlIHRvOiB7ZX0iKQogICAgdHJ5OgogICAgICAgIF9jcmVhdGVfY29ycl9hcnRpZmFjdChjb250ZXh0LCBkZiwgZXh0cmFfZGF0YSwgbGFiZWxfY29sdW1uLCBwbG90c19kZXN0KQogICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlOgogICAgICAgIGNvbnRleHQubG9nZ2VyLndhcm4oZiJGYWlsZWQgdG8gY3JlYXRlIGZlYXR1cmVzIGNvcnJlbGF0aW9uIHBsb3QgZHVlIHRvOiB7ZX0iKQoKICAgIGlmIG5vdCBkYXRhX2l0ZW06CiAgICAgICAgcmV0dXJuCgogICAgYXJ0aWZhY3QgPSB0YWJsZS5hcnRpZmFjdF91cmwKICAgIGlmIGNyZWF0OiAgIyBkYXRhc2V0IG5vdCBzdG9yZWQKICAgICAgICBhcnRpZmFjdCA9IERhdGFzZXRBcnRpZmFjdCgKICAgICAgICAgICAga2V5PSJkYXRhc2V0Iiwgc3RhdHM9VHJ1ZSwgZGY9ZGYsIGV4dHJhX2RhdGE9ZXh0cmFfZGF0YQogICAgICAgICkKICAgICAgICBhcnRpZmFjdCA9IGNvbnRleHQubG9nX2FydGlmYWN0KGFydGlmYWN0LCBkYl9rZXk9bmFtZSkKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVGhlIGRhdGEgc2V0IGlzIGxvZ2dlZCB0byB0aGUgcHJvamVjdCB1bmRlciB7bmFtZX0gbmFtZSIpCgogICAgaWYgdXBkYXRlOgogICAgICAgIHVwZGF0ZV9kYXRhc2V0X21ldGEoYXJ0aWZhY3QsIGV4dHJhX2RhdGE9ZXh0cmFfZGF0YSkKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiVGhlIGRhdGEgc2V0IG5hbWVkIHtuYW1lfSBpcyB1cGRhdGVkIikKCiAgICAjIFRPRE8gOiAzLUQgcGxvdCBvbiBvbiBzZWxlY3RlZCBmZWF0dXJlcy4KICAgICMgVE9ETyA6IFJlaW50ZWdyYXRpb24gcGxvdCBvbiBvbiBzZWxlY3RlZCBmZWF0dXJlcy4KICAgICMgVE9ETyA6IFBDQSBwbG90ICh3aXRoIG9wdGlvbnMpCgoKZGVmIF9jcmVhdGVfaGlzdG9ncmFtX21hdF9hcnRpZmFjdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKKToKICAgICIiIgogICAgQ3JlYXRlIGFuZCBsb2cgYSBoaXN0b2dyYW0gbWF0cml4IGFydGlmYWN0CiAgICAiIiIKICAgIGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgIGl0ZW09QXJ0aWZhY3QoCiAgICAgICAgICAgIGtleT0iaGlzdCIsCiAgICAgICAgICAgIGJvZHk9YiI8Yj4gRGVwcmVjYXRlZCwgc2VlIHRoZSBhcnRpZmFjdHMgc2NhdHRlci0yZCAiCiAgICAgICAgICAgIGIiYW5kIGhpc3RvZ3JhbXMgaW5zdGVhZDxiPiIsCiAgICAgICAgKSwKICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2hpc3QuaHRtbCIsCiAgICApCgoKZGVmIF9jcmVhdGVfZmVhdHVyZXNfaGlzdG9ncmFtX2FydGlmYWN0cygKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhIGhpc3RvZ3JhbSBhcnRpZmFjdCBmb3IgZWFjaCBmZWF0dXJlCiAgICAiIiIKCiAgICBmaWdzID0gZGljdCgpCiAgICBmaXJzdF9mZWF0dXJlX25hbWUgPSAiIgogICAgaWYgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lIGFuZCBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICBhbGxfbGFiZWxzID0gZGZbbGFiZWxfY29sdW1uXS51bmlxdWUoKQogICAgdmlzaWJsZSA9IFRydWUKICAgIGZvciBjb2x1bW5fbmFtZSBpbiBkZi5jb2x1bW5zOgogICAgICAgIGlmIGNvbHVtbl9uYW1lID09IGxhYmVsX2NvbHVtbjoKICAgICAgICAgICAgY29udGludWUKCiAgICAgICAgaWYgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lIGFuZCBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICAgICAgZm9yIGxhYmVsIGluIGFsbF9sYWJlbHM6CiAgICAgICAgICAgICAgICBzdWJfZmlnID0gZ28uSGlzdG9ncmFtKAogICAgICAgICAgICAgICAgICAgIGhpc3RmdW5jPSJjb3VudCIsCiAgICAgICAgICAgICAgICAgICAgeD1kZi5sb2NbZGZbbGFiZWxfY29sdW1uXSA9PSBsYWJlbF1bY29sdW1uX25hbWVdLAogICAgICAgICAgICAgICAgICAgIG5hbWU9c3RyKGxhYmVsKSwKICAgICAgICAgICAgICAgICAgICB2aXNpYmxlPXZpc2libGUsCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICBmaWdzW2Yie2NvbHVtbl9uYW1lfUA/QHtsYWJlbH0iXSA9IHN1Yl9maWcKICAgICAgICBlbHNlOgogICAgICAgICAgICBzdWJfZmlnID0gZ28uSGlzdG9ncmFtKGhpc3RmdW5jPSJjb3VudCIsIHg9ZGZbY29sdW1uX25hbWVdLCB2aXNpYmxlPXZpc2libGUpCiAgICAgICAgICAgIGZpZ3NbZiJ7Y29sdW1uX25hbWV9QD9AezF9Il0gPSBzdWJfZmlnCiAgICAgICAgaWYgdmlzaWJsZToKICAgICAgICAgICAgZmlyc3RfZmVhdHVyZV9uYW1lID0gY29sdW1uX25hbWUKICAgICAgICB2aXNpYmxlID0gRmFsc2UKCiAgICBmaWcgPSBnby5GaWd1cmUoKQogICAgZm9yIGsgaW4gZmlncy5rZXlzKCk6CiAgICAgICAgZmlnLmFkZF90cmFjZShmaWdzW2tdKQoKICAgIGZpZy51cGRhdGVfbGF5b3V0KAogICAgICAgIHVwZGF0ZW1lbnVzPVsKICAgICAgICAgICAgewogICAgICAgICAgICAgICAgImJ1dHRvbnMiOiBbCiAgICAgICAgICAgICAgICAgICAgewogICAgICAgICAgICAgICAgICAgICAgICAibGFiZWwiOiBjb2x1bW5fbmFtZSwKICAgICAgICAgICAgICAgICAgICAgICAgIm1ldGhvZCI6ICJ1cGRhdGUiLAogICAgICAgICAgICAgICAgICAgICAgICAiYXJncyI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAidmlzaWJsZSI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAga2V5LnNwbGl0KCJAP0AiKVswXSA9PSBjb2x1bW5fbmFtZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb3Iga2V5IGluIGZpZ3Mua2V5cygpCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAieGF4aXMiOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICJyYW5nZSI6IFsKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1pbihkZltjb2x1bW5fbmFtZV0pLAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbWF4KGRmW2NvbHVtbl9uYW1lXSksCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgfSwKICAgICAgICAgICAgICAgICAgICAgICAgICAgIHsidGl0bGUiOiBmIjxpPjxiPkhpc3RvZ3JhbSBvZiB7Y29sdW1uX25hbWV9PC9iPjwvaT4ifSwKICAgICAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICAgICB9CiAgICAgICAgICAgICAgICAgICAgZm9yIGNvbHVtbl9uYW1lIGluIGRmLmNvbHVtbnMKICAgICAgICAgICAgICAgICAgICBpZiBjb2x1bW5fbmFtZSAhPSBsYWJlbF9jb2x1bW4KICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICAiZGlyZWN0aW9uIjogImRvd24iLAogICAgICAgICAgICAgICAgInBhZCI6IHsiciI6IDEwLCAidCI6IDEwfSwKICAgICAgICAgICAgICAgICJzaG93YWN0aXZlIjogVHJ1ZSwKICAgICAgICAgICAgICAgICJ4IjogMC4yNSwKICAgICAgICAgICAgICAgICJ4YW5jaG9yIjogImxlZnQiLAogICAgICAgICAgICAgICAgInkiOiAxLjEsCiAgICAgICAgICAgICAgICAieWFuY2hvciI6ICJ0b3AiLAogICAgICAgICAgICB9CiAgICAgICAgXSwKICAgICAgICBhbm5vdGF0aW9ucz1bCiAgICAgICAgICAgIGRpY3QoCiAgICAgICAgICAgICAgICB0ZXh0PSJTZWxlY3QgRmVhdHVyZSBOYW1lICIsCiAgICAgICAgICAgICAgICBzaG93YXJyb3c9RmFsc2UsCiAgICAgICAgICAgICAgICB4PTAsCiAgICAgICAgICAgICAgICB5PTEuMDUsCiAgICAgICAgICAgICAgICB5cmVmPSJwYXBlciIsCiAgICAgICAgICAgICAgICB4cmVmPSJwYXBlciIsCiAgICAgICAgICAgICAgICBhbGlnbj0ibGVmdCIsCiAgICAgICAgICAgICAgICB4YW5jaG9yPSJsZWZ0IiwKICAgICAgICAgICAgICAgIHlhbmNob3I9InRvcCIsCiAgICAgICAgICAgICAgICBmb250PXsKICAgICAgICAgICAgICAgICAgICAiY29sb3IiOiAiYmx1ZSIsCiAgICAgICAgICAgICAgICB9LAogICAgICAgICAgICApCiAgICAgICAgXSwKICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dCgKICAgICAgICB3aWR0aD02MDAsCiAgICAgICAgaGVpZ2h0PTQwMCwKICAgICAgICBhdXRvc2l6ZT1GYWxzZSwKICAgICAgICBtYXJnaW49ZGljdCh0PTEwMCwgYj0wLCBsPTAsIHI9MCksCiAgICAgICAgdGVtcGxhdGU9InBsb3RseV93aGl0ZSIsCiAgICApCgogICAgZmlnLnVwZGF0ZV9sYXlvdXQodGl0bGVfdGV4dD1mIjxpPjxiPkhpc3RvZ3JhbXMgb2Yge2ZpcnN0X2ZlYXR1cmVfbmFtZX08L2I+PC9pPiIpCiAgICBleHRyYV9kYXRhW2YiaGlzdG9ncmFtcyJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PWYiaGlzdG9ncmFtcyIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vaGlzdG9ncmFtcy5odG1sIiwKICAgICkKCgpkZWYgX2NyZWF0ZV9mZWF0dXJlc18yZF9zY2F0dGVyX2FydGlmYWN0cygKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhIHNjYXR0ZXItMmQgYXJ0aWZhY3QgZm9yIGVhY2ggY291cGxlIG9mIGZlYXR1cmVzCiAgICAiIiIKICAgIGZlYXR1cmVzID0gWwogICAgICAgIGNvbHVtbl9uYW1lIGZvciBjb2x1bW5fbmFtZSBpbiBkZi5jb2x1bW5zIGlmIGNvbHVtbl9uYW1lICE9IGxhYmVsX2NvbHVtbgogICAgXQogICAgbWF4X2ZlYXR1cmVfbGVuID0gZmxvYXQobWF4KGxlbihlbGVtKSBmb3IgZWxlbSBpbiBmZWF0dXJlcykpCiAgICBpZiBsYWJlbF9jb2x1bW4gaXMgbm90IE5vbmU6CiAgICAgICAgbGFiZWxzID0gc29ydGVkKGRmW2xhYmVsX2NvbHVtbl0udW5pcXVlKCkpCiAgICBlbHNlOgogICAgICAgIGxhYmVscyA9IFtOb25lXQogICAgZmlnID0gZ28uRmlndXJlKCkKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZSBhbmQgcHJvYmxlbV90eXBlID09ICJjbGFzc2lmaWNhdGlvbiI6CiAgICAgICAgZm9yIGwgaW4gbGFiZWxzOgogICAgICAgICAgICBmaWcuYWRkX3RyYWNlKAogICAgICAgICAgICAgICAgZ28uU2NhdHRlcigKICAgICAgICAgICAgICAgICAgICB4PWRmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgICAgICB5PWRmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgICAgICBtb2RlPSJtYXJrZXJzIiwKICAgICAgICAgICAgICAgICAgICB2aXNpYmxlPVRydWUsCiAgICAgICAgICAgICAgICAgICAgc2hvd2xlZ2VuZD1UcnVlLAogICAgICAgICAgICAgICAgICAgIG5hbWU9c3RyKGwpLAogICAgICAgICAgICAgICAgKQogICAgICAgICAgICApCiAgICBlbGlmIGxhYmVsX2NvbHVtbiBpcyBOb25lOgogICAgICAgIGZpZy5hZGRfdHJhY2UoCiAgICAgICAgICAgIGdvLlNjYXR0ZXIoCiAgICAgICAgICAgICAgICB4PWRmW2ZlYXR1cmVzWzBdXSwKICAgICAgICAgICAgICAgIHk9ZGZbZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgbW9kZT0ibWFya2VycyIsCiAgICAgICAgICAgICAgICB2aXNpYmxlPVRydWUsCiAgICAgICAgICAgICkKICAgICAgICApCiAgICBlbGlmIHByb2JsZW1fdHlwZSA9PSAicmVncmVzc2lvbiI6CiAgICAgICAgZmlnLmFkZF90cmFjZSgKICAgICAgICAgICAgZ28uU2NhdHRlcigKICAgICAgICAgICAgICAgIHg9ZGZbZmVhdHVyZXNbMF1dLAogICAgICAgICAgICAgICAgeT1kZltmZWF0dXJlc1swXV0sCiAgICAgICAgICAgICAgICBtb2RlPSJtYXJrZXJzIiwKICAgICAgICAgICAgICAgIG1hcmtlcj1kaWN0KAogICAgICAgICAgICAgICAgICAgIGNvbG9yPWRmW2xhYmVsX2NvbHVtbl0sIGNvbG9yc2NhbGU9IlZpcmlkaXMiLCBzaG93c2NhbGU9VHJ1ZQogICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgIHZpc2libGU9VHJ1ZSwKICAgICAgICAgICAgKQogICAgICAgICkKCiAgICB4X2J1dHRvbnMgPSBbXQogICAgeV9idXR0b25zID0gW10KCiAgICBmb3IgbmNvbCBpbiBmZWF0dXJlczoKICAgICAgICBpZiBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIiBhbmQgbGFiZWxfY29sdW1uIGlzIG5vdCBOb25lOgogICAgICAgICAgICB4X2J1dHRvbnMuYXBwZW5kKAogICAgICAgICAgICAgICAgZGljdCgKICAgICAgICAgICAgICAgICAgICBtZXRob2Q9InVwZGF0ZSIsCiAgICAgICAgICAgICAgICAgICAgbGFiZWw9bmNvbCwKICAgICAgICAgICAgICAgICAgICBhcmdzPVsKICAgICAgICAgICAgICAgICAgICAgICAgeyJ4IjogW2RmLmxvY1tkZltsYWJlbF9jb2x1bW5dID09IGxdW25jb2xdIGZvciBsIGluIGxhYmVsc119LAogICAgICAgICAgICAgICAgICAgICAgICBucC5hcmFuZ2UobGVuKGxhYmVscykpLnRvbGlzdCgpLAogICAgICAgICAgICAgICAgICAgIF0sCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICkKCiAgICAgICAgICAgIHlfYnV0dG9ucy5hcHBlbmQoCiAgICAgICAgICAgICAgICBkaWN0KAogICAgICAgICAgICAgICAgICAgIG1ldGhvZD0idXBkYXRlIiwKICAgICAgICAgICAgICAgICAgICBsYWJlbD1uY29sLAogICAgICAgICAgICAgICAgICAgIGFyZ3M9WwogICAgICAgICAgICAgICAgICAgICAgICB7InkiOiBbZGYubG9jW2RmW2xhYmVsX2NvbHVtbl0gPT0gbF1bbmNvbF0gZm9yIGwgaW4gbGFiZWxzXX0sCiAgICAgICAgICAgICAgICAgICAgICAgIG5wLmFyYW5nZShsZW4obGFiZWxzKSkudG9saXN0KCksCiAgICAgICAgICAgICAgICAgICAgXSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgKQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIHhfYnV0dG9ucy5hcHBlbmQoCiAgICAgICAgICAgICAgICBkaWN0KG1ldGhvZD0idXBkYXRlIiwgbGFiZWw9bmNvbCwgYXJncz1beyJ4IjogW2RmW25jb2xdXX1dKQogICAgICAgICAgICApCgogICAgICAgICAgICB5X2J1dHRvbnMuYXBwZW5kKAogICAgICAgICAgICAgICAgZGljdChtZXRob2Q9InVwZGF0ZSIsIGxhYmVsPW5jb2wsIGFyZ3M9W3sieSI6IFtkZltuY29sXV19XSkKICAgICAgICAgICAgKQoKICAgICMgUGFzcyBidXR0b25zIHRvIHRoZSB1cGRhdGVtZW51cyBhcmd1bWVudAogICAgZmlnLnVwZGF0ZV9sYXlvdXQoCiAgICAgICAgdXBkYXRlbWVudXM9WwogICAgICAgICAgICBkaWN0KGJ1dHRvbnM9eF9idXR0b25zLCBkaXJlY3Rpb249InVwIiwgeD0wLjUsIHk9LTAuMSksCiAgICAgICAgICAgIGRpY3QoYnV0dG9ucz15X2J1dHRvbnMsIGRpcmVjdGlvbj0iZG93biIsIHg9LW1heF9mZWF0dXJlX2xlbiAvIDEwMCwgeT0wLjUpLAogICAgICAgIF0KICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dCgKICAgICAgICB3aWR0aD02MDAsCiAgICAgICAgaGVpZ2h0PTQwMCwKICAgICAgICBhdXRvc2l6ZT1GYWxzZSwKICAgICAgICBtYXJnaW49ZGljdCh0PTEwMCwgYj0wLCBsPTAsIHI9MCksCiAgICAgICAgdGVtcGxhdGU9InBsb3RseV93aGl0ZSIsCiAgICApCgogICAgZmlnLnVwZGF0ZV9sYXlvdXQodGl0bGVfdGV4dD1mIjxpPjxiPlNjYXR0ZXItMmQ8L2I+PC9pPiIpCiAgICBleHRyYV9kYXRhW2Yic2NhdHRlci0yZCJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PWYic2NhdHRlci0yZCIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vc2NhdHRlci0yZC5odG1sIiwKICAgICkKCgpkZWYgX2NyZWF0ZV92aW9saW5fYXJ0aWZhY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwgZGY6IHBkLkRhdGFGcmFtZSwgZXh0cmFfZGF0YTogZGljdCwgcGxvdHNfZGVzdDogc3RyCik6CiAgICAiIiIKICAgIENyZWF0ZSBhbmQgbG9nIGEgdmlvbGluIGFydGlmYWN0CiAgICAiIiIKICAgIGNvbHMgPSA1CiAgICByb3dzID0gKGRmLnNoYXBlWzFdIC8vIGNvbHMpICsgMQogICAgZmlnID0gbWFrZV9zdWJwbG90cyhyb3dzPXJvd3MsIGNvbHM9Y29scykKCiAgICBwbG90X251bSA9IDAKCiAgICBmb3IgY29sdW1uX25hbWUgaW4gZGYuY29sdW1uczoKICAgICAgICBjb2x1bW5fZGF0YSA9IGRmW2NvbHVtbl9uYW1lXQogICAgICAgIHZpb2xpbiA9IGdvLlZpb2xpbigKICAgICAgICAgICAgeD1bY29sdW1uX25hbWVdICogY29sdW1uX2RhdGEuc2hhcGVbMF0sCiAgICAgICAgICAgIHk9Y29sdW1uX2RhdGEsCiAgICAgICAgICAgIG5hbWU9Y29sdW1uX25hbWUsCiAgICAgICAgKQoKICAgICAgICBmaWcuYWRkX3RyYWNlKAogICAgICAgICAgICB2aW9saW4sCiAgICAgICAgICAgIHJvdz0ocGxvdF9udW0gLy8gY29scykgKyAxLAogICAgICAgICAgICBjb2w9KHBsb3RfbnVtICUgY29scykgKyAxLAogICAgICAgICkKCiAgICAgICAgcGxvdF9udW0gKz0gMQoKICAgIGZpZ1sibGF5b3V0Il0udXBkYXRlKAogICAgICAgIGhlaWdodD0ocm93cyArIDEpICogMjAwLAogICAgICAgIHdpZHRoPShjb2xzICsgMSkgKiAyMDAsCiAgICAgICAgdGl0bGU9IjxpPjxiPlZpb2xpbiBQbG90czwvYj48L2k+IiwKICAgICkKCiAgICBmaWcudXBkYXRlX2xheW91dChzaG93bGVnZW5kPUZhbHNlKQogICAgZXh0cmFfZGF0YVsidmlvbGluIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICBQbG90bHlBcnRpZmFjdChrZXk9InZpb2xpbiIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vdmlvbGluLmh0bWwiLAogICAgKQoKCmRlZiBfY3JlYXRlX2ltYmFsYW5jZV9hcnRpZmFjdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGY6IHBkLkRhdGFGcmFtZSwKICAgIGV4dHJhX2RhdGE6IGRpY3QsCiAgICBsYWJlbF9jb2x1bW46IHN0ciwKICAgIHBsb3RzX2Rlc3Q6IHN0ciwKICAgIHByb2JsZW1fdHlwZTogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhbiBpbWJhbGFuY2UgY2xhc3MgYXJ0aWZhY3QgKGNzdiArIHBsb3QpCiAgICAiIiIKICAgIGlmIGxhYmVsX2NvbHVtbjoKICAgICAgICBpZiBwcm9ibGVtX3R5cGUgPT0gImNsYXNzaWZpY2F0aW9uIjoKICAgICAgICAgICAgdmFsdWVzX2NvbHVtbiA9ICJjb3VudCIKICAgICAgICAgICAgbGFiZWxzX2NvdW50ID0gZGZbbGFiZWxfY29sdW1uXS52YWx1ZV9jb3VudHMoKS5zb3J0X2luZGV4KCkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50ID0gcGQuRGF0YUZyYW1lKGxhYmVsc19jb3VudCkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W2xhYmVsX2NvbHVtbl0gPSBsYWJlbHNfY291bnQuaW5kZXgKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50LnJlbmFtZShjb2x1bW5zPXsiIjogdmFsdWVzX2NvbHVtbn0sIGlucGxhY2U9VHJ1ZSkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dID0gZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dIC8gc3VtKAogICAgICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50W3ZhbHVlc19jb2x1bW5dCiAgICAgICAgICAgICkKICAgICAgICAgICAgZmlnID0gcHgucGllKGRmX2xhYmVsc19jb3VudCwgbmFtZXM9bGFiZWxfY29sdW1uLCB2YWx1ZXM9dmFsdWVzX2NvbHVtbikKICAgICAgICBlbHNlOgogICAgICAgICAgICBmaWcgPSBweC5oaXN0b2dyYW0oCiAgICAgICAgICAgICAgICBoaXN0ZnVuYz0iY291bnQiLAogICAgICAgICAgICAgICAgeD1kZltsYWJlbF9jb2x1bW5dLAogICAgICAgICAgICApCiAgICAgICAgICAgIGhpc3QgPSBucC5oaXN0b2dyYW0oZGZbbGFiZWxfY29sdW1uXSkKICAgICAgICAgICAgZGZfbGFiZWxzX2NvdW50ID0gcGQuRGF0YUZyYW1lKAogICAgICAgICAgICAgICAgeyJtaW5fdmFsIjogaGlzdFsxXSwgImNvdW50IjogaGlzdFswXS50b2xpc3QoKSArIFswXX0KICAgICAgICAgICAgKQogICAgICAgIGZpZy51cGRhdGVfbGF5b3V0KHRpdGxlX3RleHQ9IjxpPjxiPkxhYmVscyBJbWJhbGFuY2U8L2I+PC9pPiIpCiAgICAgICAgZXh0cmFfZGF0YVsiaW1iYWxhbmNlIl0gPSBjb250ZXh0LmxvZ19hcnRpZmFjdCgKICAgICAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJpbWJhbGFuY2UiLCBmaWd1cmU9ZmlnKSwKICAgICAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9pbWJhbGFuY2UuaHRtbCIsCiAgICAgICAgKQogICAgICAgIGV4dHJhX2RhdGFbImltYmFsYW5jZS1jc3YiXSA9IGNvbnRleHQubG9nX2FydGlmYWN0KAogICAgICAgICAgICBUYWJsZUFydGlmYWN0KCJpbWJhbGFuY2Utd2VpZ2h0cy12ZWMiLCBkZj1kZl9sYWJlbHNfY291bnQpLAogICAgICAgICAgICBsb2NhbF9wYXRoPWYie3Bsb3RzX2Rlc3R9L2ltYmFsYW5jZS13ZWlnaHRzLXZlYy5jc3YiLAogICAgICAgICkKCgpkZWYgX2NyZWF0ZV9jb3JyX2FydGlmYWN0KAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkZjogcGQuRGF0YUZyYW1lLAogICAgZXh0cmFfZGF0YTogZGljdCwKICAgIGxhYmVsX2NvbHVtbjogc3RyLAogICAgcGxvdHNfZGVzdDogc3RyLAopOgogICAgIiIiCiAgICBDcmVhdGUgYW5kIGxvZyBhbiBjb3JyZWxhdGlvbi1tYXRyaXggYXJ0aWZhY3QgKGNzdiArIHBsb3QpCiAgICAiIiIKICAgIGlmIGxhYmVsX2NvbHVtbiBpcyBub3QgTm9uZToKICAgICAgICBkZiA9IGRmLmRyb3AoW2xhYmVsX2NvbHVtbl0sIGF4aXM9MSkKICAgIHRibGNvcnIgPSBkZi5jb3JyKG51bWVyaWNfb25seT1UcnVlKQogICAgZXh0cmFfZGF0YVsiY29ycmVsYXRpb24tbWF0cml4LWNzdiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgVGFibGVBcnRpZmFjdCgiY29ycmVsYXRpb24tbWF0cml4LWNzdiIsIGRmPXRibGNvcnIsIHZpc2libGU9VHJ1ZSksCiAgICAgICAgbG9jYWxfcGF0aD1mIntwbG90c19kZXN0fS9jb3JyZWxhdGlvbi1tYXRyaXguY3N2IiwKICAgICkKCiAgICB6ID0gdGJsY29yci52YWx1ZXMudG9saXN0KCkKICAgIHpfdGV4dCA9IFtbIns6LjJmfSIuZm9ybWF0KHkpIGZvciB5IGluIHhdIGZvciB4IGluIHpdCiAgICBmaWcgPSBmZi5jcmVhdGVfYW5ub3RhdGVkX2hlYXRtYXAoCiAgICAgICAgeiwKICAgICAgICB4PWxpc3QodGJsY29yci5jb2x1bW5zKSwKICAgICAgICB5PWxpc3QodGJsY29yci5jb2x1bW5zKSwKICAgICAgICBhbm5vdGF0aW9uX3RleHQ9el90ZXh0LAogICAgICAgIGNvbG9yc2NhbGU9ImFnc3Vuc2V0IiwKICAgICkKICAgIGZpZ1sibGF5b3V0Il1bInlheGlzIl1bImF1dG9yYW5nZSJdID0gInJldmVyc2VkIiAgIyBsIC0+IHIKICAgIGZpZy51cGRhdGVfbGF5b3V0KHRpdGxlX3RleHQ9IjxpPjxiPkNvcnJlbGF0aW9uIG1hdHJpeDwvYj48L2k+IikKICAgIGZpZ1siZGF0YSJdWzBdWyJzaG93c2NhbGUiXSA9IFRydWUKCiAgICBleHRyYV9kYXRhWyJjb3JyZWxhdGlvbiJdID0gY29udGV4dC5sb2dfYXJ0aWZhY3QoCiAgICAgICAgUGxvdGx5QXJ0aWZhY3Qoa2V5PSJjb3JyZWxhdGlvbiIsIGZpZ3VyZT1maWcpLAogICAgICAgIGxvY2FsX3BhdGg9ZiJ7cGxvdHNfZGVzdH0vY29ycmVsYXRpb24uaHRtbCIsCiAgICApCg== diff --git a/functions/src/describe/item.yaml b/functions/src/describe/item.yaml index da26f1501..a1aa47372 100644 --- a/functions/src/describe/item.yaml +++ b/functions/src/describe/item.yaml @@ -11,7 +11,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.0 +mlrunVersion: 1.10.0 name: describe platformVersion: 3.5.3 spec: @@ -21,4 +21,4 @@ spec: kind: job requirements: [] url: '' -version: 1.4.0 +version: 1.5.0 diff --git a/functions/src/describe/requirements.txt b/functions/src/describe/requirements.txt index 15492b176..7a15c8465 100644 --- a/functions/src/describe/requirements.txt +++ b/functions/src/describe/requirements.txt @@ -1,4 +1,4 @@ -scikit-learn~=1.0.2 +scikit-learn~=1.5 plotly~=5.23 pytest~=7.0.1 matplotlib~=3.5.1 diff --git a/functions/src/gen_class_data/function.yaml b/functions/src/gen_class_data/function.yaml index 1769bec07..fde89341e 100644 --- a/functions/src/gen_class_data/function.yaml +++ b/functions/src/gen_class_data/function.yaml @@ -1,13 +1,15 @@ metadata: - categories: - - data-generation tag: '' name: gen-class-data + categories: + - data-generation +verbose: false spec: description: Create a binary classification sample dataset and save. - default_handler: gen_class_data entry_points: gen_class_data: + lineno: 22 + has_varargs: false has_kwargs: false parameters: - name: context @@ -48,7 +50,6 @@ spec: - name: sk_params doc: additional parameters for `sklearn.datasets.make_classification` default: {} - lineno: 22 doc: 'Create a binary classification sample dataset and save. If no filename is given it will default to: @@ -59,14 +60,14 @@ spec: Additional scikit-learn parameters can be set using **sk_params, please see https://scikit-learn.org/stable/modules/generated/sklearn.datasets.make_classification.html for more details.' - has_varargs: false name: gen_class_data - command: '' - disable_auto_mount: false - image: mlrun/mlrun build: origin_filename: '' functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IHBhbmRhcyBhcyBwZApmcm9tIHR5cGluZyBpbXBvcnQgT3B0aW9uYWwsIExpc3QKZnJvbSBza2xlYXJuLmRhdGFzZXRzIGltcG9ydCBtYWtlX2NsYXNzaWZpY2F0aW9uCgpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKCgpkZWYgZ2VuX2NsYXNzX2RhdGEoCiAgICAgICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICAgICAgbl9zYW1wbGVzOiBpbnQsCiAgICAgICAgbV9mZWF0dXJlczogaW50LAogICAgICAgIGtfY2xhc3NlczogaW50LAogICAgICAgIGhlYWRlcjogT3B0aW9uYWxbTGlzdFtzdHJdXSwKICAgICAgICBsYWJlbF9jb2x1bW46IE9wdGlvbmFsW3N0cl0gPSAibGFiZWxzIiwKICAgICAgICB3ZWlnaHQ6IGZsb2F0ID0gMC41LAogICAgICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gMSwKICAgICAgICBrZXk6IHN0ciA9ICJjbGFzc2lmaWVyLWRhdGEiLAogICAgICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICAgICAgc2tfcGFyYW1zPXt9Cik6CiAgICAiIiJDcmVhdGUgYSBiaW5hcnkgY2xhc3NpZmljYXRpb24gc2FtcGxlIGRhdGFzZXQgYW5kIHNhdmUuCiAgICBJZiBubyBmaWxlbmFtZSBpcyBnaXZlbiBpdCB3aWxsIGRlZmF1bHQgdG86CiAgICAic2ltZGF0YS17bl9zYW1wbGVzfVh7bV9mZWF0dXJlc30ucGFycXVldCIuCgogICAgQWRkaXRpb25hbCBzY2lraXQtbGVhcm4gcGFyYW1ldGVycyBjYW4gYmUgc2V0IHVzaW5nICoqc2tfcGFyYW1zLCBwbGVhc2Ugc2VlIGh0dHBzOi8vc2Npa2l0LWxlYXJuLm9yZy9zdGFibGUvbW9kdWxlcy9nZW5lcmF0ZWQvc2tsZWFybi5kYXRhc2V0cy5tYWtlX2NsYXNzaWZpY2F0aW9uLmh0bWwgZm9yIG1vcmUgZGV0YWlscy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgZnVuY3Rpb24gY29udGV4dAogICAgOnBhcmFtIG5fc2FtcGxlczogICAgIG51bWJlciBvZiByb3dzL3NhbXBsZXMKICAgIDpwYXJhbSBtX2ZlYXR1cmVzOiAgICBudW1iZXIgb2YgY29scy9mZWF0dXJlcwogICAgOnBhcmFtIGtfY2xhc3NlczogICAgIG51bWJlciBvZiBjbGFzc2VzCiAgICA6cGFyYW0gaGVhZGVyOiAgICAgICAgaGVhZGVyIGZvciBmZWF0dXJlcyBhcnJheQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogIGNvbHVtbiBuYW1lIG9mIGdyb3VuZC10cnV0aCBzZXJpZXMKICAgIDpwYXJhbSB3ZWlnaHQ6ICAgICAgICBmcmFjdGlvbiBvZiBzYW1wbGUgbmVnYXRpdmUgdmFsdWUgKGdyb3VuZC10cnV0aD0wKQogICAgOnBhcmFtIHJhbmRvbV9zdGF0ZTogIHJuZyBzZWVkIChzZWUgaHR0cHM6Ly9zY2lraXQtbGVhcm4ub3JnL3N0YWJsZS9nbG9zc2FyeS5odG1sI3Rlcm0tcmFuZG9tLXN0YXRlKQogICAgOnBhcmFtIGtleTogICAgICAgICAgIGtleSBvZiBkYXRhIGluIGFydGlmYWN0IHN0b3JlCiAgICA6cGFyYW0gZmlsZV9leHQ6ICAgICAgKHBxdCkgZXh0ZW5zaW9uIGZvciBwYXJxdWV0IGZpbGUKICAgIDpwYXJhbSBza19wYXJhbXM6ICAgICBhZGRpdGlvbmFsIHBhcmFtZXRlcnMgZm9yIGBza2xlYXJuLmRhdGFzZXRzLm1ha2VfY2xhc3NpZmljYXRpb25gCiAgICAiIiIKICAgIGZlYXR1cmVzLCBsYWJlbHMgPSBtYWtlX2NsYXNzaWZpY2F0aW9uKAogICAgICAgIG5fc2FtcGxlcz1uX3NhbXBsZXMsCiAgICAgICAgbl9mZWF0dXJlcz1tX2ZlYXR1cmVzLAogICAgICAgIHdlaWdodHM9d2VpZ2h0LAogICAgICAgIG5fY2xhc3Nlcz1rX2NsYXNzZXMsCiAgICAgICAgcmFuZG9tX3N0YXRlPXJhbmRvbV9zdGF0ZSwKICAgICAgICAqKnNrX3BhcmFtcykKCiAgICAjIG1ha2UgZGF0YWZyYW1lcywgYWRkIGNvbHVtbiBuYW1lcywgY29uY2F0ZW5hdGUgKFgsIHkpCiAgICBYID0gcGQuRGF0YUZyYW1lKGZlYXR1cmVzKQogICAgaWYgbm90IGhlYWRlcjoKICAgICAgICBYLmNvbHVtbnMgPSBbImZlYXRfIiArIHN0cih4KSBmb3IgeCBpbiByYW5nZShtX2ZlYXR1cmVzKV0KICAgIGVsc2U6CiAgICAgICAgWC5jb2x1bW5zID0gaGVhZGVyCgogICAgeSA9IHBkLkRhdGFGcmFtZShsYWJlbHMsIGNvbHVtbnM9W2xhYmVsX2NvbHVtbl0pCiAgICBkYXRhID0gcGQuY29uY2F0KFtYLCB5XSwgYXhpcz0xKQoKICAgIGNvbnRleHQubG9nX2RhdGFzZXQoa2V5LCBkZj1kYXRhLCBmb3JtYXQ9ZmlsZV9leHQsIGluZGV4PUZhbHNlKQo= code_origin: '' + filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/gen_class_data/gen_class_data.py + command: '' + image: mlrun/mlrun + default_handler: gen_class_data + disable_auto_mount: false kind: job -verbose: false diff --git a/functions/src/gen_class_data/item.yaml b/functions/src/gen_class_data/item.yaml index 30f5cd21c..082b00305 100644 --- a/functions/src/gen_class_data/item.yaml +++ b/functions/src/gen_class_data/item.yaml @@ -11,7 +11,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.0 +mlrunVersion: 1.10.0 name: gen_class_data platformVersion: 3.5.3 spec: @@ -21,4 +21,4 @@ spec: kind: job requirements: [] url: '' -version: 1.3.0 +version: 1.4.0 diff --git a/functions/src/gen_class_data/requirements.txt b/functions/src/gen_class_data/requirements.txt index d7dbe376b..fc53d535f 100644 --- a/functions/src/gen_class_data/requirements.txt +++ b/functions/src/gen_class_data/requirements.txt @@ -1,2 +1,2 @@ pandas -scikit-learn==1.0.2 \ No newline at end of file +scikit-learn~=1.5 \ No newline at end of file diff --git a/functions/src/gen_class_data/test_gen_class_data.py b/functions/src/gen_class_data/test_gen_class_data.py index e06eeb16b..990075dec 100644 --- a/functions/src/gen_class_data/test_gen_class_data.py +++ b/functions/src/gen_class_data/test_gen_class_data.py @@ -36,4 +36,7 @@ def test_gen_class_data(): local=True, artifact_path="./artifacts", ) - assert os.path.isfile(run.status.artifacts[0]['spec']['target_path']), 'dataset is not available' + # In local mode, artifacts are in function-name/iteration subdirectory + # Default key is "classifier-data" (can be overridden in params) + dataset_path = "./artifacts/test-gen-class-data-gen-class-data/0/classifier-data.csv" + assert os.path.isfile(dataset_path), f'dataset is not available at {dataset_path}' diff --git a/functions/src/sklearn_classifier/function.yaml b/functions/src/sklearn_classifier/function.yaml index 205df697d..603922c95 100644 --- a/functions/src/sklearn_classifier/function.yaml +++ b/functions/src/sklearn_classifier/function.yaml @@ -1,10 +1,98 @@ +kind: job spec: - image: mlrun/mlrun - description: train any classifier using scikit-learn's API default_handler: train_model + command: '' + image: mlrun/mlrun entry_points: + get_sample: + has_kwargs: false + has_varargs: false + lineno: 33 + parameters: + - name: dataset + type: DataItem + doc: DataItem containing the dataset + - name: sample + type: int + doc: Number of samples to take. If -1, use all. If < -1, take random sample. + - name: label_column + type: str + doc: Name of the label column + outputs: + - type: Tuple[pd.DataFrame, pd.Series, list] + name: get_sample + doc: Get a sample of the dataset with labels separated. + get_splits: + has_kwargs: false + has_varargs: false + lineno: 56 + parameters: + - name: features + type: DataFrame + doc: Feature DataFrame + - name: labels + type: Series + doc: Labels Series + - name: num_splits + type: int + doc: Number of splits (3 for train/val/test) + - name: test_size + type: float + doc: Proportion for test set + - name: val_size + type: float + doc: Proportion of remaining data for validation + - name: random_state + type: int + doc: Random seed + default: 1 + outputs: + - type: List[Tuple[pd.DataFrame, pd.Series]] + name: get_splits + doc: Split data into train, validation, and test sets. + gen_sklearn_model: + has_kwargs: false + has_varargs: false + lineno: 86 + parameters: + - name: model_pkg_class + type: str + doc: Full class path (e.g., "sklearn.ensemble.RandomForestClassifier") + - name: parameters + type: list + doc: List of (key, value) parameter tuples + outputs: + - type: dict + name: gen_sklearn_model + doc: Generate sklearn model configuration from class name and parameters. + eval_model_v2: + has_kwargs: false + has_varargs: false + lineno: 117 + parameters: + - name: context + type: MLClientCtx + doc: MLRun context + - name: xvalid + type: DataFrame + doc: Validation features + - name: yvalid + type: Series + doc: Validation labels + - name: model + doc: Trained sklearn model + - name: plots_artifact_path + type: str + doc: Path for plots (not used in this simplified version) + default: null + outputs: + - type: dict + name: eval_model_v2 + doc: Evaluate a sklearn classifier model. train_model: + has_kwargs: false has_varargs: false + lineno: 148 parameters: - name: context type: MLClientCtx @@ -28,7 +116,7 @@ spec: type: int doc: Selects the first n rows, or select a sample starting from the first. If negative <-1, select a random sample - default: + default: - name: test_size type: float doc: (0.05) test set size @@ -64,6 +152,8 @@ spec: type: int doc: (1) sklearn rng seed default: 1 + outputs: + - type: None name: train_model doc: 'train a classifier @@ -76,21 +166,17 @@ spec: scalar "results", a "plots" keys with a list of PlotArtifacts, and and "tables" key containing a returned list of TableArtifacts.' - outputs: - - type: None - lineno: 32 - has_kwargs: false - disable_auto_mount: false build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgp3YXJuaW5ncy5zaW1wbGVmaWx0ZXIoYWN0aW9uPSJpZ25vcmUiLCBjYXRlZ29yeT1GdXR1cmVXYXJuaW5nKQoKCmZyb20gY2xvdWRwaWNrbGUgaW1wb3J0IGR1bXBzCmltcG9ydCBwYW5kYXMgYXMgcGQKZnJvbSB0eXBpbmcgaW1wb3J0IExpc3QKZnJvbSBtbHJ1bi5leGVjdXRpb24gaW1wb3J0IE1MQ2xpZW50Q3R4CmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLm1sdXRpbHMuZGF0YSBpbXBvcnQgZ2V0X3NhbXBsZSwgZ2V0X3NwbGl0cwpmcm9tIG1scnVuLm1sdXRpbHMubW9kZWxzIGltcG9ydCBnZW5fc2tsZWFybl9tb2RlbCwgZXZhbF9tb2RlbF92Mgpmcm9tIG1scnVuLnV0aWxzLmhlbHBlcnMgaW1wb3J0IGNyZWF0ZV9jbGFzcwoKCmRlZiB0cmFpbl9tb2RlbCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWxfcGtnX2NsYXNzOiBzdHIsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gImxhYmVscyIsCiAgICBlbmNvZGVfY29sczogTGlzdFtzdHJdID0gW10sCiAgICBzYW1wbGU6IGludCA9IC0xLAogICAgdGVzdF9zaXplOiBmbG9hdCA9IDAuMzAsCiAgICB0cmFpbl92YWxfc3BsaXQ6IGZsb2F0ID0gMC43MCwKICAgIHRlc3Rfc2V0X2tleTogc3RyID0gInRlc3Rfc2V0IiwKICAgIG1vZGVsX2V2YWx1YXRvcj1Ob25lLAogICAgbW9kZWxzX2Rlc3Q6IHN0ciA9ICIiLAogICAgcGxvdHNfZGVzdDogc3RyID0gInBsb3RzIiwKICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICBtb2RlbF9wa2dfZmlsZTogc3RyID0gIiIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDEsCikgLT4gTm9uZToKICAgICIiInRyYWluIGEgY2xhc3NpZmllcgoKICAgIEFuIG9wdGlvbmFsIGN1dG9tIG1vZGVsIGV2YWx1YXRvciBjYW4gYmUgc3VwcGxpZWQgdGhhdCBzaG91bGQgaGF2ZSB0aGUgc2lnbmF0dXJlOgogICAgYG15X2N1c3RvbV9ldmFsdWF0b3IoY29udGV4dCwgeHZhbGlkLCB5dmFsaWQsIG1vZGVsKWAgYW5kIHJldHVybiBhIGRpY3Rpb25hcnkgb2YKICAgIHNjYWxhciAicmVzdWx0cyIsIGEgInBsb3RzIiBrZXlzIHdpdGggYSBsaXN0IG9mIFBsb3RBcnRpZmFjdHMsIGFuZAogICAgYW5kICJ0YWJsZXMiIGtleSBjb250YWluaW5nIGEgcmV0dXJuZWQgbGlzdCBvZiBUYWJsZUFydGlmYWN0cy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgIHRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxfcGtnX2NsYXNzOiAgIHRoZSBtb2RlbCB0byB0cmFpbiwgZS5nLCAic2tsZWFybi5uZXVyYWxfbmV0d29ya3MuTUxQQ2xhc3NpZmllciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9yIGpzb24gbW9kZWwgY29uZmlnCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICgiZGF0YSIpIG5hbWUgb2YgcmF3IGRhdGEgZmlsZQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICBncm91bmQtdHJ1dGggKHkpIGxhYmVscwogICAgOnBhcmFtIGVuY29kZV9jb2xzOiAgICAgICBkaWN0aW9uYXJ5IG9mIG5hbWVzIGFuZCBwcmVmaXhlcyBmb3IgY29sdW1ucyB0aGF0IGFyZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0byBob3QgYmUgZW5jb2RlZC4KICAgIDpwYXJhbSBzYW1wbGU6ICAgICAgICAgICAgU2VsZWN0cyB0aGUgZmlyc3QgbiByb3dzLCBvciBzZWxlY3QgYSBzYW1wbGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RhcnRpbmcgZnJvbSB0aGUgZmlyc3QuIElmIG5lZ2F0aXZlIDwtMSwgc2VsZWN0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGEgcmFuZG9tIHNhbXBsZQogICAgOnBhcmFtIHRlc3Rfc2l6ZTogICAgICAgICAoMC4wNSkgdGVzdCBzZXQgc2l6ZQogICAgOnBhcmFtIHRyYWluX3ZhbF9zcGxpdDogICAoMC43NSkgT25jZSB0aGUgdGVzdCBzZXQgaGFzIGJlZW4gcmVtb3ZlZCB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJhaW5pbmcgc2V0IGdldHMgdGhpcyBwcm9wb3J0aW9uLgogICAgOnBhcmFtIHRlc3Rfc2V0X2tleTogICAgICBrZXkgb2YgaGVsZCBvdXQgZGF0YSBpbiBhcnRpZmFjdCBzdG9yZQogICAgOnBhcmFtIG1vZGVsX2V2YWx1YXRvcjogICAoTm9uZSkgYSBjdXN0b20gbW9kZWwgZXZhbHVhdG9yIGNhbiBiZSBzcGVjaWZpZWQKICAgIDpwYXJhbSBtb2RlbHNfZGVzdDogICAgICAgKCIiKSBtb2RlbHMgc3ViZm9sZGVyIG9uIGFydGlmYWN0IHBhdGgKICAgIDpwYXJhbSBwbG90c19kZXN0OiAgICAgICAgcGxvdCBzdWJmb2xkZXIgb24gYXJ0aWZhY3QgcGF0aAogICAgOnBhcmFtIGZpbGVfZXh0OiAgICAgICAgICAoInBhcnF1ZXQiKSBmb3JtYXQgZm9yIHRlc3Rfc2V0X2tleSBob2xkIG91dCBkYXRhCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICgxKSBza2xlYXJuIHJuZyBzZWVkCgogICAgIiIiCiAgICBtb2RlbHNfZGVzdCA9IG1vZGVsc19kZXN0IG9yICJtb2RlbCIKCiAgICByYXcsIGxhYmVscywgaGVhZGVyID0gZ2V0X3NhbXBsZShkYXRhc2V0LCBzYW1wbGUsIGxhYmVsX2NvbHVtbikKCiAgICBpZiBlbmNvZGVfY29sczoKICAgICAgICByYXcgPSBwZC5nZXRfZHVtbWllcygKICAgICAgICAgICAgcmF3LAogICAgICAgICAgICBjb2x1bW5zPWxpc3QoZW5jb2RlX2NvbHMua2V5cygpKSwKICAgICAgICAgICAgcHJlZml4PWxpc3QoZW5jb2RlX2NvbHMudmFsdWVzKCkpLAogICAgICAgICAgICBkcm9wX2ZpcnN0PVRydWUsCiAgICAgICAgKQoKICAgICh4dHJhaW4sIHl0cmFpbiksICh4dmFsaWQsIHl2YWxpZCksICh4dGVzdCwgeXRlc3QpID0gZ2V0X3NwbGl0cygKICAgICAgICByYXcsIGxhYmVscywgMywgdGVzdF9zaXplLCAxIC0gdHJhaW5fdmFsX3NwbGl0LCByYW5kb21fc3RhdGUKICAgICkKCiAgICB0ZXN0X3NldCA9IHBkLmNvbmNhdChbeHRlc3QsIHl0ZXN0LnRvX2ZyYW1lKCldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgIHRlc3Rfc2V0X2tleSwKICAgICAgICBkZj10ZXN0X3NldCwKICAgICAgICBmb3JtYXQ9ZmlsZV9leHQsCiAgICAgICAgaW5kZXg9RmFsc2UsCiAgICAgICAgbGFiZWxzPXsiZGF0YS10eXBlIjogImhlbGQtb3V0In0sCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1jb250ZXh0LmFydGlmYWN0X3N1YnBhdGgoImRhdGEiKSwKICAgICkKCiAgICBtb2RlbF9jb25maWcgPSBnZW5fc2tsZWFybl9tb2RlbChtb2RlbF9wa2dfY2xhc3MsIGNvbnRleHQucGFyYW1ldGVycy5pdGVtcygpKQoKICAgIG1vZGVsX2NvbmZpZ1siRklUIl0udXBkYXRlKHsiWCI6IHh0cmFpbiwgInkiOiB5dHJhaW4udmFsdWVzfSkKCiAgICBDbGFzc2lmaWVyQ2xhc3MgPSBjcmVhdGVfY2xhc3MobW9kZWxfY29uZmlnWyJNRVRBIl1bImNsYXNzIl0pCgogICAgbW9kZWwgPSBDbGFzc2lmaWVyQ2xhc3MoKiptb2RlbF9jb25maWdbIkNMQVNTIl0pCgogICAgbW9kZWwuZml0KCoqbW9kZWxfY29uZmlnWyJGSVQiXSkKCiAgICBhcnRpZmFjdF9wYXRoID0gY29udGV4dC5hcnRpZmFjdF9zdWJwYXRoKG1vZGVsc19kZXN0KQogICAgcGxvdHNfcGF0aCA9IGNvbnRleHQuYXJ0aWZhY3Rfc3VicGF0aChtb2RlbHNfZGVzdCwgcGxvdHNfZGVzdCkKICAgIGlmIG1vZGVsX2V2YWx1YXRvcjoKICAgICAgICBldmFsX21ldHJpY3MgPSBtb2RlbF9ldmFsdWF0b3IoCiAgICAgICAgICAgIGNvbnRleHQsIHh2YWxpZCwgeXZhbGlkLCBtb2RlbCwgcGxvdHNfYXJ0aWZhY3RfcGF0aD1wbG90c19wYXRoCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICBldmFsX21ldHJpY3MgPSBldmFsX21vZGVsX3YyKAogICAgICAgICAgICBjb250ZXh0LCB4dmFsaWQsIHl2YWxpZCwgbW9kZWwsIHBsb3RzX2FydGlmYWN0X3BhdGg9cGxvdHNfcGF0aAogICAgICAgICkKCiAgICBrd2FyZ3MgPSB7InRyYWluaW5nX3NldCI6IHRlc3Rfc2V0LCAibGFiZWxfY29sdW1uIjogbGFiZWxfY29sdW1ufQogICAgc3BsaXQgPSBtb2RlbF9wa2dfY2xhc3MucnNwbGl0KCIuIiwgMSkKICAgIGlmIHNwbGl0IGFuZCBsZW4oc3BsaXQpID09IDI6CiAgICAgICAga3dhcmdzWyJhbGdvcml0aG0iXSA9IHNwbGl0WzFdCgogICAgaWYgZGF0YXNldC5tZXRhIGFuZCBkYXRhc2V0Lm1ldGEua2luZCA9PSAiRmVhdHVyZVZlY3RvciI6CiAgICAgICAga3dhcmdzWyJmZWF0dXJlX3ZlY3RvciJdID0gZGF0YXNldC5tZXRhLnVyaQoKICAgIGNvbnRleHQuc2V0X2xhYmVsKCJjbGFzcyIsIG1vZGVsX3BrZ19jbGFzcykKICAgIGNvbnRleHQubG9nX21vZGVsKAogICAgICAgICJtb2RlbCIsCiAgICAgICAgYm9keT1kdW1wcyhtb2RlbCksCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1hcnRpZmFjdF9wYXRoLAogICAgICAgIGV4dHJhX2RhdGE9ZXZhbF9tZXRyaWNzLAogICAgICAgIG1vZGVsX2ZpbGU9Im1vZGVsLnBrbCIsCiAgICAgICAgbWV0cmljcz1jb250ZXh0LnJlc3VsdHMsCiAgICAgICAgbGFiZWxzPXsiY2xhc3MiOiBtb2RlbF9wa2dfY2xhc3N9LAogICAgICAgIGZyYW1ld29yaz0ic2tsZWFybiIsCiAgICAgICAgKiprd2FyZ3MKICAgICkK + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgp3YXJuaW5ncy5zaW1wbGVmaWx0ZXIoYWN0aW9uPSJpZ25vcmUiLCBjYXRlZ29yeT1GdXR1cmVXYXJuaW5nKQoKCmZyb20gY2xvdWRwaWNrbGUgaW1wb3J0IGR1bXBzCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IG51bXB5IGFzIG5wCmZyb20gdHlwaW5nIGltcG9ydCBMaXN0LCBUdXBsZQpmcm9tIHNrbGVhcm4ubW9kZWxfc2VsZWN0aW9uIGltcG9ydCB0cmFpbl90ZXN0X3NwbGl0CmZyb20gc2tsZWFybi5tZXRyaWNzIGltcG9ydCBhY2N1cmFjeV9zY29yZSwgcHJlY2lzaW9uX3Njb3JlLCByZWNhbGxfc2NvcmUsIGYxX3Njb3JlCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmRhdGFzdG9yZSBpbXBvcnQgRGF0YUl0ZW0KZnJvbSBtbHJ1bi51dGlscy5oZWxwZXJzIGltcG9ydCBjcmVhdGVfY2xhc3MKCgpkZWYgZ2V0X3NhbXBsZShkYXRhc2V0OiBEYXRhSXRlbSwgc2FtcGxlOiBpbnQsIGxhYmVsX2NvbHVtbjogc3RyKSAtPiBUdXBsZVtwZC5EYXRhRnJhbWUsIHBkLlNlcmllcywgbGlzdF06CiAgICAiIiJHZXQgYSBzYW1wbGUgb2YgdGhlIGRhdGFzZXQgd2l0aCBsYWJlbHMgc2VwYXJhdGVkLgoKICAgIDpwYXJhbSBkYXRhc2V0OiBEYXRhSXRlbSBjb250YWluaW5nIHRoZSBkYXRhc2V0CiAgICA6cGFyYW0gc2FtcGxlOiBOdW1iZXIgb2Ygc2FtcGxlcyB0byB0YWtlLiBJZiAtMSwgdXNlIGFsbC4gSWYgPCAtMSwgdGFrZSByYW5kb20gc2FtcGxlLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogTmFtZSBvZiB0aGUgbGFiZWwgY29sdW1uCiAgICAiIiIKICAgIGRmID0gZGF0YXNldC5hc19kZigpCgogICAgaWYgc2FtcGxlID09IC0xOgogICAgICAgIHNhbXBsZWRfZGYgPSBkZgogICAgZWxpZiBzYW1wbGUgPCAtMToKICAgICAgICBzYW1wbGVkX2RmID0gZGYuc2FtcGxlKG49YWJzKHNhbXBsZSksIHJhbmRvbV9zdGF0ZT0xKQogICAgZWxzZToKICAgICAgICBzYW1wbGVkX2RmID0gZGYuaGVhZChzYW1wbGUpCgogICAgbGFiZWxzID0gc2FtcGxlZF9kZltsYWJlbF9jb2x1bW5dCiAgICBmZWF0dXJlcyA9IHNhbXBsZWRfZGYuZHJvcChsYWJlbF9jb2x1bW4sIGF4aXM9MSkKICAgIGhlYWRlciA9IGxpc3QoZmVhdHVyZXMuY29sdW1ucykKCiAgICByZXR1cm4gZmVhdHVyZXMsIGxhYmVscywgaGVhZGVyCgoKZGVmIGdldF9zcGxpdHMoCiAgICBmZWF0dXJlczogcGQuRGF0YUZyYW1lLAogICAgbGFiZWxzOiBwZC5TZXJpZXMsCiAgICBudW1fc3BsaXRzOiBpbnQsCiAgICB0ZXN0X3NpemU6IGZsb2F0LAogICAgdmFsX3NpemU6IGZsb2F0LAogICAgcmFuZG9tX3N0YXRlOiBpbnQgPSAxCikgLT4gTGlzdFtUdXBsZVtwZC5EYXRhRnJhbWUsIHBkLlNlcmllc11dOgogICAgIiIiU3BsaXQgZGF0YSBpbnRvIHRyYWluLCB2YWxpZGF0aW9uLCBhbmQgdGVzdCBzZXRzLgoKICAgIDpwYXJhbSBmZWF0dXJlczogRmVhdHVyZSBEYXRhRnJhbWUKICAgIDpwYXJhbSBsYWJlbHM6IExhYmVscyBTZXJpZXMKICAgIDpwYXJhbSBudW1fc3BsaXRzOiBOdW1iZXIgb2Ygc3BsaXRzICgzIGZvciB0cmFpbi92YWwvdGVzdCkKICAgIDpwYXJhbSB0ZXN0X3NpemU6IFByb3BvcnRpb24gZm9yIHRlc3Qgc2V0CiAgICA6cGFyYW0gdmFsX3NpemU6IFByb3BvcnRpb24gb2YgcmVtYWluaW5nIGRhdGEgZm9yIHZhbGlkYXRpb24KICAgIDpwYXJhbSByYW5kb21fc3RhdGU6IFJhbmRvbSBzZWVkCiAgICAiIiIKICAgICMgRmlyc3Qgc3BsaXQ6IHNlcGFyYXRlIHRlc3Qgc2V0CiAgICBYX3RlbXAsIFhfdGVzdCwgeV90ZW1wLCB5X3Rlc3QgPSB0cmFpbl90ZXN0X3NwbGl0KAogICAgICAgIGZlYXR1cmVzLCBsYWJlbHMsIHRlc3Rfc2l6ZT10ZXN0X3NpemUsIHJhbmRvbV9zdGF0ZT1yYW5kb21fc3RhdGUKICAgICkKCiAgICAjIFNlY29uZCBzcGxpdDogc2VwYXJhdGUgdHJhaW4gYW5kIHZhbGlkYXRpb24gZnJvbSByZW1haW5pbmcgZGF0YQogICAgWF90cmFpbiwgWF92YWwsIHlfdHJhaW4sIHlfdmFsID0gdHJhaW5fdGVzdF9zcGxpdCgKICAgICAgICBYX3RlbXAsIHlfdGVtcCwgdGVzdF9zaXplPXZhbF9zaXplLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlCiAgICApCgogICAgcmV0dXJuIFsoWF90cmFpbiwgeV90cmFpbiksIChYX3ZhbCwgeV92YWwpLCAoWF90ZXN0LCB5X3Rlc3QpXQoKCmRlZiBnZW5fc2tsZWFybl9tb2RlbChtb2RlbF9wa2dfY2xhc3M6IHN0ciwgcGFyYW1ldGVyczogbGlzdCkgLT4gZGljdDoKICAgICIiIkdlbmVyYXRlIHNrbGVhcm4gbW9kZWwgY29uZmlndXJhdGlvbiBmcm9tIGNsYXNzIG5hbWUgYW5kIHBhcmFtZXRlcnMuCgogICAgOnBhcmFtIG1vZGVsX3BrZ19jbGFzczogRnVsbCBjbGFzcyBwYXRoIChlLmcuLCAic2tsZWFybi5lbnNlbWJsZS5SYW5kb21Gb3Jlc3RDbGFzc2lmaWVyIikKICAgIDpwYXJhbSBwYXJhbWV0ZXJzOiBMaXN0IG9mIChrZXksIHZhbHVlKSBwYXJhbWV0ZXIgdHVwbGVzCiAgICAiIiIKICAgIGNvbmZpZyA9IHsKICAgICAgICAiTUVUQSI6IHsiY2xhc3MiOiBtb2RlbF9wa2dfY2xhc3N9LAogICAgICAgICJDTEFTUyI6IHt9LAogICAgICAgICJGSVQiOiB7fQogICAgfQoKICAgICMgUGFyYW1ldGVycyB0aGF0IHNob3VsZCBub3QgYmUgcGFzc2VkIHRvIHNrbGVhcm4gbW9kZWwKICAgIGV4Y2x1ZGVkX3BhcmFtcyA9IHsKICAgICAgICAnbW9kZWxfcGtnX2NsYXNzJywgJ2RhdGFzZXQnLCAnbGFiZWxfY29sdW1uJywgJ2VuY29kZV9jb2xzJywKICAgICAgICAnc2FtcGxlJywgJ3Rlc3Rfc2l6ZScsICd0cmFpbl92YWxfc3BsaXQnLCAndGVzdF9zZXRfa2V5JywKICAgICAgICAnbW9kZWxfZXZhbHVhdG9yJywgJ21vZGVsc19kZXN0JywgJ3Bsb3RzX2Rlc3QnLCAnZmlsZV9leHQnLAogICAgICAgICdtb2RlbF9wa2dfZmlsZScsICdjb250ZXh0JwogICAgfQoKICAgICMgU2VwYXJhdGUgcGFyYW1ldGVycyBpbnRvIG1vZGVsIGluaXQgcGFyYW1zIGFuZCBmaXQgcGFyYW1zCiAgICBmb3Iga2V5LCB2YWx1ZSBpbiBwYXJhbWV0ZXJzOgogICAgICAgIGlmIGtleSBpbiBbJ1gnLCAneScsICdzYW1wbGVfd2VpZ2h0J106CiAgICAgICAgICAgIGNvbmZpZ1siRklUIl1ba2V5XSA9IHZhbHVlCiAgICAgICAgZWxpZiBrZXkgbm90IGluIGV4Y2x1ZGVkX3BhcmFtczoKICAgICAgICAgICAgIyBPbmx5IGFkZCBwYXJhbWV0ZXJzIHRoYXQgYXJlIG5vdCBmdW5jdGlvbi1zcGVjaWZpYwogICAgICAgICAgICBjb25maWdbIkNMQVNTIl1ba2V5XSA9IHZhbHVlCgogICAgcmV0dXJuIGNvbmZpZwoKCmRlZiBldmFsX21vZGVsX3YyKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICB4dmFsaWQ6IHBkLkRhdGFGcmFtZSwKICAgIHl2YWxpZDogcGQuU2VyaWVzLAogICAgbW9kZWwsCiAgICBwbG90c19hcnRpZmFjdF9wYXRoOiBzdHIgPSBOb25lCikgLT4gZGljdDoKICAgICIiIkV2YWx1YXRlIGEgc2tsZWFybiBjbGFzc2lmaWVyIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiBNTFJ1biBjb250ZXh0CiAgICA6cGFyYW0geHZhbGlkOiBWYWxpZGF0aW9uIGZlYXR1cmVzCiAgICA6cGFyYW0geXZhbGlkOiBWYWxpZGF0aW9uIGxhYmVscwogICAgOnBhcmFtIG1vZGVsOiBUcmFpbmVkIHNrbGVhcm4gbW9kZWwKICAgIDpwYXJhbSBwbG90c19hcnRpZmFjdF9wYXRoOiBQYXRoIGZvciBwbG90cyAobm90IHVzZWQgaW4gdGhpcyBzaW1wbGlmaWVkIHZlcnNpb24pCiAgICAiIiIKICAgIHlfcHJlZCA9IG1vZGVsLnByZWRpY3QoeHZhbGlkKQoKICAgIG1ldHJpY3MgPSB7CiAgICAgICAgImFjY3VyYWN5IjogYWNjdXJhY3lfc2NvcmUoeXZhbGlkLCB5X3ByZWQpLAogICAgICAgICJwcmVjaXNpb24iOiBwcmVjaXNpb25fc2NvcmUoeXZhbGlkLCB5X3ByZWQsIGF2ZXJhZ2U9J3dlaWdodGVkJywgemVyb19kaXZpc2lvbj0wKSwKICAgICAgICAicmVjYWxsIjogcmVjYWxsX3Njb3JlKHl2YWxpZCwgeV9wcmVkLCBhdmVyYWdlPSd3ZWlnaHRlZCcsIHplcm9fZGl2aXNpb249MCksCiAgICAgICAgImYxX3Njb3JlIjogZjFfc2NvcmUoeXZhbGlkLCB5X3ByZWQsIGF2ZXJhZ2U9J3dlaWdodGVkJywgemVyb19kaXZpc2lvbj0wKQogICAgfQoKICAgICMgTG9nIG1ldHJpY3MgdG8gY29udGV4dAogICAgZm9yIGtleSwgdmFsdWUgaW4gbWV0cmljcy5pdGVtcygpOgogICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdChrZXksIHZhbHVlKQoKICAgIHJldHVybiB7fQoKCmRlZiB0cmFpbl9tb2RlbCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWxfcGtnX2NsYXNzOiBzdHIsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gImxhYmVscyIsCiAgICBlbmNvZGVfY29sczogTGlzdFtzdHJdID0gW10sCiAgICBzYW1wbGU6IGludCA9IC0xLAogICAgdGVzdF9zaXplOiBmbG9hdCA9IDAuMzAsCiAgICB0cmFpbl92YWxfc3BsaXQ6IGZsb2F0ID0gMC43MCwKICAgIHRlc3Rfc2V0X2tleTogc3RyID0gInRlc3Rfc2V0IiwKICAgIG1vZGVsX2V2YWx1YXRvcj1Ob25lLAogICAgbW9kZWxzX2Rlc3Q6IHN0ciA9ICIiLAogICAgcGxvdHNfZGVzdDogc3RyID0gInBsb3RzIiwKICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICBtb2RlbF9wa2dfZmlsZTogc3RyID0gIiIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDEsCikgLT4gTm9uZToKICAgICIiInRyYWluIGEgY2xhc3NpZmllcgoKICAgIEFuIG9wdGlvbmFsIGN1dG9tIG1vZGVsIGV2YWx1YXRvciBjYW4gYmUgc3VwcGxpZWQgdGhhdCBzaG91bGQgaGF2ZSB0aGUgc2lnbmF0dXJlOgogICAgYG15X2N1c3RvbV9ldmFsdWF0b3IoY29udGV4dCwgeHZhbGlkLCB5dmFsaWQsIG1vZGVsKWAgYW5kIHJldHVybiBhIGRpY3Rpb25hcnkgb2YKICAgIHNjYWxhciAicmVzdWx0cyIsIGEgInBsb3RzIiBrZXlzIHdpdGggYSBsaXN0IG9mIFBsb3RBcnRpZmFjdHMsIGFuZAogICAgYW5kICJ0YWJsZXMiIGtleSBjb250YWluaW5nIGEgcmV0dXJuZWQgbGlzdCBvZiBUYWJsZUFydGlmYWN0cy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgIHRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxfcGtnX2NsYXNzOiAgIHRoZSBtb2RlbCB0byB0cmFpbiwgZS5nLCAic2tsZWFybi5uZXVyYWxfbmV0d29ya3MuTUxQQ2xhc3NpZmllciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9yIGpzb24gbW9kZWwgY29uZmlnCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICgiZGF0YSIpIG5hbWUgb2YgcmF3IGRhdGEgZmlsZQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICBncm91bmQtdHJ1dGggKHkpIGxhYmVscwogICAgOnBhcmFtIGVuY29kZV9jb2xzOiAgICAgICBkaWN0aW9uYXJ5IG9mIG5hbWVzIGFuZCBwcmVmaXhlcyBmb3IgY29sdW1ucyB0aGF0IGFyZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0byBob3QgYmUgZW5jb2RlZC4KICAgIDpwYXJhbSBzYW1wbGU6ICAgICAgICAgICAgU2VsZWN0cyB0aGUgZmlyc3QgbiByb3dzLCBvciBzZWxlY3QgYSBzYW1wbGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RhcnRpbmcgZnJvbSB0aGUgZmlyc3QuIElmIG5lZ2F0aXZlIDwtMSwgc2VsZWN0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGEgcmFuZG9tIHNhbXBsZQogICAgOnBhcmFtIHRlc3Rfc2l6ZTogICAgICAgICAoMC4wNSkgdGVzdCBzZXQgc2l6ZQogICAgOnBhcmFtIHRyYWluX3ZhbF9zcGxpdDogICAoMC43NSkgT25jZSB0aGUgdGVzdCBzZXQgaGFzIGJlZW4gcmVtb3ZlZCB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJhaW5pbmcgc2V0IGdldHMgdGhpcyBwcm9wb3J0aW9uLgogICAgOnBhcmFtIHRlc3Rfc2V0X2tleTogICAgICBrZXkgb2YgaGVsZCBvdXQgZGF0YSBpbiBhcnRpZmFjdCBzdG9yZQogICAgOnBhcmFtIG1vZGVsX2V2YWx1YXRvcjogICAoTm9uZSkgYSBjdXN0b20gbW9kZWwgZXZhbHVhdG9yIGNhbiBiZSBzcGVjaWZpZWQKICAgIDpwYXJhbSBtb2RlbHNfZGVzdDogICAgICAgKCIiKSBtb2RlbHMgc3ViZm9sZGVyIG9uIGFydGlmYWN0IHBhdGgKICAgIDpwYXJhbSBwbG90c19kZXN0OiAgICAgICAgcGxvdCBzdWJmb2xkZXIgb24gYXJ0aWZhY3QgcGF0aAogICAgOnBhcmFtIGZpbGVfZXh0OiAgICAgICAgICAoInBhcnF1ZXQiKSBmb3JtYXQgZm9yIHRlc3Rfc2V0X2tleSBob2xkIG91dCBkYXRhCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICgxKSBza2xlYXJuIHJuZyBzZWVkCgogICAgIiIiCiAgICBtb2RlbHNfZGVzdCA9IG1vZGVsc19kZXN0IG9yICJtb2RlbCIKCiAgICByYXcsIGxhYmVscywgaGVhZGVyID0gZ2V0X3NhbXBsZShkYXRhc2V0LCBzYW1wbGUsIGxhYmVsX2NvbHVtbikKCiAgICBpZiBlbmNvZGVfY29sczoKICAgICAgICByYXcgPSBwZC5nZXRfZHVtbWllcygKICAgICAgICAgICAgcmF3LAogICAgICAgICAgICBjb2x1bW5zPWxpc3QoZW5jb2RlX2NvbHMua2V5cygpKSwKICAgICAgICAgICAgcHJlZml4PWxpc3QoZW5jb2RlX2NvbHMudmFsdWVzKCkpLAogICAgICAgICAgICBkcm9wX2ZpcnN0PVRydWUsCiAgICAgICAgKQoKICAgICh4dHJhaW4sIHl0cmFpbiksICh4dmFsaWQsIHl2YWxpZCksICh4dGVzdCwgeXRlc3QpID0gZ2V0X3NwbGl0cygKICAgICAgICByYXcsIGxhYmVscywgMywgdGVzdF9zaXplLCAxIC0gdHJhaW5fdmFsX3NwbGl0LCByYW5kb21fc3RhdGUKICAgICkKCiAgICB0ZXN0X3NldCA9IHBkLmNvbmNhdChbeHRlc3QsIHl0ZXN0LnRvX2ZyYW1lKCldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgIHRlc3Rfc2V0X2tleSwKICAgICAgICBkZj10ZXN0X3NldCwKICAgICAgICBmb3JtYXQ9ZmlsZV9leHQsCiAgICAgICAgaW5kZXg9RmFsc2UsCiAgICAgICAgbGFiZWxzPXsiZGF0YS10eXBlIjogImhlbGQtb3V0In0sCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1jb250ZXh0LmFydGlmYWN0X3N1YnBhdGgoImRhdGEiKSwKICAgICkKCiAgICBtb2RlbF9jb25maWcgPSBnZW5fc2tsZWFybl9tb2RlbChtb2RlbF9wa2dfY2xhc3MsIGNvbnRleHQucGFyYW1ldGVycy5pdGVtcygpKQoKICAgIG1vZGVsX2NvbmZpZ1siRklUIl0udXBkYXRlKHsiWCI6IHh0cmFpbiwgInkiOiB5dHJhaW4udmFsdWVzfSkKCiAgICBDbGFzc2lmaWVyQ2xhc3MgPSBjcmVhdGVfY2xhc3MobW9kZWxfY29uZmlnWyJNRVRBIl1bImNsYXNzIl0pCgogICAgbW9kZWwgPSBDbGFzc2lmaWVyQ2xhc3MoKiptb2RlbF9jb25maWdbIkNMQVNTIl0pCgogICAgbW9kZWwuZml0KCoqbW9kZWxfY29uZmlnWyJGSVQiXSkKCiAgICBhcnRpZmFjdF9wYXRoID0gY29udGV4dC5hcnRpZmFjdF9zdWJwYXRoKG1vZGVsc19kZXN0KQogICAgcGxvdHNfcGF0aCA9IGNvbnRleHQuYXJ0aWZhY3Rfc3VicGF0aChtb2RlbHNfZGVzdCwgcGxvdHNfZGVzdCkKICAgIGlmIG1vZGVsX2V2YWx1YXRvcjoKICAgICAgICBldmFsX21ldHJpY3MgPSBtb2RlbF9ldmFsdWF0b3IoCiAgICAgICAgICAgIGNvbnRleHQsIHh2YWxpZCwgeXZhbGlkLCBtb2RlbCwgcGxvdHNfYXJ0aWZhY3RfcGF0aD1wbG90c19wYXRoCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICBldmFsX21ldHJpY3MgPSBldmFsX21vZGVsX3YyKAogICAgICAgICAgICBjb250ZXh0LCB4dmFsaWQsIHl2YWxpZCwgbW9kZWwsIHBsb3RzX2FydGlmYWN0X3BhdGg9cGxvdHNfcGF0aAogICAgICAgICkKCiAgICBrd2FyZ3MgPSB7InRyYWluaW5nX3NldCI6IHRlc3Rfc2V0LCAibGFiZWxfY29sdW1uIjogbGFiZWxfY29sdW1ufQogICAgc3BsaXQgPSBtb2RlbF9wa2dfY2xhc3MucnNwbGl0KCIuIiwgMSkKICAgIGlmIHNwbGl0IGFuZCBsZW4oc3BsaXQpID09IDI6CiAgICAgICAga3dhcmdzWyJhbGdvcml0aG0iXSA9IHNwbGl0WzFdCgogICAgaWYgZGF0YXNldC5tZXRhIGFuZCBkYXRhc2V0Lm1ldGEua2luZCA9PSAiRmVhdHVyZVZlY3RvciI6CiAgICAgICAga3dhcmdzWyJmZWF0dXJlX3ZlY3RvciJdID0gZGF0YXNldC5tZXRhLnVyaQoKICAgIGNvbnRleHQuc2V0X2xhYmVsKCJjbGFzcyIsIG1vZGVsX3BrZ19jbGFzcykKICAgIGNvbnRleHQubG9nX21vZGVsKAogICAgICAgICJtb2RlbCIsCiAgICAgICAgYm9keT1kdW1wcyhtb2RlbCksCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1hcnRpZmFjdF9wYXRoLAogICAgICAgIGV4dHJhX2RhdGE9ZXZhbF9tZXRyaWNzLAogICAgICAgIG1vZGVsX2ZpbGU9Im1vZGVsLnBrbCIsCiAgICAgICAgbWV0cmljcz1jb250ZXh0LnJlc3VsdHMsCiAgICAgICAgbGFiZWxzPXsiY2xhc3MiOiBtb2RlbF9wa2dfY2xhc3N9LAogICAgICAgIGZyYW1ld29yaz0ic2tsZWFybiIsCiAgICAgICAgKiprd2FyZ3MKICAgICkK origin_filename: '' code_origin: '' - command: '' + disable_auto_mount: false + filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/sklearn_classifier/sklearn_classifier.py + description: train any classifier using scikit-learn's API metadata: - tag: '' name: sklearn-classifier categories: - machine-learning - model-training + tag: '' verbose: false -kind: job diff --git a/functions/src/sklearn_classifier/item.yaml b/functions/src/sklearn_classifier/item.yaml index b9726fb79..4fa374938 100644 --- a/functions/src/sklearn_classifier/item.yaml +++ b/functions/src/sklearn_classifier/item.yaml @@ -13,7 +13,7 @@ labels: framework: sklearn maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.0 +mlrunVersion: 1.10.0 name: sklearn-classifier platformVersion: 3.5.3 spec: @@ -23,5 +23,5 @@ spec: kind: job requirements: [] url: '' -version: 1.2.0 +version: 1.3.0 test_valid: false diff --git a/functions/src/sklearn_classifier/requirements.txt b/functions/src/sklearn_classifier/requirements.txt index 4d9e097f9..97a565a9e 100644 --- a/functions/src/sklearn_classifier/requirements.txt +++ b/functions/src/sklearn_classifier/requirements.txt @@ -1,5 +1,5 @@ pandas -scikit-learn==1.0.2 +scikit-learn~=1.5 matplotlib seaborn scikit-plot diff --git a/functions/src/sklearn_classifier/sklearn_classifier.py b/functions/src/sklearn_classifier/sklearn_classifier.py index 1a73d4045..724f78356 100644 --- a/functions/src/sklearn_classifier/sklearn_classifier.py +++ b/functions/src/sklearn_classifier/sklearn_classifier.py @@ -21,14 +21,130 @@ from cloudpickle import dumps import pandas as pd -from typing import List +import numpy as np +from typing import List, Tuple +from sklearn.model_selection import train_test_split +from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score from mlrun.execution import MLClientCtx from mlrun.datastore import DataItem -from mlrun.mlutils.data import get_sample, get_splits -from mlrun.mlutils.models import gen_sklearn_model, eval_model_v2 from mlrun.utils.helpers import create_class +def get_sample(dataset: DataItem, sample: int, label_column: str) -> Tuple[pd.DataFrame, pd.Series, list]: + """Get a sample of the dataset with labels separated. + + :param dataset: DataItem containing the dataset + :param sample: Number of samples to take. If -1, use all. If < -1, take random sample. + :param label_column: Name of the label column + """ + df = dataset.as_df() + + if sample == -1: + sampled_df = df + elif sample < -1: + sampled_df = df.sample(n=abs(sample), random_state=1) + else: + sampled_df = df.head(sample) + + labels = sampled_df[label_column] + features = sampled_df.drop(label_column, axis=1) + header = list(features.columns) + + return features, labels, header + + +def get_splits( + features: pd.DataFrame, + labels: pd.Series, + num_splits: int, + test_size: float, + val_size: float, + random_state: int = 1 +) -> List[Tuple[pd.DataFrame, pd.Series]]: + """Split data into train, validation, and test sets. + + :param features: Feature DataFrame + :param labels: Labels Series + :param num_splits: Number of splits (3 for train/val/test) + :param test_size: Proportion for test set + :param val_size: Proportion of remaining data for validation + :param random_state: Random seed + """ + # First split: separate test set + X_temp, X_test, y_temp, y_test = train_test_split( + features, labels, test_size=test_size, random_state=random_state + ) + + # Second split: separate train and validation from remaining data + X_train, X_val, y_train, y_val = train_test_split( + X_temp, y_temp, test_size=val_size, random_state=random_state + ) + + return [(X_train, y_train), (X_val, y_val), (X_test, y_test)] + + +def gen_sklearn_model(model_pkg_class: str, parameters: list) -> dict: + """Generate sklearn model configuration from class name and parameters. + + :param model_pkg_class: Full class path (e.g., "sklearn.ensemble.RandomForestClassifier") + :param parameters: List of (key, value) parameter tuples + """ + config = { + "META": {"class": model_pkg_class}, + "CLASS": {}, + "FIT": {} + } + + # Parameters that should not be passed to sklearn model + excluded_params = { + 'model_pkg_class', 'dataset', 'label_column', 'encode_cols', + 'sample', 'test_size', 'train_val_split', 'test_set_key', + 'model_evaluator', 'models_dest', 'plots_dest', 'file_ext', + 'model_pkg_file', 'context' + } + + # Separate parameters into model init params and fit params + for key, value in parameters: + if key in ['X', 'y', 'sample_weight']: + config["FIT"][key] = value + elif key not in excluded_params: + # Only add parameters that are not function-specific + config["CLASS"][key] = value + + return config + + +def eval_model_v2( + context: MLClientCtx, + xvalid: pd.DataFrame, + yvalid: pd.Series, + model, + plots_artifact_path: str = None +) -> dict: + """Evaluate a sklearn classifier model. + + :param context: MLRun context + :param xvalid: Validation features + :param yvalid: Validation labels + :param model: Trained sklearn model + :param plots_artifact_path: Path for plots (not used in this simplified version) + """ + y_pred = model.predict(xvalid) + + metrics = { + "accuracy": accuracy_score(yvalid, y_pred), + "precision": precision_score(yvalid, y_pred, average='weighted', zero_division=0), + "recall": recall_score(yvalid, y_pred, average='weighted', zero_division=0), + "f1_score": f1_score(yvalid, y_pred, average='weighted', zero_division=0) + } + + # Log metrics to context + for key, value in metrics.items(): + context.log_result(key, value) + + return {} + + def train_model( context: MLClientCtx, model_pkg_class: str, diff --git a/functions/src/sklearn_classifier/test_sklearn_classifier.py b/functions/src/sklearn_classifier/test_sklearn_classifier.py index 5c29e85b3..78afd623b 100644 --- a/functions/src/sklearn_classifier/test_sklearn_classifier.py +++ b/functions/src/sklearn_classifier/test_sklearn_classifier.py @@ -38,19 +38,29 @@ def test_import_sklearn_classifier(): params = {"model_pkg_class": "sklearn.ensemble.RandomForestClassifier", "label_column": "labels"} + # In local mode, artifacts are in function-name/iteration subdirectory + dataset_path = "./artifacts/gen-class-data-gen-class-data/0/classifier-data.csv" + assert os.path.exists(dataset_path), f"Dataset not found at {dataset_path}" + train_run = fn.run(params=params, - inputs={"dataset": acquire_run.status.artifacts[0]['spec']['target_path']}, + inputs={"dataset": dataset_path}, local=True, - artifact_path="./") + artifact_path="./artifacts") + + # Check that the run completed successfully + assert train_run.status.state == "completed", f"Run failed with state: {train_run.status.state}" + + # In local mode, check if model metrics were logged + assert "accuracy" in train_run.status.results or len(train_run.status.results) > 0, \ + "No metrics were logged" - for artifact in train_run.status.artifacts: - if artifact['kind'] == 'model': - assert os.path.exists(artifact['spec']['target_path']), 'Could not find model dir' - break + # In local mode, the model is saved to artifacts/model/function-name/iteration/model/model.pkl + model_path = "./artifacts/model/sklearn-classifier-train-model/0/model/model.pkl" + assert os.path.exists(model_path), f'Could not find model file at {model_path}' - assert os.path.exists(train_run.status.artifacts[0]['spec']['target_path']) - model = pickle.load(open(artifact['spec']['target_path'] + artifact['spec']['model_file'], 'rb')) - df = pd.read_csv(acquire_run.status.artifacts[0]['spec']['target_path']) + # Load the model and verify it can make predictions + model = pickle.load(open(model_path, 'rb')) + df = pd.read_csv(dataset_path) x = df.drop(['labels'], axis=1).iloc[0:1] y_true = df['labels'][0] y_pred = model.predict_proba(x).argmax() From 613d020f0136fb3b23dfb12e2fbd39b5822896e0 Mon Sep 17 00:00:00 2001 From: tomerbv Date: Mon, 19 Jan 2026 16:50:09 +0200 Subject: [PATCH 02/15] remove filename --- functions/src/auto_trainer/function.yaml | 1 - functions/src/describe/function.yaml | 1 - functions/src/gen_class_data/function.yaml | 1 - functions/src/sklearn_classifier/function.yaml | 1 - 4 files changed, 4 deletions(-) diff --git a/functions/src/auto_trainer/function.yaml b/functions/src/auto_trainer/function.yaml index 3020b6521..e560064c3 100644 --- a/functions/src/auto_trainer/function.yaml +++ b/functions/src/auto_trainer/function.yaml @@ -8,7 +8,6 @@ spec: default_handler: train description: Automatic train, evaluate and predict functions for the ML frameworks - Scikit-Learn, XGBoost and LightGBM. - filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/auto_trainer/auto_trainer.py command: '' disable_auto_mount: false entry_points: diff --git a/functions/src/describe/function.yaml b/functions/src/describe/function.yaml index 7116fae92..1c254c3c4 100644 --- a/functions/src/describe/function.yaml +++ b/functions/src/describe/function.yaml @@ -84,7 +84,6 @@ spec: - type: None name: analyze lineno: 46 - filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/describe/describe.py build: origin_filename: '' code_origin: '' diff --git a/functions/src/gen_class_data/function.yaml b/functions/src/gen_class_data/function.yaml index fde89341e..fa802964e 100644 --- a/functions/src/gen_class_data/function.yaml +++ b/functions/src/gen_class_data/function.yaml @@ -65,7 +65,6 @@ spec: origin_filename: '' functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKaW1wb3J0IHBhbmRhcyBhcyBwZApmcm9tIHR5cGluZyBpbXBvcnQgT3B0aW9uYWwsIExpc3QKZnJvbSBza2xlYXJuLmRhdGFzZXRzIGltcG9ydCBtYWtlX2NsYXNzaWZpY2F0aW9uCgpmcm9tIG1scnVuLmV4ZWN1dGlvbiBpbXBvcnQgTUxDbGllbnRDdHgKCgpkZWYgZ2VuX2NsYXNzX2RhdGEoCiAgICAgICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICAgICAgbl9zYW1wbGVzOiBpbnQsCiAgICAgICAgbV9mZWF0dXJlczogaW50LAogICAgICAgIGtfY2xhc3NlczogaW50LAogICAgICAgIGhlYWRlcjogT3B0aW9uYWxbTGlzdFtzdHJdXSwKICAgICAgICBsYWJlbF9jb2x1bW46IE9wdGlvbmFsW3N0cl0gPSAibGFiZWxzIiwKICAgICAgICB3ZWlnaHQ6IGZsb2F0ID0gMC41LAogICAgICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gMSwKICAgICAgICBrZXk6IHN0ciA9ICJjbGFzc2lmaWVyLWRhdGEiLAogICAgICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICAgICAgc2tfcGFyYW1zPXt9Cik6CiAgICAiIiJDcmVhdGUgYSBiaW5hcnkgY2xhc3NpZmljYXRpb24gc2FtcGxlIGRhdGFzZXQgYW5kIHNhdmUuCiAgICBJZiBubyBmaWxlbmFtZSBpcyBnaXZlbiBpdCB3aWxsIGRlZmF1bHQgdG86CiAgICAic2ltZGF0YS17bl9zYW1wbGVzfVh7bV9mZWF0dXJlc30ucGFycXVldCIuCgogICAgQWRkaXRpb25hbCBzY2lraXQtbGVhcm4gcGFyYW1ldGVycyBjYW4gYmUgc2V0IHVzaW5nICoqc2tfcGFyYW1zLCBwbGVhc2Ugc2VlIGh0dHBzOi8vc2Npa2l0LWxlYXJuLm9yZy9zdGFibGUvbW9kdWxlcy9nZW5lcmF0ZWQvc2tsZWFybi5kYXRhc2V0cy5tYWtlX2NsYXNzaWZpY2F0aW9uLmh0bWwgZm9yIG1vcmUgZGV0YWlscy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgZnVuY3Rpb24gY29udGV4dAogICAgOnBhcmFtIG5fc2FtcGxlczogICAgIG51bWJlciBvZiByb3dzL3NhbXBsZXMKICAgIDpwYXJhbSBtX2ZlYXR1cmVzOiAgICBudW1iZXIgb2YgY29scy9mZWF0dXJlcwogICAgOnBhcmFtIGtfY2xhc3NlczogICAgIG51bWJlciBvZiBjbGFzc2VzCiAgICA6cGFyYW0gaGVhZGVyOiAgICAgICAgaGVhZGVyIGZvciBmZWF0dXJlcyBhcnJheQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogIGNvbHVtbiBuYW1lIG9mIGdyb3VuZC10cnV0aCBzZXJpZXMKICAgIDpwYXJhbSB3ZWlnaHQ6ICAgICAgICBmcmFjdGlvbiBvZiBzYW1wbGUgbmVnYXRpdmUgdmFsdWUgKGdyb3VuZC10cnV0aD0wKQogICAgOnBhcmFtIHJhbmRvbV9zdGF0ZTogIHJuZyBzZWVkIChzZWUgaHR0cHM6Ly9zY2lraXQtbGVhcm4ub3JnL3N0YWJsZS9nbG9zc2FyeS5odG1sI3Rlcm0tcmFuZG9tLXN0YXRlKQogICAgOnBhcmFtIGtleTogICAgICAgICAgIGtleSBvZiBkYXRhIGluIGFydGlmYWN0IHN0b3JlCiAgICA6cGFyYW0gZmlsZV9leHQ6ICAgICAgKHBxdCkgZXh0ZW5zaW9uIGZvciBwYXJxdWV0IGZpbGUKICAgIDpwYXJhbSBza19wYXJhbXM6ICAgICBhZGRpdGlvbmFsIHBhcmFtZXRlcnMgZm9yIGBza2xlYXJuLmRhdGFzZXRzLm1ha2VfY2xhc3NpZmljYXRpb25gCiAgICAiIiIKICAgIGZlYXR1cmVzLCBsYWJlbHMgPSBtYWtlX2NsYXNzaWZpY2F0aW9uKAogICAgICAgIG5fc2FtcGxlcz1uX3NhbXBsZXMsCiAgICAgICAgbl9mZWF0dXJlcz1tX2ZlYXR1cmVzLAogICAgICAgIHdlaWdodHM9d2VpZ2h0LAogICAgICAgIG5fY2xhc3Nlcz1rX2NsYXNzZXMsCiAgICAgICAgcmFuZG9tX3N0YXRlPXJhbmRvbV9zdGF0ZSwKICAgICAgICAqKnNrX3BhcmFtcykKCiAgICAjIG1ha2UgZGF0YWZyYW1lcywgYWRkIGNvbHVtbiBuYW1lcywgY29uY2F0ZW5hdGUgKFgsIHkpCiAgICBYID0gcGQuRGF0YUZyYW1lKGZlYXR1cmVzKQogICAgaWYgbm90IGhlYWRlcjoKICAgICAgICBYLmNvbHVtbnMgPSBbImZlYXRfIiArIHN0cih4KSBmb3IgeCBpbiByYW5nZShtX2ZlYXR1cmVzKV0KICAgIGVsc2U6CiAgICAgICAgWC5jb2x1bW5zID0gaGVhZGVyCgogICAgeSA9IHBkLkRhdGFGcmFtZShsYWJlbHMsIGNvbHVtbnM9W2xhYmVsX2NvbHVtbl0pCiAgICBkYXRhID0gcGQuY29uY2F0KFtYLCB5XSwgYXhpcz0xKQoKICAgIGNvbnRleHQubG9nX2RhdGFzZXQoa2V5LCBkZj1kYXRhLCBmb3JtYXQ9ZmlsZV9leHQsIGluZGV4PUZhbHNlKQo= code_origin: '' - filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/gen_class_data/gen_class_data.py command: '' image: mlrun/mlrun default_handler: gen_class_data diff --git a/functions/src/sklearn_classifier/function.yaml b/functions/src/sklearn_classifier/function.yaml index 603922c95..208497ecc 100644 --- a/functions/src/sklearn_classifier/function.yaml +++ b/functions/src/sklearn_classifier/function.yaml @@ -171,7 +171,6 @@ spec: origin_filename: '' code_origin: '' disable_auto_mount: false - filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/sklearn_classifier/sklearn_classifier.py description: train any classifier using scikit-learn's API metadata: name: sklearn-classifier From 6256dcf6d6794c74e4ac04b288678c232279de98 Mon Sep 17 00:00:00 2001 From: tomerbv Date: Mon, 19 Jan 2026 17:37:10 +0200 Subject: [PATCH 03/15] remove numpy import --- functions/src/sklearn_classifier/sklearn_classifier.py | 1 - 1 file changed, 1 deletion(-) diff --git a/functions/src/sklearn_classifier/sklearn_classifier.py b/functions/src/sklearn_classifier/sklearn_classifier.py index 724f78356..2fc60a102 100644 --- a/functions/src/sklearn_classifier/sklearn_classifier.py +++ b/functions/src/sklearn_classifier/sklearn_classifier.py @@ -21,7 +21,6 @@ from cloudpickle import dumps import pandas as pd -import numpy as np from typing import List, Tuple from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score From 227cc95c7e61b29c0c5a505f794cc14bb63efdbc Mon Sep 17 00:00:00 2001 From: tomerbv Date: Tue, 20 Jan 2026 13:57:27 +0200 Subject: [PATCH 04/15] revert sklearn.metrics monkey patch fix _get_dataframe to handle list/dict before accessing artifact_url added feature name preservation logic in predict function --- functions/src/auto_trainer/auto_trainer.py | 81 ++++++++++++------- functions/src/auto_trainer/function.yaml | 71 ++++++++-------- .../src/auto_trainer/test_auto_trainer.py | 31 ------- 3 files changed, 86 insertions(+), 97 deletions(-) diff --git a/functions/src/auto_trainer/auto_trainer.py b/functions/src/auto_trainer/auto_trainer.py index 7b4764700..ab2c6ee88 100755 --- a/functions/src/auto_trainer/auto_trainer.py +++ b/functions/src/auto_trainer/auto_trainer.py @@ -67,30 +67,14 @@ def _get_dataframe( Classification tasks. :param drop_columns: str/int or a list of strings/ints that represent the column names/indices to drop. """ - store_uri_prefix, _ = mlrun.datastore.parse_store_uri(dataset.artifact_url) - - # Getting the dataset: - if mlrun.utils.StorePrefix.FeatureVector == store_uri_prefix: - label_columns = label_columns or dataset.meta.status.label_column - context.logger.info(f"label columns: {label_columns}") - # FeatureVector case: - try: - fv = mlrun.datastore.get_store_resource(dataset.artifact_url) - dataset = fv.get_offline_features(drop_columns=drop_columns).to_dataframe() - except AttributeError: - # Leave here for backwards compatibility - dataset = fs.get_offline_features( - dataset.meta.uri, drop_columns=drop_columns - ).to_dataframe() - - elif not label_columns: - context.logger.info( - "label_columns not provided, mandatory when dataset is not a FeatureVector" - ) - raise ValueError - - elif isinstance(dataset, (list, dict)): + # Check if dataset is list/dict first (before trying to access artifact_url) + if isinstance(dataset, (list, dict)): # list/dict case: + if not label_columns: + context.logger.info( + "label_columns not provided, mandatory when dataset is not a FeatureVector" + ) + raise ValueError dataset = pd.DataFrame(dataset) # Checking if drop_columns provided by integer type: if drop_columns: @@ -103,17 +87,38 @@ def _get_dataframe( ) raise ValueError dataset.drop(drop_columns, axis=1, inplace=True) - else: - # simple URL case: - dataset = dataset.as_df() - if drop_columns: - if all(col in dataset for col in drop_columns): - dataset = dataset.drop(drop_columns, axis=1) - else: + # Dataset is a DataItem with artifact_url (URI or FeatureVector) + store_uri_prefix, _ = mlrun.datastore.parse_store_uri(dataset.artifact_url) + + # Getting the dataset: + if mlrun.utils.StorePrefix.FeatureVector == store_uri_prefix: + label_columns = label_columns or dataset.meta.status.label_column + context.logger.info(f"label columns: {label_columns}") + # FeatureVector case: + try: + fv = mlrun.datastore.get_store_resource(dataset.artifact_url) + dataset = fv.get_offline_features(drop_columns=drop_columns).to_dataframe() + except AttributeError: + # Leave here for backwards compatibility + dataset = fs.get_offline_features( + dataset.meta.uri, drop_columns=drop_columns + ).to_dataframe() + else: + # simple URL case: + if not label_columns: context.logger.info( - "not all of the columns to drop in the dataset, drop columns process skipped" + "label_columns not provided, mandatory when dataset is not a FeatureVector" ) + raise ValueError + dataset = dataset.as_df() + if drop_columns: + if all(col in dataset for col in drop_columns): + dataset = dataset.drop(drop_columns, axis=1) + else: + context.logger.info( + "not all of the columns to drop in the dataset, drop columns process skipped" + ) return dataset, label_columns @@ -361,6 +366,20 @@ def predict( # loading the model, and getting the model handler: model_handler = AutoMLRun.load_model(model_path=model, context=context) + # Fix feature names for models that require them (e.g., XGBoost) + # When dataset comes from a list, pandas assigns default integer column names + # but some models expect specific feature names they were trained with + if hasattr(model_handler.model, 'feature_names_in_'): + expected_features = model_handler.model.feature_names_in_ + if len(dataset.columns) == len(expected_features): + # Only rename if the number of columns matches + # This handles the case where a list was converted to DataFrame with default column names + if not all(col == feat for col, feat in zip(dataset.columns, expected_features)): + context.logger.info( + f"Renaming dataset columns to match model's expected feature names" + ) + dataset.columns = expected_features + # Dropping label columns if necessary: if not label_columns: label_columns = [] diff --git a/functions/src/auto_trainer/function.yaml b/functions/src/auto_trainer/function.yaml index e560064c3..32e1f67dc 100644 --- a/functions/src/auto_trainer/function.yaml +++ b/functions/src/auto_trainer/function.yaml @@ -1,30 +1,15 @@ -kind: job +metadata: + name: auto-trainer + categories: + - machine-learning + - model-training + tag: '' spec: - build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICBzdG9yZV91cmlfcHJlZml4LCBfID0gbWxydW4uZGF0YXN0b3JlLnBhcnNlX3N0b3JlX3VyaShkYXRhc2V0LmFydGlmYWN0X3VybCkKCiAgICAjIEdldHRpbmcgdGhlIGRhdGFzZXQ6CiAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IGxhYmVsX2NvbHVtbnMgb3IgZGF0YXNldC5tZXRhLnN0YXR1cy5sYWJlbF9jb2x1bW4KICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYibGFiZWwgY29sdW1uczoge2xhYmVsX2NvbHVtbnN9IikKICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICB0cnk6CiAgICAgICAgICAgIGZ2ID0gbWxydW4uZGF0YXN0b3JlLmdldF9zdG9yZV9yZXNvdXJjZShkYXRhc2V0LmFydGlmYWN0X3VybCkKICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZXhjZXB0IEF0dHJpYnV0ZUVycm9yOgogICAgICAgICAgICAjIExlYXZlIGhlcmUgZm9yIGJhY2t3YXJkcyBjb21wYXRpYmlsaXR5CiAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgIGRhdGFzZXQubWV0YS51cmksIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMKICAgICAgICAgICAgKS50b19kYXRhZnJhbWUoKQoKICAgIGVsaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgImxhYmVsX2NvbHVtbnMgbm90IHByb3ZpZGVkLCBtYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBlbGlmIGlzaW5zdGFuY2UoZGF0YXNldCwgKGxpc3QsIGRpY3QpKToKICAgICAgICAjIGxpc3QvZGljdCBjYXNlOgogICAgICAgIGRhdGFzZXQgPSBwZC5EYXRhRnJhbWUoZGF0YXNldCkKICAgICAgICAjIENoZWNraW5nIGlmIGRyb3BfY29sdW1ucyBwcm92aWRlZCBieSBpbnRlZ2VyIHR5cGU6CiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICBpZiBpc2luc3RhbmNlKGRyb3BfY29sdW1ucywgc3RyKSBvciAoCiAgICAgICAgICAgICAgICBpc2luc3RhbmNlKGRyb3BfY29sdW1ucywgbGlzdCkKICAgICAgICAgICAgICAgIGFuZCBhbnkoaXNpbnN0YW5jZShjb2wsIHN0cikgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpCiAgICAgICAgICAgICk6CiAgICAgICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIG11c3QgYmUgYW4gaW50ZWdlci9saXN0IG9mIGludGVnZXJzIGlmIG5vdCBwcm92aWRlZCB3aXRoIGEgVVJJL0ZlYXR1cmVWZWN0b3IgZGF0YXNldCIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xLCBpbnBsYWNlPVRydWUpCgogICAgZWxzZToKICAgICAgICAjIHNpbXBsZSBVUkwgY2FzZToKICAgICAgICBkYXRhc2V0ID0gZGF0YXNldC5hc19kZigpCiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSkKICAgICAgICAgICAgZWxzZToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgIm5vdCBhbGwgb2YgdGhlIGNvbHVtbnMgdG8gZHJvcCBpbiB0aGUgZGF0YXNldCwgZHJvcCBjb2x1bW5zIHByb2Nlc3Mgc2tpcHBlZCIKICAgICAgICAgICAgICAgICkKCiAgICByZXR1cm4gZGF0YXNldCwgbGFiZWxfY29sdW1ucwoKCmRlZiB0cmFpbigKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBtb2RlbF9jbGFzczogc3RyLAogICAgbGFiZWxfY29sdW1uczogT3B0aW9uYWxbVW5pb25bc3RyLCBMaXN0W3N0cl1dXSA9IE5vbmUsCiAgICBkcm9wX2NvbHVtbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBtb2RlbF9uYW1lOiBzdHIgPSAibW9kZWwiLAogICAgdGFnOiBzdHIgPSAiIiwKICAgIHNhbXBsZV9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRlc3Rfc2V0OiBEYXRhSXRlbSA9IE5vbmUsCiAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemU6IGZsb2F0ID0gTm9uZSwKICAgIHJhbmRvbV9zdGF0ZTogaW50ID0gTm9uZSwKICAgIGxhYmVsczogZGljdCA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgVHJhaW5pbmcgYSBtb2RlbCB3aXRoIHRoZSBnaXZlbiBkYXRhc2V0LgoKICAgIGV4YW1wbGU6OgoKICAgICAgICBpbXBvcnQgbWxydW4KICAgICAgICBwcm9qZWN0ID0gbWxydW4uZ2V0X29yX2NyZWF0ZV9wcm9qZWN0KCJteS1wcm9qZWN0IikKICAgICAgICBwcm9qZWN0LnNldF9mdW5jdGlvbigiaHViOi8vYXV0b190cmFpbmVyIiwgInRyYWluIikKICAgICAgICB0cmFpbmVyX3J1biA9IHByb2plY3QucnVuKAogICAgICAgICAgICBuYW1lPSJ0cmFpbiIsCiAgICAgICAgICAgIGhhbmRsZXI9InRyYWluIiwKICAgICAgICAgICAgaW5wdXRzPXsiZGF0YXNldCI6ICIuL3BhdGgvdG8vZGF0YXNldC5jc3YifSwKICAgICAgICAgICAgcGFyYW1zPXsKICAgICAgICAgICAgICAgICJtb2RlbF9jbGFzcyI6ICJza2xlYXJuLmxpbmVhcl9tb2RlbC5Mb2dpc3RpY1JlZ3Jlc3Npb24iLAogICAgICAgICAgICAgICAgImxhYmVsX2NvbHVtbnMiOiAibGFiZWwiLAogICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyI6ICJpZCIsCiAgICAgICAgICAgICAgICAibW9kZWxfbmFtZSI6ICJteS1tb2RlbCIsCiAgICAgICAgICAgICAgICAidGFnIjogInYxLjAuMCIsCiAgICAgICAgICAgICAgICAic2FtcGxlX3NldCI6ICIuL3BhdGgvdG8vc2FtcGxlX3NldC5jc3YiLAogICAgICAgICAgICAgICAgInRlc3Rfc2V0IjogIi4vcGF0aC90by90ZXN0X3NldC5jc3YiLAogICAgICAgICAgICAgICAgIkNMQVNTX3NvbHZlciI6ICJsaWJsaW5lYXIiLAogICAgICAgICAgICB9LAogICAgICAgICkKCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQKICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3RvcgogICAgOnBhcmFtIG1vZGVsX2NsYXNzOiAgICAgICAgICAgICBUaGUgY2xhc3Mgb2YgdGhlIG1vZGVsLCBlLmcuIGBza2xlYXJuLmxpbmVhcl9tb2RlbC5Mb2dpc3RpY1JlZ3Jlc3Npb25gCiAgICA6cGFyYW0gbGFiZWxfY29sdW1uczogICAgICAgICAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQ2xhc3NpZmljYXRpb24gdGFza3MuIE1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyIG9yIGEgbGlzdCBvZiBzdHJpbmdzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW5zIHRvIGRyb3AKICAgIDpwYXJhbSBtb2RlbF9uYW1lOiAgICAgICAgICAgICAgVGhlIG1vZGVsJ3MgbmFtZSB0byB1c2UgZm9yIHN0b3JpbmcgdGhlIG1vZGVsIGFydGlmYWN0LCBkZWZhdWx0IHRvICdtb2RlbCcKICAgIDpwYXJhbSB0YWc6ICAgICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsJ3MgdGFnIHRvIGxvZyB3aXRoCiAgICA6cGFyYW0gc2FtcGxlX3NldDogICAgICAgICAgICAgIEEgc2FtcGxlIHNldCBvZiBpbnB1dHMgZm9yIHRoZSBtb2RlbCBmb3IgbG9nZ2luZyBpdHMgc3RhdHMgYWxvbmcgdGhlIG1vZGVsIGluIGZhdm91cgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBvZiBtb2RlbCBtb25pdG9yaW5nLiBDYW4gYmUgZWl0aGVyIGEgVVJJIG9yIGEgRmVhdHVyZVZlY3RvcgogICAgOnBhcmFtIHRlc3Rfc2V0OiAgICAgICAgICAgICAgICBUaGUgdGVzdCBzZXQgdG8gdHJhaW4gdGhlIG1vZGVsIHdpdGguCiAgICA6cGFyYW0gdHJhaW5fdGVzdF9zcGxpdF9zaXplOiAgIGlmIHRlc3Rfc2V0IHdhcyBwcm92aWRlZCB0aGVuIHRoaXMgYXJndW1lbnQgaXMgaWdub3JlZC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgU2hvdWxkIGJlIGJldHdlZW4gMC4wIGFuZCAxLjAgYW5kIHJlcHJlc2VudCB0aGUgcHJvcG9ydGlvbiBvZiB0aGUgZGF0YXNldCB0byBpbmNsdWRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGluIHRoZSB0ZXN0IHNwbGl0LiBUaGUgc2l6ZSBvZiB0aGUgVHJhaW5pbmcgc2V0IGlzIHNldCB0byB0aGUgY29tcGxlbWVudCBvZiB0aGlzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHZhbHVlLiBEZWZhdWx0ID0gMC4yCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICAgICAgIFJlbGV2YW50IG9ubHkgd2hlbiB1c2luZyB0cmFpbl90ZXN0X3NwbGl0X3NpemUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIEEgcmFuZG9tIHN0YXRlIHNlZWQgdG8gc2h1ZmZsZSB0aGUgZGF0YS4gRm9yIG1vcmUgaW5mb3JtYXRpb24sIHNlZToKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaHR0cHM6Ly9zY2lraXQtbGVhcm4ub3JnL3N0YWJsZS9nbG9zc2FyeS5odG1sI3Rlcm0tcmFuZG9tX3N0YXRlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIE5vdGljZSB0aGF0IGhlcmUgd2Ugb25seSBwYXNzIGludGVnZXIgdmFsdWVzLgogICAgOnBhcmFtIGxhYmVsczogICAgICAgICAgICAgICAgICBMYWJlbHMgdG8gbG9nIHdpdGggdGhlIG1vZGVsCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHdpdGggcHJlZml4ZXMsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHRoYXQgd2lsbCBiZSBwYXJzZWQgYW5kIHBhc3NlZCB0byB0aGUgcmVsZXZhbnQgZnVuY3Rpb24sIGJ5IHRoZSBmb2xsb3dpbmcgcHJlZml4ZXM6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYENMQVNTX2AgLSBmb3IgdGhlIG1vZGVsIGNsYXNzIGFyZ3VtZW50cwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAtIGBGSVRfYCAtIGZvciB0aGUgYGZpdGAgZnVuY3Rpb24gYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYFRSQUlOX2AgLSBmb3IgdGhlIGB0cmFpbmAgZnVuY3Rpb24gKGluIHhnYiBvciBsZ2JtIHRyYWluIGZ1bmN0aW9uIC0gZnV0dXJlKQoKICAgICIiIgogICAgIyBWYWxpZGF0ZSBpbnB1dHM6CiAgICAjIENoZWNrIGlmIGV4YWN0bHkgb25lIG9mIHRoZW0gaXMgc3VwcGxpZWQ6CiAgICBpZiB0ZXN0X3NldCBpcyBOb25lOgogICAgICAgIGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSBpcyBOb25lOgogICAgICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAgICAgInRlc3Rfc2V0IG9yIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSBhcmUgbm90IHByb3ZpZGVkLCBzZXR0aW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB0byAwLjIiCiAgICAgICAgICAgICkKICAgICAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gMC4yCgogICAgZWxpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgInRlc3Rfc2V0IHByb3ZpZGVkLCBpZ25vcmluZyBnaXZlbiB0cmFpbl90ZXN0X3NwbGl0X3NpemUgdmFsdWUiCiAgICAgICAgKQogICAgICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSA9IE5vbmUKCiAgICAjIEdldCBEYXRhRnJhbWUgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIEdldHRpbmcgdGhlIHNhbXBsZSBzZXQ6CiAgICBpZiBzYW1wbGVfc2V0IGlzIE5vbmU6CiAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgZiJTYW1wbGUgc2V0IG5vdCBnaXZlbiwgdXNpbmcgdGhlIHdob2xlIHRyYWluaW5nIHNldCBhcyB0aGUgc2FtcGxlIHNldCIKICAgICAgICApCiAgICAgICAgc2FtcGxlX3NldCA9IGRhdGFzZXQKICAgIGVsc2U6CiAgICAgICAgc2FtcGxlX3NldCwgXyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgICAgIGRhdGFzZXQ9c2FtcGxlX3NldCwKICAgICAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgICAgICkKCiAgICAjIFBhcnNpbmcga3dhcmdzOgogICAgIyBUT0RPOiBVc2UgaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24uCiAgICB0cmFpbl9rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeChzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLlRSQUlOKQogICAgZml0X2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuRklUKQogICAgbW9kZWxfY2xhc3Nfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoCiAgICAgICAgc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5NT0RFTF9DTEFTUwogICAgKQoKICAgICMgQ2hlY2sgaWYgbW9kZWwgb3IgZnVuY3Rpb246CiAgICBpZiBoYXNhdHRyKG1vZGVsX2NsYXNzLCAidHJhaW4iKToKICAgICAgICAjIFRPRE86IE5lZWQgdG8gY2FsbDogbW9kZWwoKSwgYWZ0ZXJ3YXJkcyB0byBzdGFydCB0aGUgdHJhaW4gZnVuY3Rpb24uCiAgICAgICAgIyBtb2RlbCA9IGNyZWF0ZV9mdW5jdGlvbihmInttb2RlbF9jbGFzc30udHJhaW4iKQogICAgICAgIHJhaXNlIE5vdEltcGxlbWVudGVkRXJyb3IKICAgIGVsc2U6CiAgICAgICAgIyBDcmVhdGluZyBtb2RlbCBpbnN0YW5jZToKICAgICAgICBtb2RlbCA9IGNyZWF0ZV9jbGFzcyhtb2RlbF9jbGFzcykoKiptb2RlbF9jbGFzc19rd2FyZ3MpCgogICAgeCA9IGRhdGFzZXQuZHJvcChsYWJlbF9jb2x1bW5zLCBheGlzPTEpCiAgICB5ID0gZGF0YXNldFtsYWJlbF9jb2x1bW5zXQogICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplOgogICAgICAgIHhfdHJhaW4sIHhfdGVzdCwgeV90cmFpbiwgeV90ZXN0ID0gdHJhaW5fdGVzdF9zcGxpdCgKICAgICAgICAgICAgeCwgeSwgdGVzdF9zaXplPXRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSwgcmFuZG9tX3N0YXRlPXJhbmRvbV9zdGF0ZQogICAgICAgICkKICAgIGVsc2U6CiAgICAgICAgeF90cmFpbiwgeV90cmFpbiA9IHgsIHkKCiAgICAgICAgdGVzdF9zZXQgPSB0ZXN0X3NldC5hc19kZigpCiAgICAgICAgaWYgZHJvcF9jb2x1bW5zOgogICAgICAgICAgICB0ZXN0X3NldCA9IGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSkKCiAgICAgICAgeF90ZXN0LCB5X3Rlc3QgPSB0ZXN0X3NldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSksIHRlc3Rfc2V0W2xhYmVsX2NvbHVtbnNdCgogICAgQXV0b01MUnVuLmFwcGx5X21scnVuKAogICAgICAgIG1vZGVsPW1vZGVsLAogICAgICAgIG1vZGVsX25hbWU9bW9kZWxfbmFtZSwKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgdGFnPXRhZywKICAgICAgICBzYW1wbGVfc2V0PXNhbXBsZV9zZXQsCiAgICAgICAgeV9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgdGVzdF9zZXQ9dGVzdF9zZXQsCiAgICAgICAgeF90ZXN0PXhfdGVzdCwKICAgICAgICB5X3Rlc3Q9eV90ZXN0LAogICAgICAgIGFydGlmYWN0cz1jb250ZXh0LmFydGlmYWN0cywKICAgICAgICBsYWJlbHM9bGFiZWxzLAogICAgKQogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmInRyYWluaW5nICd7bW9kZWxfbmFtZX0nIikKICAgIG1vZGVsLmZpdCh4X3RyYWluLCB5X3RyYWluLCAqKmZpdF9rd2FyZ3MpCgoKZGVmIGV2YWx1YXRlKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBtb2RlbDogc3RyLAogICAgZGF0YXNldDogbWxydW4uRGF0YUl0ZW0sCiAgICBkcm9wX2NvbHVtbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBFdmFsdWF0aW5nIGEgbW9kZWwuIEFydGlmYWN0cyBnZW5lcmF0ZWQgYnkgdGhlIE1MSGFuZGxlci4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gbW9kZWw6ICAgICAgICAgICAgICAgICAgIFRoZSBtb2RlbCBTdG9yZSBwYXRoLgogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byBldmFsdWF0ZSB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcC4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIFBhcnNpbmcgbGFiZWxfY29sdW1uczoKICAgIHBhcnNlZF9sYWJlbF9jb2x1bW5zID0gW10KICAgIGlmIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9ICgKICAgICAgICAgICAgbGFiZWxfY29sdW1ucyBpZiBpc2luc3RhbmNlKGxhYmVsX2NvbHVtbnMsIGxpc3QpIGVsc2UgW2xhYmVsX2NvbHVtbnNdCiAgICAgICAgKQogICAgICAgIGZvciBsYyBpbiBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICBpZiBmcy5jb21tb24uZmVhdHVyZV9zZXBhcmF0b3IgaW4gbGM6CiAgICAgICAgICAgICAgICBmZWF0dXJlX3NldF9uYW1lLCBsYWJlbF9uYW1lLCBhbGlhcyA9IGZzLmNvbW1vbi5wYXJzZV9mZWF0dXJlX3N0cmluZyhsYykKICAgICAgICAgICAgICAgIHBhcnNlZF9sYWJlbF9jb2x1bW5zLmFwcGVuZChhbGlhcyBvciBsYWJlbF9uYW1lKQogICAgICAgIGlmIHBhcnNlZF9sYWJlbF9jb2x1bW5zOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gcGFyc2VkX2xhYmVsX2NvbHVtbnMKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCgogICAgIyBMb2FkaW5nIHRoZSBtb2RlbCBhbmQgcHJlZGljdGluZzoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbCgKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQsIG1vZGVsX25hbWU9Im1vZGVsX0xpbmVhclJlZ3Jlc3Npb24iCiAgICApCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4obW9kZWxfaGFuZGxlci5tb2RlbCwgeV90ZXN0PXksIG1vZGVsX3BhdGg9bW9kZWwpCgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmImV2YWx1YXRpbmcgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoeCwgKiprd2FyZ3MpCgoKZGVmIHByZWRpY3QoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIHJlc3VsdF9zZXQ6IE9wdGlvbmFsW3N0cl0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIFByZWRpY3RpbmcgZGF0YXNldCBieSBhIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIHByZWRpY3QgdGhlIG1vZGVsIG9uLiBDYW4gYmUgZWl0aGVyIGEgVVJJLCBhIEZlYXR1cmVWZWN0b3Igb3IgYQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBzYW1wbGUgaW4gYSBzaGFwZSBvZiBhIGxpc3QvZGljdC4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgV2hlbiBwYXNzaW5nIGEgc2FtcGxlLCBwYXNzIHRoZSBkYXRhc2V0IGFzIGEgZmllbGQgaW4gYHBhcmFtc2AgaW5zdGVhZCBvZiBgaW5wdXRzYC4KICAgIDpwYXJhbSBkcm9wX2NvbHVtbnM6ICAgICAgICAgICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHRoZSBkYXRhc2V0IGlzIGEgbGlzdC9kaWN0IHRoaXMgcGFyYW1ldGVyIHNob3VsZCBiZSByZXByZXNlbnRlZCBieSBpbnRlZ2Vycy4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIHJlc3VsdF9zZXQ6ICAgICAgICAgICAgICBUaGUgZGIga2V5IHRvIHNldCBuYW1lIG9mIHRoZSBwcmVkaWN0aW9uIHJlc3VsdCBhbmQgdGhlIGZpbGVuYW1lLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0IHRvICdwcmVkaWN0aW9uJy4KICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgdG8gdGhlIHByZWRpY3QgZnVuY3Rpb24KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgKFBSRURJQ1RfIHByZWZpeCBpcyBub3QgcmVxdWlyZWQpLgogICAgIiIiCiAgICAjIEdldCBkYXRhc2V0IGJ5IFVSTCBvciBieSBGZWF0dXJlVmVjdG9yOgogICAgZGF0YXNldCwgbGFiZWxfY29sdW1ucyA9IF9nZXRfZGF0YWZyYW1lKAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICBkYXRhc2V0PWRhdGFzZXQsCiAgICAgICAgbGFiZWxfY29sdW1ucz1sYWJlbF9jb2x1bW5zLAogICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICApCgogICAgIyBsb2FkaW5nIHRoZSBtb2RlbCwgYW5kIGdldHRpbmcgdGhlIG1vZGVsIGhhbmRsZXI6CiAgICBtb2RlbF9oYW5kbGVyID0gQXV0b01MUnVuLmxvYWRfbW9kZWwobW9kZWxfcGF0aD1tb2RlbCwgY29udGV4dD1jb250ZXh0KQoKICAgICMgRHJvcHBpbmcgbGFiZWwgY29sdW1ucyBpZiBuZWNlc3Nhcnk6CiAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW10KICAgIGVsaWYgaXNpbnN0YW5jZShsYWJlbF9jb2x1bW5zLCBzdHIpOgogICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbbGFiZWxfY29sdW1uc10KCiAgICAjIFByZWRpY3Rpbmc6CiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYibWFraW5nIHByZWRpY3Rpb24gYnkgJ3ttb2RlbF9oYW5kbGVyLm1vZGVsX25hbWV9JyIpCiAgICB5X3ByZWQgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLnByZWRpY3QoZGF0YXNldCwgKiprd2FyZ3MpCgogICAgIyBQcmVwYXJpbmcgYW5kIHZhbGlkYXRpbmcgbGFiZWwgY29sdW1ucyBmb3IgdGhlIGRhdGFmcmFtZSBvZiB0aGUgcHJlZGljdGlvbiByZXN1bHQ6CiAgICBudW1fcHJlZGljdGVkID0gMSBpZiBsZW4oeV9wcmVkLnNoYXBlKSA9PSAxIGVsc2UgeV9wcmVkLnNoYXBlWzFdCgogICAgaWYgbnVtX3ByZWRpY3RlZCA+IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBpZiBudW1fcHJlZGljdGVkID09IDE6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBbInByZWRpY3RlZCBsYWJlbHMiXQogICAgICAgIGVsc2U6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMuZXh0ZW5kKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIGYicHJlZGljdGVkX2xhYmVsX3tpICsgMSArIGxlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgICAgICAgICAgICAgZm9yIGkgaW4gcmFuZ2UobnVtX3ByZWRpY3RlZCAtIGxlbihsYWJlbF9jb2x1bW5zKSkKICAgICAgICAgICAgICAgIF0KICAgICAgICAgICAgKQogICAgZWxpZiBudW1fcHJlZGljdGVkIDwgbGVuKGxhYmVsX2NvbHVtbnMpOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIm51bWJlciBvZiBwcmVkaWN0ZWQgbGFiZWxzOiB7bnVtX3ByZWRpY3RlZH0gaXMgc21hbGxlciB0aGFuIG51bWJlciBvZiBsYWJlbCBjb2x1bW5zOiB7bGVuKGxhYmVsX2NvbHVtbnMpfSIKICAgICAgICApCiAgICAgICAgcmFpc2UgVmFsdWVFcnJvcgoKICAgIGFydGlmYWN0X25hbWUgPSByZXN1bHRfc2V0IG9yICJwcmVkaWN0aW9uIgogICAgbGFiZWxzX2luc2lkZV9kZiA9IHNldChsYWJlbF9jb2x1bW5zKSAmIHNldChkYXRhc2V0LmNvbHVtbnMudG9saXN0KCkpCiAgICBpZiBsYWJlbHNfaW5zaWRlX2RmOgogICAgICAgIGNvbnRleHQubG9nZ2VyLmVycm9yKAogICAgICAgICAgICBmIlRoZSBsYWJlbHM6IHtsYWJlbHNfaW5zaWRlX2RmfSBhcmUgYWxyZWFkeSBleGlzdGVkIGluIHRoZSBkYXRhZnJhbWUiCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgIHByZWRfZGYgPSBwZC5jb25jYXQoW2RhdGFzZXQsIHBkLkRhdGFGcmFtZSh5X3ByZWQsIGNvbHVtbnM9bGFiZWxfY29sdW1ucyldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KGFydGlmYWN0X25hbWUsIHByZWRfZGYsIGRiX2tleT1yZXN1bHRfc2V0KQo= - code_origin: '' - origin_filename: '' - image: mlrun/mlrun - default_handler: train - description: Automatic train, evaluate and predict functions for the ML frameworks - - Scikit-Learn, XGBoost and LightGBM. - command: '' - disable_auto_mount: false + filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/auto_trainer/auto_trainer.py entry_points: train: - lineno: 121 - has_varargs: false name: train - doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ - \ project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ - hub://auto_trainer\", \"train\")\n trainer_run = project.run(\n \ - \ name=\"train\",\n handler=\"train\",\n inputs={\"dataset\"\ - : \"./path/to/dataset.csv\"},\n params={\n \"model_class\"\ - : \"sklearn.linear_model.LogisticRegression\",\n \"label_columns\"\ - : \"label\",\n \"drop_columns\": \"id\",\n \"model_name\"\ - : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ - : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ - ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" + lineno: 126 parameters: - name: context type: MLClientCtx @@ -79,11 +64,20 @@ spec: doc: Labels to log with the model default: null has_kwargs: true - evaluate: - lineno: 273 + doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ + \ project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ + hub://auto_trainer\", \"train\")\n trainer_run = project.run(\n \ + \ name=\"train\",\n handler=\"train\",\n inputs={\"dataset\"\ + : \"./path/to/dataset.csv\"},\n params={\n \"model_class\"\ + : \"sklearn.linear_model.LogisticRegression\",\n \"label_columns\"\ + : \"label\",\n \"drop_columns\": \"id\",\n \"model_name\"\ + : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ + : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ + ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" has_varargs: false + evaluate: name: evaluate - doc: Evaluating a model. Artifacts generated by the MLHandler. + lineno: 278 parameters: - name: context type: MLClientCtx @@ -104,11 +98,11 @@ spec: Classification tasks. Mandatory when dataset is not a FeatureVector. default: null has_kwargs: true - predict: - lineno: 327 + doc: Evaluating a model. Artifacts generated by the MLHandler. has_varargs: false + predict: name: predict - doc: Predicting dataset by a model. + lineno: 332 parameters: - name: context type: MLClientCtx @@ -138,10 +132,17 @@ spec: to 'prediction'. default: null has_kwargs: true + doc: Predicting dataset by a model. + has_varargs: false + description: Automatic train, evaluate and predict functions for the ML frameworks + - Scikit-Learn, XGBoost and LightGBM. + command: '' + disable_auto_mount: false + image: mlrun/mlrun + default_handler: train + build: + code_origin: '' + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICAjIENoZWNrIGlmIGRhdGFzZXQgaXMgbGlzdC9kaWN0IGZpcnN0IChiZWZvcmUgdHJ5aW5nIHRvIGFjY2VzcyBhcnRpZmFjdF91cmwpCiAgICBpZiBpc2luc3RhbmNlKGRhdGFzZXQsIChsaXN0LCBkaWN0KSk6CiAgICAgICAgIyBsaXN0L2RpY3QgY2FzZToKICAgICAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICJsYWJlbF9jb2x1bW5zIG5vdCBwcm92aWRlZCwgbWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yIgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICBkYXRhc2V0ID0gcGQuRGF0YUZyYW1lKGRhdGFzZXQpCiAgICAgICAgIyBDaGVja2luZyBpZiBkcm9wX2NvbHVtbnMgcHJvdmlkZWQgYnkgaW50ZWdlciB0eXBlOgogICAgICAgIGlmIGRyb3BfY29sdW1uczoKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIHN0cikgb3IgKAogICAgICAgICAgICAgICAgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIGxpc3QpCiAgICAgICAgICAgICAgICBhbmQgYW55KGlzaW5zdGFuY2UoY29sLCBzdHIpIGZvciBjb2wgaW4gZHJvcF9jb2x1bW5zKQogICAgICAgICAgICApOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyBtdXN0IGJlIGFuIGludGVnZXIvbGlzdCBvZiBpbnRlZ2VycyBpZiBub3QgcHJvdmlkZWQgd2l0aCBhIFVSSS9GZWF0dXJlVmVjdG9yIGRhdGFzZXQiCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICAgICAgICAgIGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSwgaW5wbGFjZT1UcnVlKQogICAgZWxzZToKICAgICAgICAjIERhdGFzZXQgaXMgYSBEYXRhSXRlbSB3aXRoIGFydGlmYWN0X3VybCAoVVJJIG9yIEZlYXR1cmVWZWN0b3IpCiAgICAgICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGF0YXNldC5hcnRpZmFjdF91cmwpCgogICAgICAgICMgR2V0dGluZyB0aGUgZGF0YXNldDoKICAgICAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBsYWJlbF9jb2x1bW5zIG9yIGRhdGFzZXQubWV0YS5zdGF0dXMubGFiZWxfY29sdW1uCiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJsYWJlbCBjb2x1bW5zOiB7bGFiZWxfY29sdW1uc30iKQogICAgICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnYgPSBtbHJ1bi5kYXRhc3RvcmUuZ2V0X3N0b3JlX3Jlc291cmNlKGRhdGFzZXQuYXJ0aWZhY3RfdXJsKQogICAgICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgICAgIGV4Y2VwdCBBdHRyaWJ1dGVFcnJvcjoKICAgICAgICAgICAgICAgICMgTGVhdmUgaGVyZSBmb3IgYmFja3dhcmRzIGNvbXBhdGliaWxpdHkKICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgICAgICBkYXRhc2V0Lm1ldGEudXJpLCBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zCiAgICAgICAgICAgICAgICApLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgIyBzaW1wbGUgVVJMIGNhc2U6CiAgICAgICAgICAgIGlmIG5vdCBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyBub3QgcHJvdmlkZWQsIG1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3RvciIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuYXNfZGYoKQogICAgICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBkYXRhc2V0LmRyb3AoZHJvcF9jb2x1bW5zLCBheGlzPTEpCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgICAgICJub3QgYWxsIG9mIHRoZSBjb2x1bW5zIHRvIGRyb3AgaW4gdGhlIGRhdGFzZXQsIGRyb3AgY29sdW1ucyBwcm9jZXNzIHNraXBwZWQiCiAgICAgICAgICAgICAgICAgICAgKQoKICAgIHJldHVybiBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zCgoKZGVmIHRyYWluKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIG1vZGVsX2NsYXNzOiBzdHIsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIG1vZGVsX25hbWU6IHN0ciA9ICJtb2RlbCIsCiAgICB0YWc6IHN0ciA9ICIiLAogICAgc2FtcGxlX3NldDogRGF0YUl0ZW0gPSBOb25lLAogICAgdGVzdF9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZTogZmxvYXQgPSBOb25lLAogICAgcmFuZG9tX3N0YXRlOiBpbnQgPSBOb25lLAogICAgbGFiZWxzOiBkaWN0ID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBUcmFpbmluZyBhIG1vZGVsIHdpdGggdGhlIGdpdmVuIGRhdGFzZXQuCgogICAgZXhhbXBsZTo6CgogICAgICAgIGltcG9ydCBtbHJ1bgogICAgICAgIHByb2plY3QgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX3Byb2plY3QoIm15LXByb2plY3QiKQogICAgICAgIHByb2plY3Quc2V0X2Z1bmN0aW9uKCJodWI6Ly9hdXRvX3RyYWluZXIiLCAidHJhaW4iKQogICAgICAgIHRyYWluZXJfcnVuID0gcHJvamVjdC5ydW4oCiAgICAgICAgICAgIG5hbWU9InRyYWluIiwKICAgICAgICAgICAgaGFuZGxlcj0idHJhaW4iLAogICAgICAgICAgICBpbnB1dHM9eyJkYXRhc2V0IjogIi4vcGF0aC90by9kYXRhc2V0LmNzdiJ9LAogICAgICAgICAgICBwYXJhbXM9ewogICAgICAgICAgICAgICAgIm1vZGVsX2NsYXNzIjogInNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyI6ICJsYWJlbCIsCiAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIjogImlkIiwKICAgICAgICAgICAgICAgICJtb2RlbF9uYW1lIjogIm15LW1vZGVsIiwKICAgICAgICAgICAgICAgICJ0YWciOiAidjEuMC4wIiwKICAgICAgICAgICAgICAgICJzYW1wbGVfc2V0IjogIi4vcGF0aC90by9zYW1wbGVfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAidGVzdF9zZXQiOiAiLi9wYXRoL3RvL3Rlc3Rfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAiQ0xBU1Nfc29sdmVyIjogImxpYmxpbmVhciIsCiAgICAgICAgICAgIH0sCiAgICAgICAgKQoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dAogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICAgICAgICAgIFRoZSBjbGFzcyBvZiB0aGUgbW9kZWwsIGUuZy4gYHNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbmAKICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcAogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICAgICBUaGUgbW9kZWwncyBuYW1lIHRvIHVzZSBmb3Igc3RvcmluZyB0aGUgbW9kZWwgYXJ0aWZhY3QsIGRlZmF1bHQgdG8gJ21vZGVsJwogICAgOnBhcmFtIHRhZzogICAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwncyB0YWcgdG8gbG9nIHdpdGgKICAgIDpwYXJhbSBzYW1wbGVfc2V0OiAgICAgICAgICAgICAgQSBzYW1wbGUgc2V0IG9mIGlucHV0cyBmb3IgdGhlIG1vZGVsIGZvciBsb2dnaW5nIGl0cyBzdGF0cyBhbG9uZyB0aGUgbW9kZWwgaW4gZmF2b3VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9mIG1vZGVsIG1vbml0b3JpbmcuIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gdGVzdF9zZXQ6ICAgICAgICAgICAgICAgIFRoZSB0ZXN0IHNldCB0byB0cmFpbiB0aGUgbW9kZWwgd2l0aC4KICAgIDpwYXJhbSB0cmFpbl90ZXN0X3NwbGl0X3NpemU6ICAgaWYgdGVzdF9zZXQgd2FzIHByb3ZpZGVkIHRoZW4gdGhpcyBhcmd1bWVudCBpcyBpZ25vcmVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTaG91bGQgYmUgYmV0d2VlbiAwLjAgYW5kIDEuMCBhbmQgcmVwcmVzZW50IHRoZSBwcm9wb3J0aW9uIG9mIHRoZSBkYXRhc2V0IHRvIGluY2x1ZGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaW4gdGhlIHRlc3Qgc3BsaXQuIFRoZSBzaXplIG9mIHRoZSBUcmFpbmluZyBzZXQgaXMgc2V0IHRvIHRoZSBjb21wbGVtZW50IG9mIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWUuIERlZmF1bHQgPSAwLjIKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgUmVsZXZhbnQgb25seSB3aGVuIHVzaW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQSByYW5kb20gc3RhdGUgc2VlZCB0byBzaHVmZmxlIHRoZSBkYXRhLiBGb3IgbW9yZSBpbmZvcm1hdGlvbiwgc2VlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL3NjaWtpdC1sZWFybi5vcmcvc3RhYmxlL2dsb3NzYXJ5Lmh0bWwjdGVybS1yYW5kb21fc3RhdGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTm90aWNlIHRoYXQgaGVyZSB3ZSBvbmx5IHBhc3MgaW50ZWdlciB2YWx1ZXMuCiAgICA6cGFyYW0gbGFiZWxzOiAgICAgICAgICAgICAgICAgIExhYmVscyB0byBsb2cgd2l0aCB0aGUgbW9kZWwKICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgd2l0aCBwcmVmaXhlcywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCB3aWxsIGJlIHBhcnNlZCBhbmQgcGFzc2VkIHRvIHRoZSByZWxldmFudCBmdW5jdGlvbiwgYnkgdGhlIGZvbGxvd2luZyBwcmVmaXhlczoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgQ0xBU1NfYCAtIGZvciB0aGUgbW9kZWwgY2xhc3MgYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYEZJVF9gIC0gZm9yIHRoZSBgZml0YCBmdW5jdGlvbiBhcmd1bWVudHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgVFJBSU5fYCAtIGZvciB0aGUgYHRyYWluYCBmdW5jdGlvbiAoaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24gLSBmdXR1cmUpCgogICAgIiIiCiAgICAjIFZhbGlkYXRlIGlucHV0czoKICAgICMgQ2hlY2sgaWYgZXhhY3RseSBvbmUgb2YgdGhlbSBpcyBzdXBwbGllZDoKICAgIGlmIHRlc3Rfc2V0IGlzIE5vbmU6CiAgICAgICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGlzIE5vbmU6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAidGVzdF9zZXQgb3IgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGFyZSBub3QgcHJvdmlkZWQsIHNldHRpbmcgdHJhaW5fdGVzdF9zcGxpdF9zaXplIHRvIDAuMiIKICAgICAgICAgICAgKQogICAgICAgICAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemUgPSAwLjIKCiAgICBlbGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAidGVzdF9zZXQgcHJvdmlkZWQsIGlnbm9yaW5nIGdpdmVuIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB2YWx1ZSIKICAgICAgICApCiAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gTm9uZQoKICAgICMgR2V0IERhdGFGcmFtZSBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgR2V0dGluZyB0aGUgc2FtcGxlIHNldDoKICAgIGlmIHNhbXBsZV9zZXQgaXMgTm9uZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIlNhbXBsZSBzZXQgbm90IGdpdmVuLCB1c2luZyB0aGUgd2hvbGUgdHJhaW5pbmcgc2V0IGFzIHRoZSBzYW1wbGUgc2V0IgogICAgICAgICkKICAgICAgICBzYW1wbGVfc2V0ID0gZGF0YXNldAogICAgZWxzZToKICAgICAgICBzYW1wbGVfc2V0LCBfID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICAgICAgZGF0YXNldD1zYW1wbGVfc2V0LAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICAgICAgKQoKICAgICMgUGFyc2luZyBrd2FyZ3M6CiAgICAjIFRPRE86IFVzZSBpbiB4Z2Igb3IgbGdibSB0cmFpbiBmdW5jdGlvbi4KICAgIHRyYWluX2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuVFJBSU4pCiAgICBmaXRfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5GSVQpCiAgICBtb2RlbF9jbGFzc19rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeCgKICAgICAgICBzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLk1PREVMX0NMQVNTCiAgICApCgogICAgIyBDaGVjayBpZiBtb2RlbCBvciBmdW5jdGlvbjoKICAgIGlmIGhhc2F0dHIobW9kZWxfY2xhc3MsICJ0cmFpbiIpOgogICAgICAgICMgVE9ETzogTmVlZCB0byBjYWxsOiBtb2RlbCgpLCBhZnRlcndhcmRzIHRvIHN0YXJ0IHRoZSB0cmFpbiBmdW5jdGlvbi4KICAgICAgICAjIG1vZGVsID0gY3JlYXRlX2Z1bmN0aW9uKGYie21vZGVsX2NsYXNzfS50cmFpbiIpCiAgICAgICAgcmFpc2UgTm90SW1wbGVtZW50ZWRFcnJvcgogICAgZWxzZToKICAgICAgICAjIENyZWF0aW5nIG1vZGVsIGluc3RhbmNlOgogICAgICAgIG1vZGVsID0gY3JlYXRlX2NsYXNzKG1vZGVsX2NsYXNzKSgqKm1vZGVsX2NsYXNzX2t3YXJncykKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCiAgICBpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgeF90cmFpbiwgeF90ZXN0LCB5X3RyYWluLCB5X3Rlc3QgPSB0cmFpbl90ZXN0X3NwbGl0KAogICAgICAgICAgICB4LCB5LCB0ZXN0X3NpemU9dHJhaW5fdGVzdF9zcGxpdF9zaXplLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICB4X3RyYWluLCB5X3RyYWluID0geCwgeQoKICAgICAgICB0ZXN0X3NldCA9IHRlc3Rfc2V0LmFzX2RmKCkKICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgIHRlc3Rfc2V0ID0gZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xKQoKICAgICAgICB4X3Rlc3QsIHlfdGVzdCA9IHRlc3Rfc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKSwgdGVzdF9zZXRbbGFiZWxfY29sdW1uc10KCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4oCiAgICAgICAgbW9kZWw9bW9kZWwsCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICB0YWc9dGFnLAogICAgICAgIHNhbXBsZV9zZXQ9c2FtcGxlX3NldCwKICAgICAgICB5X2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICB0ZXN0X3NldD10ZXN0X3NldCwKICAgICAgICB4X3Rlc3Q9eF90ZXN0LAogICAgICAgIHlfdGVzdD15X3Rlc3QsCiAgICAgICAgYXJ0aWZhY3RzPWNvbnRleHQuYXJ0aWZhY3RzLAogICAgICAgIGxhYmVscz1sYWJlbHMsCiAgICApCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidHJhaW5pbmcgJ3ttb2RlbF9uYW1lfSciKQogICAgbW9kZWwuZml0KHhfdHJhaW4sIHlfdHJhaW4sICoqZml0X2t3YXJncykKCgpkZWYgZXZhbHVhdGUoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIEV2YWx1YXRpbmcgYSBtb2RlbC4gQXJ0aWZhY3RzIGdlbmVyYXRlZCBieSB0aGUgTUxIYW5kbGVyLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIGV2YWx1YXRlIHRoZSBtb2RlbCBvbi4gQ2FuIGJlIGVpdGhlciBhIFVSSSBvciBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICAgICAgICAgIHN0ciBvciBhIGxpc3Qgb2Ygc3RyaW5ncyB0aGF0IHJlcHJlc2VudCB0aGUgY29sdW1ucyB0byBkcm9wLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHRvIHRoZSBwcmVkaWN0IGZ1bmN0aW9uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChQUkVESUNUXyBwcmVmaXggaXMgbm90IHJlcXVpcmVkKS4KICAgICIiIgogICAgIyBHZXQgZGF0YXNldCBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgUGFyc2luZyBsYWJlbF9jb2x1bW5zOgogICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMgPSBbXQogICAgaWYgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gKAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zIGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgbGlzdCkgZWxzZSBbbGFiZWxfY29sdW1uc10KICAgICAgICApCiAgICAgICAgZm9yIGxjIGluIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGlmIGZzLmNvbW1vbi5mZWF0dXJlX3NlcGFyYXRvciBpbiBsYzoKICAgICAgICAgICAgICAgIGZlYXR1cmVfc2V0X25hbWUsIGxhYmVsX25hbWUsIGFsaWFzID0gZnMuY29tbW9uLnBhcnNlX2ZlYXR1cmVfc3RyaW5nKGxjKQogICAgICAgICAgICAgICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMuYXBwZW5kKGFsaWFzIG9yIGxhYmVsX25hbWUpCiAgICAgICAgaWYgcGFyc2VkX2xhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBwYXJzZWRfbGFiZWxfY29sdW1ucwoKICAgIHggPSBkYXRhc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKQogICAgeSA9IGRhdGFzZXRbbGFiZWxfY29sdW1uc10KCiAgICAjIExvYWRpbmcgdGhlIG1vZGVsIGFuZCBwcmVkaWN0aW5nOgogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWwsIGNvbnRleHQ9Y29udGV4dCwgbW9kZWxfbmFtZT0ibW9kZWxfTGluZWFyUmVncmVzc2lvbiIKICAgICkKICAgIEF1dG9NTFJ1bi5hcHBseV9tbHJ1bihtb2RlbF9oYW5kbGVyLm1vZGVsLCB5X3Rlc3Q9eSwgbW9kZWxfcGF0aD1tb2RlbCkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiZXZhbHVhdGluZyAne21vZGVsX2hhbmRsZXIubW9kZWxfbmFtZX0nIikKICAgIG1vZGVsX2hhbmRsZXIubW9kZWwucHJlZGljdCh4LCAqKmt3YXJncykKCgpkZWYgcHJlZGljdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWw6IHN0ciwKICAgIGRhdGFzZXQ6IG1scnVuLkRhdGFJdGVtLAogICAgZHJvcF9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXSwgaW50LCBMaXN0W2ludF1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgcmVzdWx0X3NldDogT3B0aW9uYWxbc3RyXSA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgUHJlZGljdGluZyBkYXRhc2V0IGJ5IGEgbW9kZWwuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwgU3RvcmUgcGF0aC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gcHJlZGljdCB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkksIGEgRmVhdHVyZVZlY3RvciBvciBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSBpbiBhIHNoYXBlIG9mIGEgbGlzdC9kaWN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHBhc3NpbmcgYSBzYW1wbGUsIHBhc3MgdGhlIGRhdGFzZXQgYXMgYSBmaWVsZCBpbiBgcGFyYW1zYCBpbnN0ZWFkIG9mIGBpbnB1dHNgLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIvaW50IG9yIGEgbGlzdCBvZiBzdHJpbmdzL2ludHMgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbiBuYW1lcy9pbmRpY2VzIHRvIGRyb3AuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFdoZW4gdGhlIGRhdGFzZXQgaXMgYSBsaXN0L2RpY3QgdGhpcyBwYXJhbWV0ZXIgc2hvdWxkIGJlIHJlcHJlc2VudGVkIGJ5IGludGVnZXJzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gcmVzdWx0X3NldDogICAgICAgICAgICAgIFRoZSBkYiBrZXkgdG8gc2V0IG5hbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0IGFuZCB0aGUgZmlsZW5hbWUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHQgdG8gJ3ByZWRpY3Rpb24nLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIGxvYWRpbmcgdGhlIG1vZGVsLCBhbmQgZ2V0dGluZyB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbChtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQpCgogICAgIyBGaXggZmVhdHVyZSBuYW1lcyBmb3IgbW9kZWxzIHRoYXQgcmVxdWlyZSB0aGVtIChlLmcuLCBYR0Jvb3N0KQogICAgIyBXaGVuIGRhdGFzZXQgY29tZXMgZnJvbSBhIGxpc3QsIHBhbmRhcyBhc3NpZ25zIGRlZmF1bHQgaW50ZWdlciBjb2x1bW4gbmFtZXMKICAgICMgYnV0IHNvbWUgbW9kZWxzIGV4cGVjdCBzcGVjaWZpYyBmZWF0dXJlIG5hbWVzIHRoZXkgd2VyZSB0cmFpbmVkIHdpdGgKICAgIGlmIGhhc2F0dHIobW9kZWxfaGFuZGxlci5tb2RlbCwgJ2ZlYXR1cmVfbmFtZXNfaW5fJyk6CiAgICAgICAgZXhwZWN0ZWRfZmVhdHVyZXMgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLmZlYXR1cmVfbmFtZXNfaW5fCiAgICAgICAgaWYgbGVuKGRhdGFzZXQuY29sdW1ucykgPT0gbGVuKGV4cGVjdGVkX2ZlYXR1cmVzKToKICAgICAgICAgICAgIyBPbmx5IHJlbmFtZSBpZiB0aGUgbnVtYmVyIG9mIGNvbHVtbnMgbWF0Y2hlcwogICAgICAgICAgICAjIFRoaXMgaGFuZGxlcyB0aGUgY2FzZSB3aGVyZSBhIGxpc3Qgd2FzIGNvbnZlcnRlZCB0byBEYXRhRnJhbWUgd2l0aCBkZWZhdWx0IGNvbHVtbiBuYW1lcwogICAgICAgICAgICBpZiBub3QgYWxsKGNvbCA9PSBmZWF0IGZvciBjb2wsIGZlYXQgaW4gemlwKGRhdGFzZXQuY29sdW1ucywgZXhwZWN0ZWRfZmVhdHVyZXMpKToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSZW5hbWluZyBkYXRhc2V0IGNvbHVtbnMgdG8gbWF0Y2ggbW9kZWwncyBleHBlY3RlZCBmZWF0dXJlIG5hbWVzIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZGF0YXNldC5jb2x1bW5zID0gZXhwZWN0ZWRfZmVhdHVyZXMKCiAgICAjIERyb3BwaW5nIGxhYmVsIGNvbHVtbnMgaWYgbmVjZXNzYXJ5OgogICAgaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtdCiAgICBlbGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgc3RyKToKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW2xhYmVsX2NvbHVtbnNdCgogICAgIyBQcmVkaWN0aW5nOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIm1ha2luZyBwcmVkaWN0aW9uIGJ5ICd7bW9kZWxfaGFuZGxlci5tb2RlbF9uYW1lfSciKQogICAgeV9wcmVkID0gbW9kZWxfaGFuZGxlci5tb2RlbC5wcmVkaWN0KGRhdGFzZXQsICoqa3dhcmdzKQoKICAgICMgUHJlcGFyaW5nIGFuZCB2YWxpZGF0aW5nIGxhYmVsIGNvbHVtbnMgZm9yIHRoZSBkYXRhZnJhbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0OgogICAgbnVtX3ByZWRpY3RlZCA9IDEgaWYgbGVuKHlfcHJlZC5zaGFwZSkgPT0gMSBlbHNlIHlfcHJlZC5zaGFwZVsxXQoKICAgIGlmIG51bV9wcmVkaWN0ZWQgPiBsZW4obGFiZWxfY29sdW1ucyk6CiAgICAgICAgaWYgbnVtX3ByZWRpY3RlZCA9PSAxOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWyJwcmVkaWN0ZWQgbGFiZWxzIl0KICAgICAgICBlbHNlOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zLmV4dGVuZCgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICBmInByZWRpY3RlZF9sYWJlbF97aSArIDEgKyBsZW4obGFiZWxfY29sdW1ucyl9IgogICAgICAgICAgICAgICAgICAgIGZvciBpIGluIHJhbmdlKG51bV9wcmVkaWN0ZWQgLSBsZW4obGFiZWxfY29sdW1ucykpCiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgICkKICAgIGVsaWYgbnVtX3ByZWRpY3RlZCA8IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJudW1iZXIgb2YgcHJlZGljdGVkIGxhYmVsczoge251bV9wcmVkaWN0ZWR9IGlzIHNtYWxsZXIgdGhhbiBudW1iZXIgb2YgbGFiZWwgY29sdW1uczoge2xlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBhcnRpZmFjdF9uYW1lID0gcmVzdWx0X3NldCBvciAicHJlZGljdGlvbiIKICAgIGxhYmVsc19pbnNpZGVfZGYgPSBzZXQobGFiZWxfY29sdW1ucykgJiBzZXQoZGF0YXNldC5jb2x1bW5zLnRvbGlzdCgpKQogICAgaWYgbGFiZWxzX2luc2lkZV9kZjoKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJUaGUgbGFiZWxzOiB7bGFiZWxzX2luc2lkZV9kZn0gYXJlIGFscmVhZHkgZXhpc3RlZCBpbiB0aGUgZGF0YWZyYW1lIgogICAgICAgICkKICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICBwcmVkX2RmID0gcGQuY29uY2F0KFtkYXRhc2V0LCBwZC5EYXRhRnJhbWUoeV9wcmVkLCBjb2x1bW5zPWxhYmVsX2NvbHVtbnMpXSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldChhcnRpZmFjdF9uYW1lLCBwcmVkX2RmLCBkYl9rZXk9cmVzdWx0X3NldCkK +kind: job verbose: false -metadata: - name: auto-trainer - categories: - - machine-learning - - model-training - tag: '' diff --git a/functions/src/auto_trainer/test_auto_trainer.py b/functions/src/auto_trainer/test_auto_trainer.py index 4a517f112..06b553a35 100644 --- a/functions/src/auto_trainer/test_auto_trainer.py +++ b/functions/src/auto_trainer/test_auto_trainer.py @@ -25,37 +25,6 @@ make_regression, ) -# Monkey-patch sklearn metrics to fix MLRun compatibility with sklearn 1.5+ -# MLRun 1.10.0 calls metrics with the deprecated 'squared' parameter -import sklearn.metrics -from sklearn.metrics import ( - mean_squared_error as _original_mse, - mean_absolute_error as _original_mae, - median_absolute_error as _original_medae, -) - - -def _patched_mean_squared_error(y_true, y_pred, sample_weight=None, multioutput='uniform_average', squared=None): - """Wrapper for mean_squared_error that ignores the deprecated 'squared' parameter.""" - # In sklearn 1.4+, 'squared' parameter was removed. Always return MSE (not RMSE) - return _original_mse(y_true, y_pred, sample_weight=sample_weight, multioutput=multioutput) - - -def _patched_mean_absolute_error(y_true, y_pred, sample_weight=None, multioutput='uniform_average', squared=None): - """Wrapper for mean_absolute_error that ignores any 'squared' parameter.""" - return _original_mae(y_true, y_pred, sample_weight=sample_weight, multioutput=multioutput) - - -def _patched_median_absolute_error(y_true, y_pred, multioutput='uniform_average', sample_weight=None, squared=None): - """Wrapper for median_absolute_error that ignores any 'squared' parameter.""" - return _original_medae(y_true, y_pred, multioutput=multioutput, sample_weight=sample_weight) - - -# Apply the patches -sklearn.metrics.mean_squared_error = _patched_mean_squared_error -sklearn.metrics.mean_absolute_error = _patched_mean_absolute_error -sklearn.metrics.median_absolute_error = _patched_median_absolute_error - MODELS = [ ("sklearn.linear_model.LinearRegression", "regression"), ("sklearn.ensemble.RandomForestClassifier", "classification"), From 1c2323c82edd81e619c78e380cafeeeeadbe1602 Mon Sep 17 00:00:00 2001 From: tomerbv Date: Tue, 20 Jan 2026 14:20:24 +0200 Subject: [PATCH 05/15] revert mlrun version --- functions/src/auto_trainer/item.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/src/auto_trainer/item.yaml b/functions/src/auto_trainer/item.yaml index d397a79d6..78de92ca0 100755 --- a/functions/src/auto_trainer/item.yaml +++ b/functions/src/auto_trainer/item.yaml @@ -13,7 +13,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.10.0 +mlrunVersion: 1.7.0 name: auto_trainer platformVersion: 3.5.0 spec: From ef99df27936537f0f2dd3115072e128ba08a87d7 Mon Sep 17 00:00:00 2001 From: tomerbv Date: Tue, 20 Jan 2026 14:35:00 +0200 Subject: [PATCH 06/15] revert get_or_create_project --- functions/src/auto_trainer/test_auto_trainer.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functions/src/auto_trainer/test_auto_trainer.py b/functions/src/auto_trainer/test_auto_trainer.py index 06b553a35..9a1ff554c 100644 --- a/functions/src/auto_trainer/test_auto_trainer.py +++ b/functions/src/auto_trainer/test_auto_trainer.py @@ -82,7 +82,7 @@ def test_train(model: Tuple[str, str]): dataset, label_columns = _get_dataset(model[1]) is_test_passed = True - project = mlrun.get_or_create_project("auto-trainer-test", context="./") + project = mlrun.new_project("auto-trainer-test", context="./") fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun") train_run = None @@ -119,7 +119,7 @@ def test_train_evaluate(model: Tuple[str, str]): dataset, label_columns = _get_dataset(model[1]) is_test_passed = True # Importing function: - project = mlrun.get_or_create_project("auto-trainer-test", context="./") + project = mlrun.new_project("auto-trainer-test", context="./") fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun") temp_dir = tempfile.mkdtemp() @@ -172,7 +172,7 @@ def test_train_predict(model: Tuple[str, str]): df = pd.read_csv(dataset) sample = df.head().drop("labels", axis=1).values.tolist() # Importing function: - project = mlrun.get_or_create_project("auto-trainer-test", context="./") + project = mlrun.new_project("auto-trainer-test", context="./") fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun") temp_dir = tempfile.mkdtemp() From da513958d3d7b6e1419eb71ff07d4687a7c5e993 Mon Sep 17 00:00:00 2001 From: tomerbv Date: Tue, 20 Jan 2026 15:00:15 +0200 Subject: [PATCH 07/15] revert scikit-learn version --- functions/src/auto_trainer/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/src/auto_trainer/requirements.txt b/functions/src/auto_trainer/requirements.txt index 4854d84fd..b14a0293c 100644 --- a/functions/src/auto_trainer/requirements.txt +++ b/functions/src/auto_trainer/requirements.txt @@ -1,4 +1,4 @@ pandas -scikit-learn~=1.5 +scikit-learn<1.4.0 xgboost<2.0.0 plotly From c89f1c34a3eac2ad6780f74881e3cf83a70fff54 Mon Sep 17 00:00:00 2001 From: tomerbv Date: Tue, 20 Jan 2026 15:20:46 +0200 Subject: [PATCH 08/15] scikit-learn==1.5.2 mlrun v 1.10 --- functions/src/auto_trainer/function.yaml | 42 +++++++++---------- functions/src/auto_trainer/item.yaml | 2 +- functions/src/auto_trainer/requirements.txt | 2 +- .../src/auto_trainer/test_auto_trainer.py | 6 +-- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/functions/src/auto_trainer/function.yaml b/functions/src/auto_trainer/function.yaml index 32e1f67dc..e4a36cd86 100644 --- a/functions/src/auto_trainer/function.yaml +++ b/functions/src/auto_trainer/function.yaml @@ -1,15 +1,15 @@ metadata: - name: auto-trainer categories: - machine-learning - model-training + name: auto-trainer tag: '' spec: - filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/auto_trainer/auto_trainer.py + command: '' + description: Automatic train, evaluate and predict functions for the ML frameworks + - Scikit-Learn, XGBoost and LightGBM. entry_points: train: - name: train - lineno: 126 parameters: - name: context type: MLClientCtx @@ -63,7 +63,8 @@ spec: type: dict doc: Labels to log with the model default: null - has_kwargs: true + has_varargs: false + lineno: 126 doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ \ project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ hub://auto_trainer\", \"train\")\n trainer_run = project.run(\n \ @@ -74,10 +75,9 @@ spec: : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" - has_varargs: false + has_kwargs: true + name: train evaluate: - name: evaluate - lineno: 278 parameters: - name: context type: MLClientCtx @@ -97,12 +97,12 @@ spec: doc: The target label(s) of the column(s) in the dataset. for Regression or Classification tasks. Mandatory when dataset is not a FeatureVector. default: null - has_kwargs: true - doc: Evaluating a model. Artifacts generated by the MLHandler. has_varargs: false + lineno: 278 + doc: Evaluating a model. Artifacts generated by the MLHandler. + has_kwargs: true + name: evaluate predict: - name: predict - lineno: 332 parameters: - name: context type: MLClientCtx @@ -131,18 +131,18 @@ spec: doc: The db key to set name of the prediction result and the filename. Default to 'prediction'. default: null - has_kwargs: true - doc: Predicting dataset by a model. has_varargs: false - description: Automatic train, evaluate and predict functions for the ML frameworks - - Scikit-Learn, XGBoost and LightGBM. - command: '' - disable_auto_mount: false - image: mlrun/mlrun - default_handler: train + lineno: 332 + doc: Predicting dataset by a model. + has_kwargs: true + name: predict build: - code_origin: '' origin_filename: '' + code_origin: '' functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICAjIENoZWNrIGlmIGRhdGFzZXQgaXMgbGlzdC9kaWN0IGZpcnN0IChiZWZvcmUgdHJ5aW5nIHRvIGFjY2VzcyBhcnRpZmFjdF91cmwpCiAgICBpZiBpc2luc3RhbmNlKGRhdGFzZXQsIChsaXN0LCBkaWN0KSk6CiAgICAgICAgIyBsaXN0L2RpY3QgY2FzZToKICAgICAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICJsYWJlbF9jb2x1bW5zIG5vdCBwcm92aWRlZCwgbWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yIgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICBkYXRhc2V0ID0gcGQuRGF0YUZyYW1lKGRhdGFzZXQpCiAgICAgICAgIyBDaGVja2luZyBpZiBkcm9wX2NvbHVtbnMgcHJvdmlkZWQgYnkgaW50ZWdlciB0eXBlOgogICAgICAgIGlmIGRyb3BfY29sdW1uczoKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIHN0cikgb3IgKAogICAgICAgICAgICAgICAgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIGxpc3QpCiAgICAgICAgICAgICAgICBhbmQgYW55KGlzaW5zdGFuY2UoY29sLCBzdHIpIGZvciBjb2wgaW4gZHJvcF9jb2x1bW5zKQogICAgICAgICAgICApOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyBtdXN0IGJlIGFuIGludGVnZXIvbGlzdCBvZiBpbnRlZ2VycyBpZiBub3QgcHJvdmlkZWQgd2l0aCBhIFVSSS9GZWF0dXJlVmVjdG9yIGRhdGFzZXQiCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICAgICAgICAgIGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSwgaW5wbGFjZT1UcnVlKQogICAgZWxzZToKICAgICAgICAjIERhdGFzZXQgaXMgYSBEYXRhSXRlbSB3aXRoIGFydGlmYWN0X3VybCAoVVJJIG9yIEZlYXR1cmVWZWN0b3IpCiAgICAgICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGF0YXNldC5hcnRpZmFjdF91cmwpCgogICAgICAgICMgR2V0dGluZyB0aGUgZGF0YXNldDoKICAgICAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBsYWJlbF9jb2x1bW5zIG9yIGRhdGFzZXQubWV0YS5zdGF0dXMubGFiZWxfY29sdW1uCiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJsYWJlbCBjb2x1bW5zOiB7bGFiZWxfY29sdW1uc30iKQogICAgICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnYgPSBtbHJ1bi5kYXRhc3RvcmUuZ2V0X3N0b3JlX3Jlc291cmNlKGRhdGFzZXQuYXJ0aWZhY3RfdXJsKQogICAgICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgICAgIGV4Y2VwdCBBdHRyaWJ1dGVFcnJvcjoKICAgICAgICAgICAgICAgICMgTGVhdmUgaGVyZSBmb3IgYmFja3dhcmRzIGNvbXBhdGliaWxpdHkKICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgICAgICBkYXRhc2V0Lm1ldGEudXJpLCBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zCiAgICAgICAgICAgICAgICApLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgIyBzaW1wbGUgVVJMIGNhc2U6CiAgICAgICAgICAgIGlmIG5vdCBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyBub3QgcHJvdmlkZWQsIG1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3RvciIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuYXNfZGYoKQogICAgICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBkYXRhc2V0LmRyb3AoZHJvcF9jb2x1bW5zLCBheGlzPTEpCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgICAgICJub3QgYWxsIG9mIHRoZSBjb2x1bW5zIHRvIGRyb3AgaW4gdGhlIGRhdGFzZXQsIGRyb3AgY29sdW1ucyBwcm9jZXNzIHNraXBwZWQiCiAgICAgICAgICAgICAgICAgICAgKQoKICAgIHJldHVybiBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zCgoKZGVmIHRyYWluKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIG1vZGVsX2NsYXNzOiBzdHIsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIG1vZGVsX25hbWU6IHN0ciA9ICJtb2RlbCIsCiAgICB0YWc6IHN0ciA9ICIiLAogICAgc2FtcGxlX3NldDogRGF0YUl0ZW0gPSBOb25lLAogICAgdGVzdF9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZTogZmxvYXQgPSBOb25lLAogICAgcmFuZG9tX3N0YXRlOiBpbnQgPSBOb25lLAogICAgbGFiZWxzOiBkaWN0ID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBUcmFpbmluZyBhIG1vZGVsIHdpdGggdGhlIGdpdmVuIGRhdGFzZXQuCgogICAgZXhhbXBsZTo6CgogICAgICAgIGltcG9ydCBtbHJ1bgogICAgICAgIHByb2plY3QgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX3Byb2plY3QoIm15LXByb2plY3QiKQogICAgICAgIHByb2plY3Quc2V0X2Z1bmN0aW9uKCJodWI6Ly9hdXRvX3RyYWluZXIiLCAidHJhaW4iKQogICAgICAgIHRyYWluZXJfcnVuID0gcHJvamVjdC5ydW4oCiAgICAgICAgICAgIG5hbWU9InRyYWluIiwKICAgICAgICAgICAgaGFuZGxlcj0idHJhaW4iLAogICAgICAgICAgICBpbnB1dHM9eyJkYXRhc2V0IjogIi4vcGF0aC90by9kYXRhc2V0LmNzdiJ9LAogICAgICAgICAgICBwYXJhbXM9ewogICAgICAgICAgICAgICAgIm1vZGVsX2NsYXNzIjogInNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyI6ICJsYWJlbCIsCiAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIjogImlkIiwKICAgICAgICAgICAgICAgICJtb2RlbF9uYW1lIjogIm15LW1vZGVsIiwKICAgICAgICAgICAgICAgICJ0YWciOiAidjEuMC4wIiwKICAgICAgICAgICAgICAgICJzYW1wbGVfc2V0IjogIi4vcGF0aC90by9zYW1wbGVfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAidGVzdF9zZXQiOiAiLi9wYXRoL3RvL3Rlc3Rfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAiQ0xBU1Nfc29sdmVyIjogImxpYmxpbmVhciIsCiAgICAgICAgICAgIH0sCiAgICAgICAgKQoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dAogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICAgICAgICAgIFRoZSBjbGFzcyBvZiB0aGUgbW9kZWwsIGUuZy4gYHNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbmAKICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcAogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICAgICBUaGUgbW9kZWwncyBuYW1lIHRvIHVzZSBmb3Igc3RvcmluZyB0aGUgbW9kZWwgYXJ0aWZhY3QsIGRlZmF1bHQgdG8gJ21vZGVsJwogICAgOnBhcmFtIHRhZzogICAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwncyB0YWcgdG8gbG9nIHdpdGgKICAgIDpwYXJhbSBzYW1wbGVfc2V0OiAgICAgICAgICAgICAgQSBzYW1wbGUgc2V0IG9mIGlucHV0cyBmb3IgdGhlIG1vZGVsIGZvciBsb2dnaW5nIGl0cyBzdGF0cyBhbG9uZyB0aGUgbW9kZWwgaW4gZmF2b3VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9mIG1vZGVsIG1vbml0b3JpbmcuIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gdGVzdF9zZXQ6ICAgICAgICAgICAgICAgIFRoZSB0ZXN0IHNldCB0byB0cmFpbiB0aGUgbW9kZWwgd2l0aC4KICAgIDpwYXJhbSB0cmFpbl90ZXN0X3NwbGl0X3NpemU6ICAgaWYgdGVzdF9zZXQgd2FzIHByb3ZpZGVkIHRoZW4gdGhpcyBhcmd1bWVudCBpcyBpZ25vcmVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTaG91bGQgYmUgYmV0d2VlbiAwLjAgYW5kIDEuMCBhbmQgcmVwcmVzZW50IHRoZSBwcm9wb3J0aW9uIG9mIHRoZSBkYXRhc2V0IHRvIGluY2x1ZGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaW4gdGhlIHRlc3Qgc3BsaXQuIFRoZSBzaXplIG9mIHRoZSBUcmFpbmluZyBzZXQgaXMgc2V0IHRvIHRoZSBjb21wbGVtZW50IG9mIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWUuIERlZmF1bHQgPSAwLjIKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgUmVsZXZhbnQgb25seSB3aGVuIHVzaW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQSByYW5kb20gc3RhdGUgc2VlZCB0byBzaHVmZmxlIHRoZSBkYXRhLiBGb3IgbW9yZSBpbmZvcm1hdGlvbiwgc2VlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL3NjaWtpdC1sZWFybi5vcmcvc3RhYmxlL2dsb3NzYXJ5Lmh0bWwjdGVybS1yYW5kb21fc3RhdGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTm90aWNlIHRoYXQgaGVyZSB3ZSBvbmx5IHBhc3MgaW50ZWdlciB2YWx1ZXMuCiAgICA6cGFyYW0gbGFiZWxzOiAgICAgICAgICAgICAgICAgIExhYmVscyB0byBsb2cgd2l0aCB0aGUgbW9kZWwKICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgd2l0aCBwcmVmaXhlcywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCB3aWxsIGJlIHBhcnNlZCBhbmQgcGFzc2VkIHRvIHRoZSByZWxldmFudCBmdW5jdGlvbiwgYnkgdGhlIGZvbGxvd2luZyBwcmVmaXhlczoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgQ0xBU1NfYCAtIGZvciB0aGUgbW9kZWwgY2xhc3MgYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYEZJVF9gIC0gZm9yIHRoZSBgZml0YCBmdW5jdGlvbiBhcmd1bWVudHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgVFJBSU5fYCAtIGZvciB0aGUgYHRyYWluYCBmdW5jdGlvbiAoaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24gLSBmdXR1cmUpCgogICAgIiIiCiAgICAjIFZhbGlkYXRlIGlucHV0czoKICAgICMgQ2hlY2sgaWYgZXhhY3RseSBvbmUgb2YgdGhlbSBpcyBzdXBwbGllZDoKICAgIGlmIHRlc3Rfc2V0IGlzIE5vbmU6CiAgICAgICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGlzIE5vbmU6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAidGVzdF9zZXQgb3IgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGFyZSBub3QgcHJvdmlkZWQsIHNldHRpbmcgdHJhaW5fdGVzdF9zcGxpdF9zaXplIHRvIDAuMiIKICAgICAgICAgICAgKQogICAgICAgICAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemUgPSAwLjIKCiAgICBlbGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAidGVzdF9zZXQgcHJvdmlkZWQsIGlnbm9yaW5nIGdpdmVuIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB2YWx1ZSIKICAgICAgICApCiAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gTm9uZQoKICAgICMgR2V0IERhdGFGcmFtZSBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgR2V0dGluZyB0aGUgc2FtcGxlIHNldDoKICAgIGlmIHNhbXBsZV9zZXQgaXMgTm9uZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIlNhbXBsZSBzZXQgbm90IGdpdmVuLCB1c2luZyB0aGUgd2hvbGUgdHJhaW5pbmcgc2V0IGFzIHRoZSBzYW1wbGUgc2V0IgogICAgICAgICkKICAgICAgICBzYW1wbGVfc2V0ID0gZGF0YXNldAogICAgZWxzZToKICAgICAgICBzYW1wbGVfc2V0LCBfID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICAgICAgZGF0YXNldD1zYW1wbGVfc2V0LAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICAgICAgKQoKICAgICMgUGFyc2luZyBrd2FyZ3M6CiAgICAjIFRPRE86IFVzZSBpbiB4Z2Igb3IgbGdibSB0cmFpbiBmdW5jdGlvbi4KICAgIHRyYWluX2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuVFJBSU4pCiAgICBmaXRfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5GSVQpCiAgICBtb2RlbF9jbGFzc19rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeCgKICAgICAgICBzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLk1PREVMX0NMQVNTCiAgICApCgogICAgIyBDaGVjayBpZiBtb2RlbCBvciBmdW5jdGlvbjoKICAgIGlmIGhhc2F0dHIobW9kZWxfY2xhc3MsICJ0cmFpbiIpOgogICAgICAgICMgVE9ETzogTmVlZCB0byBjYWxsOiBtb2RlbCgpLCBhZnRlcndhcmRzIHRvIHN0YXJ0IHRoZSB0cmFpbiBmdW5jdGlvbi4KICAgICAgICAjIG1vZGVsID0gY3JlYXRlX2Z1bmN0aW9uKGYie21vZGVsX2NsYXNzfS50cmFpbiIpCiAgICAgICAgcmFpc2UgTm90SW1wbGVtZW50ZWRFcnJvcgogICAgZWxzZToKICAgICAgICAjIENyZWF0aW5nIG1vZGVsIGluc3RhbmNlOgogICAgICAgIG1vZGVsID0gY3JlYXRlX2NsYXNzKG1vZGVsX2NsYXNzKSgqKm1vZGVsX2NsYXNzX2t3YXJncykKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCiAgICBpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgeF90cmFpbiwgeF90ZXN0LCB5X3RyYWluLCB5X3Rlc3QgPSB0cmFpbl90ZXN0X3NwbGl0KAogICAgICAgICAgICB4LCB5LCB0ZXN0X3NpemU9dHJhaW5fdGVzdF9zcGxpdF9zaXplLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICB4X3RyYWluLCB5X3RyYWluID0geCwgeQoKICAgICAgICB0ZXN0X3NldCA9IHRlc3Rfc2V0LmFzX2RmKCkKICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgIHRlc3Rfc2V0ID0gZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xKQoKICAgICAgICB4X3Rlc3QsIHlfdGVzdCA9IHRlc3Rfc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKSwgdGVzdF9zZXRbbGFiZWxfY29sdW1uc10KCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4oCiAgICAgICAgbW9kZWw9bW9kZWwsCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICB0YWc9dGFnLAogICAgICAgIHNhbXBsZV9zZXQ9c2FtcGxlX3NldCwKICAgICAgICB5X2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICB0ZXN0X3NldD10ZXN0X3NldCwKICAgICAgICB4X3Rlc3Q9eF90ZXN0LAogICAgICAgIHlfdGVzdD15X3Rlc3QsCiAgICAgICAgYXJ0aWZhY3RzPWNvbnRleHQuYXJ0aWZhY3RzLAogICAgICAgIGxhYmVscz1sYWJlbHMsCiAgICApCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidHJhaW5pbmcgJ3ttb2RlbF9uYW1lfSciKQogICAgbW9kZWwuZml0KHhfdHJhaW4sIHlfdHJhaW4sICoqZml0X2t3YXJncykKCgpkZWYgZXZhbHVhdGUoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIEV2YWx1YXRpbmcgYSBtb2RlbC4gQXJ0aWZhY3RzIGdlbmVyYXRlZCBieSB0aGUgTUxIYW5kbGVyLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIGV2YWx1YXRlIHRoZSBtb2RlbCBvbi4gQ2FuIGJlIGVpdGhlciBhIFVSSSBvciBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICAgICAgICAgIHN0ciBvciBhIGxpc3Qgb2Ygc3RyaW5ncyB0aGF0IHJlcHJlc2VudCB0aGUgY29sdW1ucyB0byBkcm9wLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHRvIHRoZSBwcmVkaWN0IGZ1bmN0aW9uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChQUkVESUNUXyBwcmVmaXggaXMgbm90IHJlcXVpcmVkKS4KICAgICIiIgogICAgIyBHZXQgZGF0YXNldCBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgUGFyc2luZyBsYWJlbF9jb2x1bW5zOgogICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMgPSBbXQogICAgaWYgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gKAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zIGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgbGlzdCkgZWxzZSBbbGFiZWxfY29sdW1uc10KICAgICAgICApCiAgICAgICAgZm9yIGxjIGluIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGlmIGZzLmNvbW1vbi5mZWF0dXJlX3NlcGFyYXRvciBpbiBsYzoKICAgICAgICAgICAgICAgIGZlYXR1cmVfc2V0X25hbWUsIGxhYmVsX25hbWUsIGFsaWFzID0gZnMuY29tbW9uLnBhcnNlX2ZlYXR1cmVfc3RyaW5nKGxjKQogICAgICAgICAgICAgICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMuYXBwZW5kKGFsaWFzIG9yIGxhYmVsX25hbWUpCiAgICAgICAgaWYgcGFyc2VkX2xhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBwYXJzZWRfbGFiZWxfY29sdW1ucwoKICAgIHggPSBkYXRhc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKQogICAgeSA9IGRhdGFzZXRbbGFiZWxfY29sdW1uc10KCiAgICAjIExvYWRpbmcgdGhlIG1vZGVsIGFuZCBwcmVkaWN0aW5nOgogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWwsIGNvbnRleHQ9Y29udGV4dCwgbW9kZWxfbmFtZT0ibW9kZWxfTGluZWFyUmVncmVzc2lvbiIKICAgICkKICAgIEF1dG9NTFJ1bi5hcHBseV9tbHJ1bihtb2RlbF9oYW5kbGVyLm1vZGVsLCB5X3Rlc3Q9eSwgbW9kZWxfcGF0aD1tb2RlbCkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiZXZhbHVhdGluZyAne21vZGVsX2hhbmRsZXIubW9kZWxfbmFtZX0nIikKICAgIG1vZGVsX2hhbmRsZXIubW9kZWwucHJlZGljdCh4LCAqKmt3YXJncykKCgpkZWYgcHJlZGljdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWw6IHN0ciwKICAgIGRhdGFzZXQ6IG1scnVuLkRhdGFJdGVtLAogICAgZHJvcF9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXSwgaW50LCBMaXN0W2ludF1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgcmVzdWx0X3NldDogT3B0aW9uYWxbc3RyXSA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgUHJlZGljdGluZyBkYXRhc2V0IGJ5IGEgbW9kZWwuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwgU3RvcmUgcGF0aC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gcHJlZGljdCB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkksIGEgRmVhdHVyZVZlY3RvciBvciBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSBpbiBhIHNoYXBlIG9mIGEgbGlzdC9kaWN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHBhc3NpbmcgYSBzYW1wbGUsIHBhc3MgdGhlIGRhdGFzZXQgYXMgYSBmaWVsZCBpbiBgcGFyYW1zYCBpbnN0ZWFkIG9mIGBpbnB1dHNgLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIvaW50IG9yIGEgbGlzdCBvZiBzdHJpbmdzL2ludHMgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbiBuYW1lcy9pbmRpY2VzIHRvIGRyb3AuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFdoZW4gdGhlIGRhdGFzZXQgaXMgYSBsaXN0L2RpY3QgdGhpcyBwYXJhbWV0ZXIgc2hvdWxkIGJlIHJlcHJlc2VudGVkIGJ5IGludGVnZXJzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gcmVzdWx0X3NldDogICAgICAgICAgICAgIFRoZSBkYiBrZXkgdG8gc2V0IG5hbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0IGFuZCB0aGUgZmlsZW5hbWUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHQgdG8gJ3ByZWRpY3Rpb24nLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIGxvYWRpbmcgdGhlIG1vZGVsLCBhbmQgZ2V0dGluZyB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbChtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQpCgogICAgIyBGaXggZmVhdHVyZSBuYW1lcyBmb3IgbW9kZWxzIHRoYXQgcmVxdWlyZSB0aGVtIChlLmcuLCBYR0Jvb3N0KQogICAgIyBXaGVuIGRhdGFzZXQgY29tZXMgZnJvbSBhIGxpc3QsIHBhbmRhcyBhc3NpZ25zIGRlZmF1bHQgaW50ZWdlciBjb2x1bW4gbmFtZXMKICAgICMgYnV0IHNvbWUgbW9kZWxzIGV4cGVjdCBzcGVjaWZpYyBmZWF0dXJlIG5hbWVzIHRoZXkgd2VyZSB0cmFpbmVkIHdpdGgKICAgIGlmIGhhc2F0dHIobW9kZWxfaGFuZGxlci5tb2RlbCwgJ2ZlYXR1cmVfbmFtZXNfaW5fJyk6CiAgICAgICAgZXhwZWN0ZWRfZmVhdHVyZXMgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLmZlYXR1cmVfbmFtZXNfaW5fCiAgICAgICAgaWYgbGVuKGRhdGFzZXQuY29sdW1ucykgPT0gbGVuKGV4cGVjdGVkX2ZlYXR1cmVzKToKICAgICAgICAgICAgIyBPbmx5IHJlbmFtZSBpZiB0aGUgbnVtYmVyIG9mIGNvbHVtbnMgbWF0Y2hlcwogICAgICAgICAgICAjIFRoaXMgaGFuZGxlcyB0aGUgY2FzZSB3aGVyZSBhIGxpc3Qgd2FzIGNvbnZlcnRlZCB0byBEYXRhRnJhbWUgd2l0aCBkZWZhdWx0IGNvbHVtbiBuYW1lcwogICAgICAgICAgICBpZiBub3QgYWxsKGNvbCA9PSBmZWF0IGZvciBjb2wsIGZlYXQgaW4gemlwKGRhdGFzZXQuY29sdW1ucywgZXhwZWN0ZWRfZmVhdHVyZXMpKToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSZW5hbWluZyBkYXRhc2V0IGNvbHVtbnMgdG8gbWF0Y2ggbW9kZWwncyBleHBlY3RlZCBmZWF0dXJlIG5hbWVzIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZGF0YXNldC5jb2x1bW5zID0gZXhwZWN0ZWRfZmVhdHVyZXMKCiAgICAjIERyb3BwaW5nIGxhYmVsIGNvbHVtbnMgaWYgbmVjZXNzYXJ5OgogICAgaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtdCiAgICBlbGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgc3RyKToKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW2xhYmVsX2NvbHVtbnNdCgogICAgIyBQcmVkaWN0aW5nOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIm1ha2luZyBwcmVkaWN0aW9uIGJ5ICd7bW9kZWxfaGFuZGxlci5tb2RlbF9uYW1lfSciKQogICAgeV9wcmVkID0gbW9kZWxfaGFuZGxlci5tb2RlbC5wcmVkaWN0KGRhdGFzZXQsICoqa3dhcmdzKQoKICAgICMgUHJlcGFyaW5nIGFuZCB2YWxpZGF0aW5nIGxhYmVsIGNvbHVtbnMgZm9yIHRoZSBkYXRhZnJhbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0OgogICAgbnVtX3ByZWRpY3RlZCA9IDEgaWYgbGVuKHlfcHJlZC5zaGFwZSkgPT0gMSBlbHNlIHlfcHJlZC5zaGFwZVsxXQoKICAgIGlmIG51bV9wcmVkaWN0ZWQgPiBsZW4obGFiZWxfY29sdW1ucyk6CiAgICAgICAgaWYgbnVtX3ByZWRpY3RlZCA9PSAxOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWyJwcmVkaWN0ZWQgbGFiZWxzIl0KICAgICAgICBlbHNlOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zLmV4dGVuZCgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICBmInByZWRpY3RlZF9sYWJlbF97aSArIDEgKyBsZW4obGFiZWxfY29sdW1ucyl9IgogICAgICAgICAgICAgICAgICAgIGZvciBpIGluIHJhbmdlKG51bV9wcmVkaWN0ZWQgLSBsZW4obGFiZWxfY29sdW1ucykpCiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgICkKICAgIGVsaWYgbnVtX3ByZWRpY3RlZCA8IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJudW1iZXIgb2YgcHJlZGljdGVkIGxhYmVsczoge251bV9wcmVkaWN0ZWR9IGlzIHNtYWxsZXIgdGhhbiBudW1iZXIgb2YgbGFiZWwgY29sdW1uczoge2xlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBhcnRpZmFjdF9uYW1lID0gcmVzdWx0X3NldCBvciAicHJlZGljdGlvbiIKICAgIGxhYmVsc19pbnNpZGVfZGYgPSBzZXQobGFiZWxfY29sdW1ucykgJiBzZXQoZGF0YXNldC5jb2x1bW5zLnRvbGlzdCgpKQogICAgaWYgbGFiZWxzX2luc2lkZV9kZjoKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJUaGUgbGFiZWxzOiB7bGFiZWxzX2luc2lkZV9kZn0gYXJlIGFscmVhZHkgZXhpc3RlZCBpbiB0aGUgZGF0YWZyYW1lIgogICAgICAgICkKICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICBwcmVkX2RmID0gcGQuY29uY2F0KFtkYXRhc2V0LCBwZC5EYXRhRnJhbWUoeV9wcmVkLCBjb2x1bW5zPWxhYmVsX2NvbHVtbnMpXSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldChhcnRpZmFjdF9uYW1lLCBwcmVkX2RmLCBkYl9rZXk9cmVzdWx0X3NldCkK + default_handler: train + disable_auto_mount: false + filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/auto_trainer/auto_trainer.py + image: mlrun/mlrun kind: job verbose: false diff --git a/functions/src/auto_trainer/item.yaml b/functions/src/auto_trainer/item.yaml index 78de92ca0..d397a79d6 100755 --- a/functions/src/auto_trainer/item.yaml +++ b/functions/src/auto_trainer/item.yaml @@ -13,7 +13,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.0 +mlrunVersion: 1.10.0 name: auto_trainer platformVersion: 3.5.0 spec: diff --git a/functions/src/auto_trainer/requirements.txt b/functions/src/auto_trainer/requirements.txt index b14a0293c..274a97f82 100644 --- a/functions/src/auto_trainer/requirements.txt +++ b/functions/src/auto_trainer/requirements.txt @@ -1,4 +1,4 @@ pandas -scikit-learn<1.4.0 +scikit-learn==1.5.2 xgboost<2.0.0 plotly diff --git a/functions/src/auto_trainer/test_auto_trainer.py b/functions/src/auto_trainer/test_auto_trainer.py index 9a1ff554c..06b553a35 100644 --- a/functions/src/auto_trainer/test_auto_trainer.py +++ b/functions/src/auto_trainer/test_auto_trainer.py @@ -82,7 +82,7 @@ def test_train(model: Tuple[str, str]): dataset, label_columns = _get_dataset(model[1]) is_test_passed = True - project = mlrun.new_project("auto-trainer-test", context="./") + project = mlrun.get_or_create_project("auto-trainer-test", context="./") fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun") train_run = None @@ -119,7 +119,7 @@ def test_train_evaluate(model: Tuple[str, str]): dataset, label_columns = _get_dataset(model[1]) is_test_passed = True # Importing function: - project = mlrun.new_project("auto-trainer-test", context="./") + project = mlrun.get_or_create_project("auto-trainer-test", context="./") fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun") temp_dir = tempfile.mkdtemp() @@ -172,7 +172,7 @@ def test_train_predict(model: Tuple[str, str]): df = pd.read_csv(dataset) sample = df.head().drop("labels", axis=1).values.tolist() # Importing function: - project = mlrun.new_project("auto-trainer-test", context="./") + project = mlrun.get_or_create_project("auto-trainer-test", context="./") fn = project.set_function("function.yaml", "train", kind="job", image="mlrun/mlrun") temp_dir = tempfile.mkdtemp() From 63b968c3409801c25964137995a0e00af23bdf40 Mon Sep 17 00:00:00 2001 From: tomerbv Date: Tue, 20 Jan 2026 15:34:13 +0200 Subject: [PATCH 09/15] scikit-learn==1.4.2 --- functions/src/auto_trainer/function.yaml | 36 ++++++++++----------- functions/src/auto_trainer/requirements.txt | 2 +- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/functions/src/auto_trainer/function.yaml b/functions/src/auto_trainer/function.yaml index e4a36cd86..04b9975c1 100644 --- a/functions/src/auto_trainer/function.yaml +++ b/functions/src/auto_trainer/function.yaml @@ -1,13 +1,11 @@ metadata: + tag: '' + name: auto-trainer categories: - machine-learning - model-training - name: auto-trainer - tag: '' spec: - command: '' - description: Automatic train, evaluate and predict functions for the ML frameworks - - Scikit-Learn, XGBoost and LightGBM. + default_handler: train entry_points: train: parameters: @@ -63,7 +61,8 @@ spec: type: dict doc: Labels to log with the model default: null - has_varargs: false + name: train + has_kwargs: true lineno: 126 doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ \ project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ @@ -75,8 +74,7 @@ spec: : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" - has_kwargs: true - name: train + has_varargs: false evaluate: parameters: - name: context @@ -97,11 +95,11 @@ spec: doc: The target label(s) of the column(s) in the dataset. for Regression or Classification tasks. Mandatory when dataset is not a FeatureVector. default: null - has_varargs: false + name: evaluate + has_kwargs: true lineno: 278 doc: Evaluating a model. Artifacts generated by the MLHandler. - has_kwargs: true - name: evaluate + has_varargs: false predict: parameters: - name: context @@ -131,18 +129,20 @@ spec: doc: The db key to set name of the prediction result and the filename. Default to 'prediction'. default: null - has_varargs: false + name: predict + has_kwargs: true lineno: 332 doc: Predicting dataset by a model. - has_kwargs: true - name: predict + has_varargs: false + command: '' + image: mlrun/mlrun + description: Automatic train, evaluate and predict functions for the ML frameworks + - Scikit-Learn, XGBoost and LightGBM. + filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/auto_trainer/auto_trainer.py build: origin_filename: '' - code_origin: '' functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICAjIENoZWNrIGlmIGRhdGFzZXQgaXMgbGlzdC9kaWN0IGZpcnN0IChiZWZvcmUgdHJ5aW5nIHRvIGFjY2VzcyBhcnRpZmFjdF91cmwpCiAgICBpZiBpc2luc3RhbmNlKGRhdGFzZXQsIChsaXN0LCBkaWN0KSk6CiAgICAgICAgIyBsaXN0L2RpY3QgY2FzZToKICAgICAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICJsYWJlbF9jb2x1bW5zIG5vdCBwcm92aWRlZCwgbWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yIgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICBkYXRhc2V0ID0gcGQuRGF0YUZyYW1lKGRhdGFzZXQpCiAgICAgICAgIyBDaGVja2luZyBpZiBkcm9wX2NvbHVtbnMgcHJvdmlkZWQgYnkgaW50ZWdlciB0eXBlOgogICAgICAgIGlmIGRyb3BfY29sdW1uczoKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIHN0cikgb3IgKAogICAgICAgICAgICAgICAgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIGxpc3QpCiAgICAgICAgICAgICAgICBhbmQgYW55KGlzaW5zdGFuY2UoY29sLCBzdHIpIGZvciBjb2wgaW4gZHJvcF9jb2x1bW5zKQogICAgICAgICAgICApOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyBtdXN0IGJlIGFuIGludGVnZXIvbGlzdCBvZiBpbnRlZ2VycyBpZiBub3QgcHJvdmlkZWQgd2l0aCBhIFVSSS9GZWF0dXJlVmVjdG9yIGRhdGFzZXQiCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICAgICAgICAgIGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSwgaW5wbGFjZT1UcnVlKQogICAgZWxzZToKICAgICAgICAjIERhdGFzZXQgaXMgYSBEYXRhSXRlbSB3aXRoIGFydGlmYWN0X3VybCAoVVJJIG9yIEZlYXR1cmVWZWN0b3IpCiAgICAgICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGF0YXNldC5hcnRpZmFjdF91cmwpCgogICAgICAgICMgR2V0dGluZyB0aGUgZGF0YXNldDoKICAgICAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBsYWJlbF9jb2x1bW5zIG9yIGRhdGFzZXQubWV0YS5zdGF0dXMubGFiZWxfY29sdW1uCiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJsYWJlbCBjb2x1bW5zOiB7bGFiZWxfY29sdW1uc30iKQogICAgICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnYgPSBtbHJ1bi5kYXRhc3RvcmUuZ2V0X3N0b3JlX3Jlc291cmNlKGRhdGFzZXQuYXJ0aWZhY3RfdXJsKQogICAgICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgICAgIGV4Y2VwdCBBdHRyaWJ1dGVFcnJvcjoKICAgICAgICAgICAgICAgICMgTGVhdmUgaGVyZSBmb3IgYmFja3dhcmRzIGNvbXBhdGliaWxpdHkKICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgICAgICBkYXRhc2V0Lm1ldGEudXJpLCBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zCiAgICAgICAgICAgICAgICApLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgIyBzaW1wbGUgVVJMIGNhc2U6CiAgICAgICAgICAgIGlmIG5vdCBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyBub3QgcHJvdmlkZWQsIG1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3RvciIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuYXNfZGYoKQogICAgICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBkYXRhc2V0LmRyb3AoZHJvcF9jb2x1bW5zLCBheGlzPTEpCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgICAgICJub3QgYWxsIG9mIHRoZSBjb2x1bW5zIHRvIGRyb3AgaW4gdGhlIGRhdGFzZXQsIGRyb3AgY29sdW1ucyBwcm9jZXNzIHNraXBwZWQiCiAgICAgICAgICAgICAgICAgICAgKQoKICAgIHJldHVybiBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zCgoKZGVmIHRyYWluKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIG1vZGVsX2NsYXNzOiBzdHIsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIG1vZGVsX25hbWU6IHN0ciA9ICJtb2RlbCIsCiAgICB0YWc6IHN0ciA9ICIiLAogICAgc2FtcGxlX3NldDogRGF0YUl0ZW0gPSBOb25lLAogICAgdGVzdF9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZTogZmxvYXQgPSBOb25lLAogICAgcmFuZG9tX3N0YXRlOiBpbnQgPSBOb25lLAogICAgbGFiZWxzOiBkaWN0ID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBUcmFpbmluZyBhIG1vZGVsIHdpdGggdGhlIGdpdmVuIGRhdGFzZXQuCgogICAgZXhhbXBsZTo6CgogICAgICAgIGltcG9ydCBtbHJ1bgogICAgICAgIHByb2plY3QgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX3Byb2plY3QoIm15LXByb2plY3QiKQogICAgICAgIHByb2plY3Quc2V0X2Z1bmN0aW9uKCJodWI6Ly9hdXRvX3RyYWluZXIiLCAidHJhaW4iKQogICAgICAgIHRyYWluZXJfcnVuID0gcHJvamVjdC5ydW4oCiAgICAgICAgICAgIG5hbWU9InRyYWluIiwKICAgICAgICAgICAgaGFuZGxlcj0idHJhaW4iLAogICAgICAgICAgICBpbnB1dHM9eyJkYXRhc2V0IjogIi4vcGF0aC90by9kYXRhc2V0LmNzdiJ9LAogICAgICAgICAgICBwYXJhbXM9ewogICAgICAgICAgICAgICAgIm1vZGVsX2NsYXNzIjogInNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyI6ICJsYWJlbCIsCiAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIjogImlkIiwKICAgICAgICAgICAgICAgICJtb2RlbF9uYW1lIjogIm15LW1vZGVsIiwKICAgICAgICAgICAgICAgICJ0YWciOiAidjEuMC4wIiwKICAgICAgICAgICAgICAgICJzYW1wbGVfc2V0IjogIi4vcGF0aC90by9zYW1wbGVfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAidGVzdF9zZXQiOiAiLi9wYXRoL3RvL3Rlc3Rfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAiQ0xBU1Nfc29sdmVyIjogImxpYmxpbmVhciIsCiAgICAgICAgICAgIH0sCiAgICAgICAgKQoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dAogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICAgICAgICAgIFRoZSBjbGFzcyBvZiB0aGUgbW9kZWwsIGUuZy4gYHNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbmAKICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcAogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICAgICBUaGUgbW9kZWwncyBuYW1lIHRvIHVzZSBmb3Igc3RvcmluZyB0aGUgbW9kZWwgYXJ0aWZhY3QsIGRlZmF1bHQgdG8gJ21vZGVsJwogICAgOnBhcmFtIHRhZzogICAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwncyB0YWcgdG8gbG9nIHdpdGgKICAgIDpwYXJhbSBzYW1wbGVfc2V0OiAgICAgICAgICAgICAgQSBzYW1wbGUgc2V0IG9mIGlucHV0cyBmb3IgdGhlIG1vZGVsIGZvciBsb2dnaW5nIGl0cyBzdGF0cyBhbG9uZyB0aGUgbW9kZWwgaW4gZmF2b3VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9mIG1vZGVsIG1vbml0b3JpbmcuIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gdGVzdF9zZXQ6ICAgICAgICAgICAgICAgIFRoZSB0ZXN0IHNldCB0byB0cmFpbiB0aGUgbW9kZWwgd2l0aC4KICAgIDpwYXJhbSB0cmFpbl90ZXN0X3NwbGl0X3NpemU6ICAgaWYgdGVzdF9zZXQgd2FzIHByb3ZpZGVkIHRoZW4gdGhpcyBhcmd1bWVudCBpcyBpZ25vcmVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTaG91bGQgYmUgYmV0d2VlbiAwLjAgYW5kIDEuMCBhbmQgcmVwcmVzZW50IHRoZSBwcm9wb3J0aW9uIG9mIHRoZSBkYXRhc2V0IHRvIGluY2x1ZGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaW4gdGhlIHRlc3Qgc3BsaXQuIFRoZSBzaXplIG9mIHRoZSBUcmFpbmluZyBzZXQgaXMgc2V0IHRvIHRoZSBjb21wbGVtZW50IG9mIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWUuIERlZmF1bHQgPSAwLjIKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgUmVsZXZhbnQgb25seSB3aGVuIHVzaW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQSByYW5kb20gc3RhdGUgc2VlZCB0byBzaHVmZmxlIHRoZSBkYXRhLiBGb3IgbW9yZSBpbmZvcm1hdGlvbiwgc2VlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL3NjaWtpdC1sZWFybi5vcmcvc3RhYmxlL2dsb3NzYXJ5Lmh0bWwjdGVybS1yYW5kb21fc3RhdGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTm90aWNlIHRoYXQgaGVyZSB3ZSBvbmx5IHBhc3MgaW50ZWdlciB2YWx1ZXMuCiAgICA6cGFyYW0gbGFiZWxzOiAgICAgICAgICAgICAgICAgIExhYmVscyB0byBsb2cgd2l0aCB0aGUgbW9kZWwKICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgd2l0aCBwcmVmaXhlcywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCB3aWxsIGJlIHBhcnNlZCBhbmQgcGFzc2VkIHRvIHRoZSByZWxldmFudCBmdW5jdGlvbiwgYnkgdGhlIGZvbGxvd2luZyBwcmVmaXhlczoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgQ0xBU1NfYCAtIGZvciB0aGUgbW9kZWwgY2xhc3MgYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYEZJVF9gIC0gZm9yIHRoZSBgZml0YCBmdW5jdGlvbiBhcmd1bWVudHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgVFJBSU5fYCAtIGZvciB0aGUgYHRyYWluYCBmdW5jdGlvbiAoaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24gLSBmdXR1cmUpCgogICAgIiIiCiAgICAjIFZhbGlkYXRlIGlucHV0czoKICAgICMgQ2hlY2sgaWYgZXhhY3RseSBvbmUgb2YgdGhlbSBpcyBzdXBwbGllZDoKICAgIGlmIHRlc3Rfc2V0IGlzIE5vbmU6CiAgICAgICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGlzIE5vbmU6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAidGVzdF9zZXQgb3IgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGFyZSBub3QgcHJvdmlkZWQsIHNldHRpbmcgdHJhaW5fdGVzdF9zcGxpdF9zaXplIHRvIDAuMiIKICAgICAgICAgICAgKQogICAgICAgICAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemUgPSAwLjIKCiAgICBlbGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAidGVzdF9zZXQgcHJvdmlkZWQsIGlnbm9yaW5nIGdpdmVuIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB2YWx1ZSIKICAgICAgICApCiAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gTm9uZQoKICAgICMgR2V0IERhdGFGcmFtZSBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgR2V0dGluZyB0aGUgc2FtcGxlIHNldDoKICAgIGlmIHNhbXBsZV9zZXQgaXMgTm9uZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIlNhbXBsZSBzZXQgbm90IGdpdmVuLCB1c2luZyB0aGUgd2hvbGUgdHJhaW5pbmcgc2V0IGFzIHRoZSBzYW1wbGUgc2V0IgogICAgICAgICkKICAgICAgICBzYW1wbGVfc2V0ID0gZGF0YXNldAogICAgZWxzZToKICAgICAgICBzYW1wbGVfc2V0LCBfID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICAgICAgZGF0YXNldD1zYW1wbGVfc2V0LAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICAgICAgKQoKICAgICMgUGFyc2luZyBrd2FyZ3M6CiAgICAjIFRPRE86IFVzZSBpbiB4Z2Igb3IgbGdibSB0cmFpbiBmdW5jdGlvbi4KICAgIHRyYWluX2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuVFJBSU4pCiAgICBmaXRfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5GSVQpCiAgICBtb2RlbF9jbGFzc19rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeCgKICAgICAgICBzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLk1PREVMX0NMQVNTCiAgICApCgogICAgIyBDaGVjayBpZiBtb2RlbCBvciBmdW5jdGlvbjoKICAgIGlmIGhhc2F0dHIobW9kZWxfY2xhc3MsICJ0cmFpbiIpOgogICAgICAgICMgVE9ETzogTmVlZCB0byBjYWxsOiBtb2RlbCgpLCBhZnRlcndhcmRzIHRvIHN0YXJ0IHRoZSB0cmFpbiBmdW5jdGlvbi4KICAgICAgICAjIG1vZGVsID0gY3JlYXRlX2Z1bmN0aW9uKGYie21vZGVsX2NsYXNzfS50cmFpbiIpCiAgICAgICAgcmFpc2UgTm90SW1wbGVtZW50ZWRFcnJvcgogICAgZWxzZToKICAgICAgICAjIENyZWF0aW5nIG1vZGVsIGluc3RhbmNlOgogICAgICAgIG1vZGVsID0gY3JlYXRlX2NsYXNzKG1vZGVsX2NsYXNzKSgqKm1vZGVsX2NsYXNzX2t3YXJncykKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCiAgICBpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgeF90cmFpbiwgeF90ZXN0LCB5X3RyYWluLCB5X3Rlc3QgPSB0cmFpbl90ZXN0X3NwbGl0KAogICAgICAgICAgICB4LCB5LCB0ZXN0X3NpemU9dHJhaW5fdGVzdF9zcGxpdF9zaXplLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICB4X3RyYWluLCB5X3RyYWluID0geCwgeQoKICAgICAgICB0ZXN0X3NldCA9IHRlc3Rfc2V0LmFzX2RmKCkKICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgIHRlc3Rfc2V0ID0gZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xKQoKICAgICAgICB4X3Rlc3QsIHlfdGVzdCA9IHRlc3Rfc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKSwgdGVzdF9zZXRbbGFiZWxfY29sdW1uc10KCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4oCiAgICAgICAgbW9kZWw9bW9kZWwsCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICB0YWc9dGFnLAogICAgICAgIHNhbXBsZV9zZXQ9c2FtcGxlX3NldCwKICAgICAgICB5X2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICB0ZXN0X3NldD10ZXN0X3NldCwKICAgICAgICB4X3Rlc3Q9eF90ZXN0LAogICAgICAgIHlfdGVzdD15X3Rlc3QsCiAgICAgICAgYXJ0aWZhY3RzPWNvbnRleHQuYXJ0aWZhY3RzLAogICAgICAgIGxhYmVscz1sYWJlbHMsCiAgICApCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidHJhaW5pbmcgJ3ttb2RlbF9uYW1lfSciKQogICAgbW9kZWwuZml0KHhfdHJhaW4sIHlfdHJhaW4sICoqZml0X2t3YXJncykKCgpkZWYgZXZhbHVhdGUoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIEV2YWx1YXRpbmcgYSBtb2RlbC4gQXJ0aWZhY3RzIGdlbmVyYXRlZCBieSB0aGUgTUxIYW5kbGVyLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIGV2YWx1YXRlIHRoZSBtb2RlbCBvbi4gQ2FuIGJlIGVpdGhlciBhIFVSSSBvciBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICAgICAgICAgIHN0ciBvciBhIGxpc3Qgb2Ygc3RyaW5ncyB0aGF0IHJlcHJlc2VudCB0aGUgY29sdW1ucyB0byBkcm9wLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHRvIHRoZSBwcmVkaWN0IGZ1bmN0aW9uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChQUkVESUNUXyBwcmVmaXggaXMgbm90IHJlcXVpcmVkKS4KICAgICIiIgogICAgIyBHZXQgZGF0YXNldCBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgUGFyc2luZyBsYWJlbF9jb2x1bW5zOgogICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMgPSBbXQogICAgaWYgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gKAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zIGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgbGlzdCkgZWxzZSBbbGFiZWxfY29sdW1uc10KICAgICAgICApCiAgICAgICAgZm9yIGxjIGluIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGlmIGZzLmNvbW1vbi5mZWF0dXJlX3NlcGFyYXRvciBpbiBsYzoKICAgICAgICAgICAgICAgIGZlYXR1cmVfc2V0X25hbWUsIGxhYmVsX25hbWUsIGFsaWFzID0gZnMuY29tbW9uLnBhcnNlX2ZlYXR1cmVfc3RyaW5nKGxjKQogICAgICAgICAgICAgICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMuYXBwZW5kKGFsaWFzIG9yIGxhYmVsX25hbWUpCiAgICAgICAgaWYgcGFyc2VkX2xhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBwYXJzZWRfbGFiZWxfY29sdW1ucwoKICAgIHggPSBkYXRhc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKQogICAgeSA9IGRhdGFzZXRbbGFiZWxfY29sdW1uc10KCiAgICAjIExvYWRpbmcgdGhlIG1vZGVsIGFuZCBwcmVkaWN0aW5nOgogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWwsIGNvbnRleHQ9Y29udGV4dCwgbW9kZWxfbmFtZT0ibW9kZWxfTGluZWFyUmVncmVzc2lvbiIKICAgICkKICAgIEF1dG9NTFJ1bi5hcHBseV9tbHJ1bihtb2RlbF9oYW5kbGVyLm1vZGVsLCB5X3Rlc3Q9eSwgbW9kZWxfcGF0aD1tb2RlbCkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiZXZhbHVhdGluZyAne21vZGVsX2hhbmRsZXIubW9kZWxfbmFtZX0nIikKICAgIG1vZGVsX2hhbmRsZXIubW9kZWwucHJlZGljdCh4LCAqKmt3YXJncykKCgpkZWYgcHJlZGljdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWw6IHN0ciwKICAgIGRhdGFzZXQ6IG1scnVuLkRhdGFJdGVtLAogICAgZHJvcF9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXSwgaW50LCBMaXN0W2ludF1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgcmVzdWx0X3NldDogT3B0aW9uYWxbc3RyXSA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgUHJlZGljdGluZyBkYXRhc2V0IGJ5IGEgbW9kZWwuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwgU3RvcmUgcGF0aC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gcHJlZGljdCB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkksIGEgRmVhdHVyZVZlY3RvciBvciBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSBpbiBhIHNoYXBlIG9mIGEgbGlzdC9kaWN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHBhc3NpbmcgYSBzYW1wbGUsIHBhc3MgdGhlIGRhdGFzZXQgYXMgYSBmaWVsZCBpbiBgcGFyYW1zYCBpbnN0ZWFkIG9mIGBpbnB1dHNgLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIvaW50IG9yIGEgbGlzdCBvZiBzdHJpbmdzL2ludHMgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbiBuYW1lcy9pbmRpY2VzIHRvIGRyb3AuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFdoZW4gdGhlIGRhdGFzZXQgaXMgYSBsaXN0L2RpY3QgdGhpcyBwYXJhbWV0ZXIgc2hvdWxkIGJlIHJlcHJlc2VudGVkIGJ5IGludGVnZXJzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gcmVzdWx0X3NldDogICAgICAgICAgICAgIFRoZSBkYiBrZXkgdG8gc2V0IG5hbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0IGFuZCB0aGUgZmlsZW5hbWUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHQgdG8gJ3ByZWRpY3Rpb24nLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIGxvYWRpbmcgdGhlIG1vZGVsLCBhbmQgZ2V0dGluZyB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbChtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQpCgogICAgIyBGaXggZmVhdHVyZSBuYW1lcyBmb3IgbW9kZWxzIHRoYXQgcmVxdWlyZSB0aGVtIChlLmcuLCBYR0Jvb3N0KQogICAgIyBXaGVuIGRhdGFzZXQgY29tZXMgZnJvbSBhIGxpc3QsIHBhbmRhcyBhc3NpZ25zIGRlZmF1bHQgaW50ZWdlciBjb2x1bW4gbmFtZXMKICAgICMgYnV0IHNvbWUgbW9kZWxzIGV4cGVjdCBzcGVjaWZpYyBmZWF0dXJlIG5hbWVzIHRoZXkgd2VyZSB0cmFpbmVkIHdpdGgKICAgIGlmIGhhc2F0dHIobW9kZWxfaGFuZGxlci5tb2RlbCwgJ2ZlYXR1cmVfbmFtZXNfaW5fJyk6CiAgICAgICAgZXhwZWN0ZWRfZmVhdHVyZXMgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLmZlYXR1cmVfbmFtZXNfaW5fCiAgICAgICAgaWYgbGVuKGRhdGFzZXQuY29sdW1ucykgPT0gbGVuKGV4cGVjdGVkX2ZlYXR1cmVzKToKICAgICAgICAgICAgIyBPbmx5IHJlbmFtZSBpZiB0aGUgbnVtYmVyIG9mIGNvbHVtbnMgbWF0Y2hlcwogICAgICAgICAgICAjIFRoaXMgaGFuZGxlcyB0aGUgY2FzZSB3aGVyZSBhIGxpc3Qgd2FzIGNvbnZlcnRlZCB0byBEYXRhRnJhbWUgd2l0aCBkZWZhdWx0IGNvbHVtbiBuYW1lcwogICAgICAgICAgICBpZiBub3QgYWxsKGNvbCA9PSBmZWF0IGZvciBjb2wsIGZlYXQgaW4gemlwKGRhdGFzZXQuY29sdW1ucywgZXhwZWN0ZWRfZmVhdHVyZXMpKToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSZW5hbWluZyBkYXRhc2V0IGNvbHVtbnMgdG8gbWF0Y2ggbW9kZWwncyBleHBlY3RlZCBmZWF0dXJlIG5hbWVzIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZGF0YXNldC5jb2x1bW5zID0gZXhwZWN0ZWRfZmVhdHVyZXMKCiAgICAjIERyb3BwaW5nIGxhYmVsIGNvbHVtbnMgaWYgbmVjZXNzYXJ5OgogICAgaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtdCiAgICBlbGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgc3RyKToKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW2xhYmVsX2NvbHVtbnNdCgogICAgIyBQcmVkaWN0aW5nOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIm1ha2luZyBwcmVkaWN0aW9uIGJ5ICd7bW9kZWxfaGFuZGxlci5tb2RlbF9uYW1lfSciKQogICAgeV9wcmVkID0gbW9kZWxfaGFuZGxlci5tb2RlbC5wcmVkaWN0KGRhdGFzZXQsICoqa3dhcmdzKQoKICAgICMgUHJlcGFyaW5nIGFuZCB2YWxpZGF0aW5nIGxhYmVsIGNvbHVtbnMgZm9yIHRoZSBkYXRhZnJhbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0OgogICAgbnVtX3ByZWRpY3RlZCA9IDEgaWYgbGVuKHlfcHJlZC5zaGFwZSkgPT0gMSBlbHNlIHlfcHJlZC5zaGFwZVsxXQoKICAgIGlmIG51bV9wcmVkaWN0ZWQgPiBsZW4obGFiZWxfY29sdW1ucyk6CiAgICAgICAgaWYgbnVtX3ByZWRpY3RlZCA9PSAxOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWyJwcmVkaWN0ZWQgbGFiZWxzIl0KICAgICAgICBlbHNlOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zLmV4dGVuZCgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICBmInByZWRpY3RlZF9sYWJlbF97aSArIDEgKyBsZW4obGFiZWxfY29sdW1ucyl9IgogICAgICAgICAgICAgICAgICAgIGZvciBpIGluIHJhbmdlKG51bV9wcmVkaWN0ZWQgLSBsZW4obGFiZWxfY29sdW1ucykpCiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgICkKICAgIGVsaWYgbnVtX3ByZWRpY3RlZCA8IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJudW1iZXIgb2YgcHJlZGljdGVkIGxhYmVsczoge251bV9wcmVkaWN0ZWR9IGlzIHNtYWxsZXIgdGhhbiBudW1iZXIgb2YgbGFiZWwgY29sdW1uczoge2xlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBhcnRpZmFjdF9uYW1lID0gcmVzdWx0X3NldCBvciAicHJlZGljdGlvbiIKICAgIGxhYmVsc19pbnNpZGVfZGYgPSBzZXQobGFiZWxfY29sdW1ucykgJiBzZXQoZGF0YXNldC5jb2x1bW5zLnRvbGlzdCgpKQogICAgaWYgbGFiZWxzX2luc2lkZV9kZjoKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJUaGUgbGFiZWxzOiB7bGFiZWxzX2luc2lkZV9kZn0gYXJlIGFscmVhZHkgZXhpc3RlZCBpbiB0aGUgZGF0YWZyYW1lIgogICAgICAgICkKICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICBwcmVkX2RmID0gcGQuY29uY2F0KFtkYXRhc2V0LCBwZC5EYXRhRnJhbWUoeV9wcmVkLCBjb2x1bW5zPWxhYmVsX2NvbHVtbnMpXSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldChhcnRpZmFjdF9uYW1lLCBwcmVkX2RmLCBkYl9rZXk9cmVzdWx0X3NldCkK - default_handler: train + code_origin: '' disable_auto_mount: false - filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/auto_trainer/auto_trainer.py - image: mlrun/mlrun kind: job verbose: false diff --git a/functions/src/auto_trainer/requirements.txt b/functions/src/auto_trainer/requirements.txt index 274a97f82..80346561b 100644 --- a/functions/src/auto_trainer/requirements.txt +++ b/functions/src/auto_trainer/requirements.txt @@ -1,4 +1,4 @@ pandas -scikit-learn==1.5.2 +scikit-learn==1.4.2 xgboost<2.0.0 plotly From dc2a7663682984095f8023b8f5ed52c1cb767545 Mon Sep 17 00:00:00 2001 From: tomerbv Date: Wed, 21 Jan 2026 11:44:50 +0200 Subject: [PATCH 10/15] revert scikit-learn<1.4.0 --- functions/src/auto_trainer/function.yaml | 60 ++++++++++----------- functions/src/auto_trainer/item.yaml | 2 +- functions/src/auto_trainer/requirements.txt | 2 +- 3 files changed, 32 insertions(+), 32 deletions(-) diff --git a/functions/src/auto_trainer/function.yaml b/functions/src/auto_trainer/function.yaml index 04b9975c1..c879673f6 100644 --- a/functions/src/auto_trainer/function.yaml +++ b/functions/src/auto_trainer/function.yaml @@ -1,13 +1,32 @@ +kind: job metadata: - tag: '' name: auto-trainer categories: - machine-learning - model-training + tag: '' +verbose: false spec: - default_handler: train + image: mlrun/mlrun + description: Automatic train, evaluate and predict functions for the ML frameworks + - Scikit-Learn, XGBoost and LightGBM. + disable_auto_mount: false + filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/auto_trainer/auto_trainer.py entry_points: train: + name: train + lineno: 126 + doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ + \ project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ + hub://auto_trainer\", \"train\")\n trainer_run = project.run(\n \ + \ name=\"train\",\n handler=\"train\",\n inputs={\"dataset\"\ + : \"./path/to/dataset.csv\"},\n params={\n \"model_class\"\ + : \"sklearn.linear_model.LogisticRegression\",\n \"label_columns\"\ + : \"label\",\n \"drop_columns\": \"id\",\n \"model_name\"\ + : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ + : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ + ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" + has_varargs: false parameters: - name: context type: MLClientCtx @@ -61,21 +80,12 @@ spec: type: dict doc: Labels to log with the model default: null - name: train has_kwargs: true - lineno: 126 - doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ - \ project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ - hub://auto_trainer\", \"train\")\n trainer_run = project.run(\n \ - \ name=\"train\",\n handler=\"train\",\n inputs={\"dataset\"\ - : \"./path/to/dataset.csv\"},\n params={\n \"model_class\"\ - : \"sklearn.linear_model.LogisticRegression\",\n \"label_columns\"\ - : \"label\",\n \"drop_columns\": \"id\",\n \"model_name\"\ - : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ - : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ - ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" - has_varargs: false evaluate: + name: evaluate + lineno: 278 + doc: Evaluating a model. Artifacts generated by the MLHandler. + has_varargs: false parameters: - name: context type: MLClientCtx @@ -95,12 +105,12 @@ spec: doc: The target label(s) of the column(s) in the dataset. for Regression or Classification tasks. Mandatory when dataset is not a FeatureVector. default: null - name: evaluate has_kwargs: true - lineno: 278 - doc: Evaluating a model. Artifacts generated by the MLHandler. - has_varargs: false predict: + name: predict + lineno: 332 + doc: Predicting dataset by a model. + has_varargs: false parameters: - name: context type: MLClientCtx @@ -129,20 +139,10 @@ spec: doc: The db key to set name of the prediction result and the filename. Default to 'prediction'. default: null - name: predict has_kwargs: true - lineno: 332 - doc: Predicting dataset by a model. - has_varargs: false command: '' - image: mlrun/mlrun - description: Automatic train, evaluate and predict functions for the ML frameworks - - Scikit-Learn, XGBoost and LightGBM. - filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/auto_trainer/auto_trainer.py + default_handler: train build: origin_filename: '' functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICAjIENoZWNrIGlmIGRhdGFzZXQgaXMgbGlzdC9kaWN0IGZpcnN0IChiZWZvcmUgdHJ5aW5nIHRvIGFjY2VzcyBhcnRpZmFjdF91cmwpCiAgICBpZiBpc2luc3RhbmNlKGRhdGFzZXQsIChsaXN0LCBkaWN0KSk6CiAgICAgICAgIyBsaXN0L2RpY3QgY2FzZToKICAgICAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICJsYWJlbF9jb2x1bW5zIG5vdCBwcm92aWRlZCwgbWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yIgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICBkYXRhc2V0ID0gcGQuRGF0YUZyYW1lKGRhdGFzZXQpCiAgICAgICAgIyBDaGVja2luZyBpZiBkcm9wX2NvbHVtbnMgcHJvdmlkZWQgYnkgaW50ZWdlciB0eXBlOgogICAgICAgIGlmIGRyb3BfY29sdW1uczoKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIHN0cikgb3IgKAogICAgICAgICAgICAgICAgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIGxpc3QpCiAgICAgICAgICAgICAgICBhbmQgYW55KGlzaW5zdGFuY2UoY29sLCBzdHIpIGZvciBjb2wgaW4gZHJvcF9jb2x1bW5zKQogICAgICAgICAgICApOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyBtdXN0IGJlIGFuIGludGVnZXIvbGlzdCBvZiBpbnRlZ2VycyBpZiBub3QgcHJvdmlkZWQgd2l0aCBhIFVSSS9GZWF0dXJlVmVjdG9yIGRhdGFzZXQiCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICAgICAgICAgIGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSwgaW5wbGFjZT1UcnVlKQogICAgZWxzZToKICAgICAgICAjIERhdGFzZXQgaXMgYSBEYXRhSXRlbSB3aXRoIGFydGlmYWN0X3VybCAoVVJJIG9yIEZlYXR1cmVWZWN0b3IpCiAgICAgICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGF0YXNldC5hcnRpZmFjdF91cmwpCgogICAgICAgICMgR2V0dGluZyB0aGUgZGF0YXNldDoKICAgICAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBsYWJlbF9jb2x1bW5zIG9yIGRhdGFzZXQubWV0YS5zdGF0dXMubGFiZWxfY29sdW1uCiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJsYWJlbCBjb2x1bW5zOiB7bGFiZWxfY29sdW1uc30iKQogICAgICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnYgPSBtbHJ1bi5kYXRhc3RvcmUuZ2V0X3N0b3JlX3Jlc291cmNlKGRhdGFzZXQuYXJ0aWZhY3RfdXJsKQogICAgICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgICAgIGV4Y2VwdCBBdHRyaWJ1dGVFcnJvcjoKICAgICAgICAgICAgICAgICMgTGVhdmUgaGVyZSBmb3IgYmFja3dhcmRzIGNvbXBhdGliaWxpdHkKICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgICAgICBkYXRhc2V0Lm1ldGEudXJpLCBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zCiAgICAgICAgICAgICAgICApLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgIyBzaW1wbGUgVVJMIGNhc2U6CiAgICAgICAgICAgIGlmIG5vdCBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyBub3QgcHJvdmlkZWQsIG1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3RvciIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuYXNfZGYoKQogICAgICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBkYXRhc2V0LmRyb3AoZHJvcF9jb2x1bW5zLCBheGlzPTEpCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgICAgICJub3QgYWxsIG9mIHRoZSBjb2x1bW5zIHRvIGRyb3AgaW4gdGhlIGRhdGFzZXQsIGRyb3AgY29sdW1ucyBwcm9jZXNzIHNraXBwZWQiCiAgICAgICAgICAgICAgICAgICAgKQoKICAgIHJldHVybiBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zCgoKZGVmIHRyYWluKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIG1vZGVsX2NsYXNzOiBzdHIsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIG1vZGVsX25hbWU6IHN0ciA9ICJtb2RlbCIsCiAgICB0YWc6IHN0ciA9ICIiLAogICAgc2FtcGxlX3NldDogRGF0YUl0ZW0gPSBOb25lLAogICAgdGVzdF9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZTogZmxvYXQgPSBOb25lLAogICAgcmFuZG9tX3N0YXRlOiBpbnQgPSBOb25lLAogICAgbGFiZWxzOiBkaWN0ID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBUcmFpbmluZyBhIG1vZGVsIHdpdGggdGhlIGdpdmVuIGRhdGFzZXQuCgogICAgZXhhbXBsZTo6CgogICAgICAgIGltcG9ydCBtbHJ1bgogICAgICAgIHByb2plY3QgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX3Byb2plY3QoIm15LXByb2plY3QiKQogICAgICAgIHByb2plY3Quc2V0X2Z1bmN0aW9uKCJodWI6Ly9hdXRvX3RyYWluZXIiLCAidHJhaW4iKQogICAgICAgIHRyYWluZXJfcnVuID0gcHJvamVjdC5ydW4oCiAgICAgICAgICAgIG5hbWU9InRyYWluIiwKICAgICAgICAgICAgaGFuZGxlcj0idHJhaW4iLAogICAgICAgICAgICBpbnB1dHM9eyJkYXRhc2V0IjogIi4vcGF0aC90by9kYXRhc2V0LmNzdiJ9LAogICAgICAgICAgICBwYXJhbXM9ewogICAgICAgICAgICAgICAgIm1vZGVsX2NsYXNzIjogInNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyI6ICJsYWJlbCIsCiAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIjogImlkIiwKICAgICAgICAgICAgICAgICJtb2RlbF9uYW1lIjogIm15LW1vZGVsIiwKICAgICAgICAgICAgICAgICJ0YWciOiAidjEuMC4wIiwKICAgICAgICAgICAgICAgICJzYW1wbGVfc2V0IjogIi4vcGF0aC90by9zYW1wbGVfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAidGVzdF9zZXQiOiAiLi9wYXRoL3RvL3Rlc3Rfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAiQ0xBU1Nfc29sdmVyIjogImxpYmxpbmVhciIsCiAgICAgICAgICAgIH0sCiAgICAgICAgKQoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dAogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICAgICAgICAgIFRoZSBjbGFzcyBvZiB0aGUgbW9kZWwsIGUuZy4gYHNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbmAKICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcAogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICAgICBUaGUgbW9kZWwncyBuYW1lIHRvIHVzZSBmb3Igc3RvcmluZyB0aGUgbW9kZWwgYXJ0aWZhY3QsIGRlZmF1bHQgdG8gJ21vZGVsJwogICAgOnBhcmFtIHRhZzogICAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwncyB0YWcgdG8gbG9nIHdpdGgKICAgIDpwYXJhbSBzYW1wbGVfc2V0OiAgICAgICAgICAgICAgQSBzYW1wbGUgc2V0IG9mIGlucHV0cyBmb3IgdGhlIG1vZGVsIGZvciBsb2dnaW5nIGl0cyBzdGF0cyBhbG9uZyB0aGUgbW9kZWwgaW4gZmF2b3VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9mIG1vZGVsIG1vbml0b3JpbmcuIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gdGVzdF9zZXQ6ICAgICAgICAgICAgICAgIFRoZSB0ZXN0IHNldCB0byB0cmFpbiB0aGUgbW9kZWwgd2l0aC4KICAgIDpwYXJhbSB0cmFpbl90ZXN0X3NwbGl0X3NpemU6ICAgaWYgdGVzdF9zZXQgd2FzIHByb3ZpZGVkIHRoZW4gdGhpcyBhcmd1bWVudCBpcyBpZ25vcmVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTaG91bGQgYmUgYmV0d2VlbiAwLjAgYW5kIDEuMCBhbmQgcmVwcmVzZW50IHRoZSBwcm9wb3J0aW9uIG9mIHRoZSBkYXRhc2V0IHRvIGluY2x1ZGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaW4gdGhlIHRlc3Qgc3BsaXQuIFRoZSBzaXplIG9mIHRoZSBUcmFpbmluZyBzZXQgaXMgc2V0IHRvIHRoZSBjb21wbGVtZW50IG9mIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWUuIERlZmF1bHQgPSAwLjIKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgUmVsZXZhbnQgb25seSB3aGVuIHVzaW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQSByYW5kb20gc3RhdGUgc2VlZCB0byBzaHVmZmxlIHRoZSBkYXRhLiBGb3IgbW9yZSBpbmZvcm1hdGlvbiwgc2VlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL3NjaWtpdC1sZWFybi5vcmcvc3RhYmxlL2dsb3NzYXJ5Lmh0bWwjdGVybS1yYW5kb21fc3RhdGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTm90aWNlIHRoYXQgaGVyZSB3ZSBvbmx5IHBhc3MgaW50ZWdlciB2YWx1ZXMuCiAgICA6cGFyYW0gbGFiZWxzOiAgICAgICAgICAgICAgICAgIExhYmVscyB0byBsb2cgd2l0aCB0aGUgbW9kZWwKICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgd2l0aCBwcmVmaXhlcywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCB3aWxsIGJlIHBhcnNlZCBhbmQgcGFzc2VkIHRvIHRoZSByZWxldmFudCBmdW5jdGlvbiwgYnkgdGhlIGZvbGxvd2luZyBwcmVmaXhlczoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgQ0xBU1NfYCAtIGZvciB0aGUgbW9kZWwgY2xhc3MgYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYEZJVF9gIC0gZm9yIHRoZSBgZml0YCBmdW5jdGlvbiBhcmd1bWVudHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgVFJBSU5fYCAtIGZvciB0aGUgYHRyYWluYCBmdW5jdGlvbiAoaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24gLSBmdXR1cmUpCgogICAgIiIiCiAgICAjIFZhbGlkYXRlIGlucHV0czoKICAgICMgQ2hlY2sgaWYgZXhhY3RseSBvbmUgb2YgdGhlbSBpcyBzdXBwbGllZDoKICAgIGlmIHRlc3Rfc2V0IGlzIE5vbmU6CiAgICAgICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGlzIE5vbmU6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAidGVzdF9zZXQgb3IgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGFyZSBub3QgcHJvdmlkZWQsIHNldHRpbmcgdHJhaW5fdGVzdF9zcGxpdF9zaXplIHRvIDAuMiIKICAgICAgICAgICAgKQogICAgICAgICAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemUgPSAwLjIKCiAgICBlbGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAidGVzdF9zZXQgcHJvdmlkZWQsIGlnbm9yaW5nIGdpdmVuIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB2YWx1ZSIKICAgICAgICApCiAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gTm9uZQoKICAgICMgR2V0IERhdGFGcmFtZSBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgR2V0dGluZyB0aGUgc2FtcGxlIHNldDoKICAgIGlmIHNhbXBsZV9zZXQgaXMgTm9uZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIlNhbXBsZSBzZXQgbm90IGdpdmVuLCB1c2luZyB0aGUgd2hvbGUgdHJhaW5pbmcgc2V0IGFzIHRoZSBzYW1wbGUgc2V0IgogICAgICAgICkKICAgICAgICBzYW1wbGVfc2V0ID0gZGF0YXNldAogICAgZWxzZToKICAgICAgICBzYW1wbGVfc2V0LCBfID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICAgICAgZGF0YXNldD1zYW1wbGVfc2V0LAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICAgICAgKQoKICAgICMgUGFyc2luZyBrd2FyZ3M6CiAgICAjIFRPRE86IFVzZSBpbiB4Z2Igb3IgbGdibSB0cmFpbiBmdW5jdGlvbi4KICAgIHRyYWluX2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuVFJBSU4pCiAgICBmaXRfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5GSVQpCiAgICBtb2RlbF9jbGFzc19rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeCgKICAgICAgICBzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLk1PREVMX0NMQVNTCiAgICApCgogICAgIyBDaGVjayBpZiBtb2RlbCBvciBmdW5jdGlvbjoKICAgIGlmIGhhc2F0dHIobW9kZWxfY2xhc3MsICJ0cmFpbiIpOgogICAgICAgICMgVE9ETzogTmVlZCB0byBjYWxsOiBtb2RlbCgpLCBhZnRlcndhcmRzIHRvIHN0YXJ0IHRoZSB0cmFpbiBmdW5jdGlvbi4KICAgICAgICAjIG1vZGVsID0gY3JlYXRlX2Z1bmN0aW9uKGYie21vZGVsX2NsYXNzfS50cmFpbiIpCiAgICAgICAgcmFpc2UgTm90SW1wbGVtZW50ZWRFcnJvcgogICAgZWxzZToKICAgICAgICAjIENyZWF0aW5nIG1vZGVsIGluc3RhbmNlOgogICAgICAgIG1vZGVsID0gY3JlYXRlX2NsYXNzKG1vZGVsX2NsYXNzKSgqKm1vZGVsX2NsYXNzX2t3YXJncykKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCiAgICBpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgeF90cmFpbiwgeF90ZXN0LCB5X3RyYWluLCB5X3Rlc3QgPSB0cmFpbl90ZXN0X3NwbGl0KAogICAgICAgICAgICB4LCB5LCB0ZXN0X3NpemU9dHJhaW5fdGVzdF9zcGxpdF9zaXplLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICB4X3RyYWluLCB5X3RyYWluID0geCwgeQoKICAgICAgICB0ZXN0X3NldCA9IHRlc3Rfc2V0LmFzX2RmKCkKICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgIHRlc3Rfc2V0ID0gZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xKQoKICAgICAgICB4X3Rlc3QsIHlfdGVzdCA9IHRlc3Rfc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKSwgdGVzdF9zZXRbbGFiZWxfY29sdW1uc10KCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4oCiAgICAgICAgbW9kZWw9bW9kZWwsCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICB0YWc9dGFnLAogICAgICAgIHNhbXBsZV9zZXQ9c2FtcGxlX3NldCwKICAgICAgICB5X2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICB0ZXN0X3NldD10ZXN0X3NldCwKICAgICAgICB4X3Rlc3Q9eF90ZXN0LAogICAgICAgIHlfdGVzdD15X3Rlc3QsCiAgICAgICAgYXJ0aWZhY3RzPWNvbnRleHQuYXJ0aWZhY3RzLAogICAgICAgIGxhYmVscz1sYWJlbHMsCiAgICApCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidHJhaW5pbmcgJ3ttb2RlbF9uYW1lfSciKQogICAgbW9kZWwuZml0KHhfdHJhaW4sIHlfdHJhaW4sICoqZml0X2t3YXJncykKCgpkZWYgZXZhbHVhdGUoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIEV2YWx1YXRpbmcgYSBtb2RlbC4gQXJ0aWZhY3RzIGdlbmVyYXRlZCBieSB0aGUgTUxIYW5kbGVyLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIGV2YWx1YXRlIHRoZSBtb2RlbCBvbi4gQ2FuIGJlIGVpdGhlciBhIFVSSSBvciBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICAgICAgICAgIHN0ciBvciBhIGxpc3Qgb2Ygc3RyaW5ncyB0aGF0IHJlcHJlc2VudCB0aGUgY29sdW1ucyB0byBkcm9wLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHRvIHRoZSBwcmVkaWN0IGZ1bmN0aW9uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChQUkVESUNUXyBwcmVmaXggaXMgbm90IHJlcXVpcmVkKS4KICAgICIiIgogICAgIyBHZXQgZGF0YXNldCBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgUGFyc2luZyBsYWJlbF9jb2x1bW5zOgogICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMgPSBbXQogICAgaWYgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gKAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zIGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgbGlzdCkgZWxzZSBbbGFiZWxfY29sdW1uc10KICAgICAgICApCiAgICAgICAgZm9yIGxjIGluIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGlmIGZzLmNvbW1vbi5mZWF0dXJlX3NlcGFyYXRvciBpbiBsYzoKICAgICAgICAgICAgICAgIGZlYXR1cmVfc2V0X25hbWUsIGxhYmVsX25hbWUsIGFsaWFzID0gZnMuY29tbW9uLnBhcnNlX2ZlYXR1cmVfc3RyaW5nKGxjKQogICAgICAgICAgICAgICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMuYXBwZW5kKGFsaWFzIG9yIGxhYmVsX25hbWUpCiAgICAgICAgaWYgcGFyc2VkX2xhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBwYXJzZWRfbGFiZWxfY29sdW1ucwoKICAgIHggPSBkYXRhc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKQogICAgeSA9IGRhdGFzZXRbbGFiZWxfY29sdW1uc10KCiAgICAjIExvYWRpbmcgdGhlIG1vZGVsIGFuZCBwcmVkaWN0aW5nOgogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWwsIGNvbnRleHQ9Y29udGV4dCwgbW9kZWxfbmFtZT0ibW9kZWxfTGluZWFyUmVncmVzc2lvbiIKICAgICkKICAgIEF1dG9NTFJ1bi5hcHBseV9tbHJ1bihtb2RlbF9oYW5kbGVyLm1vZGVsLCB5X3Rlc3Q9eSwgbW9kZWxfcGF0aD1tb2RlbCkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiZXZhbHVhdGluZyAne21vZGVsX2hhbmRsZXIubW9kZWxfbmFtZX0nIikKICAgIG1vZGVsX2hhbmRsZXIubW9kZWwucHJlZGljdCh4LCAqKmt3YXJncykKCgpkZWYgcHJlZGljdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWw6IHN0ciwKICAgIGRhdGFzZXQ6IG1scnVuLkRhdGFJdGVtLAogICAgZHJvcF9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXSwgaW50LCBMaXN0W2ludF1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgcmVzdWx0X3NldDogT3B0aW9uYWxbc3RyXSA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgUHJlZGljdGluZyBkYXRhc2V0IGJ5IGEgbW9kZWwuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwgU3RvcmUgcGF0aC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gcHJlZGljdCB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkksIGEgRmVhdHVyZVZlY3RvciBvciBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSBpbiBhIHNoYXBlIG9mIGEgbGlzdC9kaWN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHBhc3NpbmcgYSBzYW1wbGUsIHBhc3MgdGhlIGRhdGFzZXQgYXMgYSBmaWVsZCBpbiBgcGFyYW1zYCBpbnN0ZWFkIG9mIGBpbnB1dHNgLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIvaW50IG9yIGEgbGlzdCBvZiBzdHJpbmdzL2ludHMgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbiBuYW1lcy9pbmRpY2VzIHRvIGRyb3AuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFdoZW4gdGhlIGRhdGFzZXQgaXMgYSBsaXN0L2RpY3QgdGhpcyBwYXJhbWV0ZXIgc2hvdWxkIGJlIHJlcHJlc2VudGVkIGJ5IGludGVnZXJzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gcmVzdWx0X3NldDogICAgICAgICAgICAgIFRoZSBkYiBrZXkgdG8gc2V0IG5hbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0IGFuZCB0aGUgZmlsZW5hbWUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHQgdG8gJ3ByZWRpY3Rpb24nLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIGxvYWRpbmcgdGhlIG1vZGVsLCBhbmQgZ2V0dGluZyB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbChtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQpCgogICAgIyBGaXggZmVhdHVyZSBuYW1lcyBmb3IgbW9kZWxzIHRoYXQgcmVxdWlyZSB0aGVtIChlLmcuLCBYR0Jvb3N0KQogICAgIyBXaGVuIGRhdGFzZXQgY29tZXMgZnJvbSBhIGxpc3QsIHBhbmRhcyBhc3NpZ25zIGRlZmF1bHQgaW50ZWdlciBjb2x1bW4gbmFtZXMKICAgICMgYnV0IHNvbWUgbW9kZWxzIGV4cGVjdCBzcGVjaWZpYyBmZWF0dXJlIG5hbWVzIHRoZXkgd2VyZSB0cmFpbmVkIHdpdGgKICAgIGlmIGhhc2F0dHIobW9kZWxfaGFuZGxlci5tb2RlbCwgJ2ZlYXR1cmVfbmFtZXNfaW5fJyk6CiAgICAgICAgZXhwZWN0ZWRfZmVhdHVyZXMgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLmZlYXR1cmVfbmFtZXNfaW5fCiAgICAgICAgaWYgbGVuKGRhdGFzZXQuY29sdW1ucykgPT0gbGVuKGV4cGVjdGVkX2ZlYXR1cmVzKToKICAgICAgICAgICAgIyBPbmx5IHJlbmFtZSBpZiB0aGUgbnVtYmVyIG9mIGNvbHVtbnMgbWF0Y2hlcwogICAgICAgICAgICAjIFRoaXMgaGFuZGxlcyB0aGUgY2FzZSB3aGVyZSBhIGxpc3Qgd2FzIGNvbnZlcnRlZCB0byBEYXRhRnJhbWUgd2l0aCBkZWZhdWx0IGNvbHVtbiBuYW1lcwogICAgICAgICAgICBpZiBub3QgYWxsKGNvbCA9PSBmZWF0IGZvciBjb2wsIGZlYXQgaW4gemlwKGRhdGFzZXQuY29sdW1ucywgZXhwZWN0ZWRfZmVhdHVyZXMpKToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSZW5hbWluZyBkYXRhc2V0IGNvbHVtbnMgdG8gbWF0Y2ggbW9kZWwncyBleHBlY3RlZCBmZWF0dXJlIG5hbWVzIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZGF0YXNldC5jb2x1bW5zID0gZXhwZWN0ZWRfZmVhdHVyZXMKCiAgICAjIERyb3BwaW5nIGxhYmVsIGNvbHVtbnMgaWYgbmVjZXNzYXJ5OgogICAgaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtdCiAgICBlbGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgc3RyKToKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW2xhYmVsX2NvbHVtbnNdCgogICAgIyBQcmVkaWN0aW5nOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIm1ha2luZyBwcmVkaWN0aW9uIGJ5ICd7bW9kZWxfaGFuZGxlci5tb2RlbF9uYW1lfSciKQogICAgeV9wcmVkID0gbW9kZWxfaGFuZGxlci5tb2RlbC5wcmVkaWN0KGRhdGFzZXQsICoqa3dhcmdzKQoKICAgICMgUHJlcGFyaW5nIGFuZCB2YWxpZGF0aW5nIGxhYmVsIGNvbHVtbnMgZm9yIHRoZSBkYXRhZnJhbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0OgogICAgbnVtX3ByZWRpY3RlZCA9IDEgaWYgbGVuKHlfcHJlZC5zaGFwZSkgPT0gMSBlbHNlIHlfcHJlZC5zaGFwZVsxXQoKICAgIGlmIG51bV9wcmVkaWN0ZWQgPiBsZW4obGFiZWxfY29sdW1ucyk6CiAgICAgICAgaWYgbnVtX3ByZWRpY3RlZCA9PSAxOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWyJwcmVkaWN0ZWQgbGFiZWxzIl0KICAgICAgICBlbHNlOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zLmV4dGVuZCgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICBmInByZWRpY3RlZF9sYWJlbF97aSArIDEgKyBsZW4obGFiZWxfY29sdW1ucyl9IgogICAgICAgICAgICAgICAgICAgIGZvciBpIGluIHJhbmdlKG51bV9wcmVkaWN0ZWQgLSBsZW4obGFiZWxfY29sdW1ucykpCiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgICkKICAgIGVsaWYgbnVtX3ByZWRpY3RlZCA8IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJudW1iZXIgb2YgcHJlZGljdGVkIGxhYmVsczoge251bV9wcmVkaWN0ZWR9IGlzIHNtYWxsZXIgdGhhbiBudW1iZXIgb2YgbGFiZWwgY29sdW1uczoge2xlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBhcnRpZmFjdF9uYW1lID0gcmVzdWx0X3NldCBvciAicHJlZGljdGlvbiIKICAgIGxhYmVsc19pbnNpZGVfZGYgPSBzZXQobGFiZWxfY29sdW1ucykgJiBzZXQoZGF0YXNldC5jb2x1bW5zLnRvbGlzdCgpKQogICAgaWYgbGFiZWxzX2luc2lkZV9kZjoKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJUaGUgbGFiZWxzOiB7bGFiZWxzX2luc2lkZV9kZn0gYXJlIGFscmVhZHkgZXhpc3RlZCBpbiB0aGUgZGF0YWZyYW1lIgogICAgICAgICkKICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICBwcmVkX2RmID0gcGQuY29uY2F0KFtkYXRhc2V0LCBwZC5EYXRhRnJhbWUoeV9wcmVkLCBjb2x1bW5zPWxhYmVsX2NvbHVtbnMpXSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldChhcnRpZmFjdF9uYW1lLCBwcmVkX2RmLCBkYl9rZXk9cmVzdWx0X3NldCkK code_origin: '' - disable_auto_mount: false -kind: job -verbose: false diff --git a/functions/src/auto_trainer/item.yaml b/functions/src/auto_trainer/item.yaml index d397a79d6..78de92ca0 100755 --- a/functions/src/auto_trainer/item.yaml +++ b/functions/src/auto_trainer/item.yaml @@ -13,7 +13,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.10.0 +mlrunVersion: 1.7.0 name: auto_trainer platformVersion: 3.5.0 spec: diff --git a/functions/src/auto_trainer/requirements.txt b/functions/src/auto_trainer/requirements.txt index 80346561b..b14a0293c 100644 --- a/functions/src/auto_trainer/requirements.txt +++ b/functions/src/auto_trainer/requirements.txt @@ -1,4 +1,4 @@ pandas -scikit-learn==1.4.2 +scikit-learn<1.4.0 xgboost<2.0.0 plotly From 514f4a480098d24702785165baaad4676a187f1c Mon Sep 17 00:00:00 2001 From: tomerbv Date: Wed, 21 Jan 2026 11:48:57 +0200 Subject: [PATCH 11/15] scikit-learn~=1.5 --- functions/src/auto_trainer/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/src/auto_trainer/requirements.txt b/functions/src/auto_trainer/requirements.txt index b14a0293c..4854d84fd 100644 --- a/functions/src/auto_trainer/requirements.txt +++ b/functions/src/auto_trainer/requirements.txt @@ -1,4 +1,4 @@ pandas -scikit-learn<1.4.0 +scikit-learn~=1.5 xgboost<2.0.0 plotly From 1f76feb1fbce477f0419b1a2f59eb4f0d59f52cc Mon Sep 17 00:00:00 2001 From: tomerbv Date: Wed, 21 Jan 2026 11:58:13 +0200 Subject: [PATCH 12/15] mlrun 1.10 with scikit-learn<1.4.0 --- functions/src/auto_trainer/function.yaml | 47 ++++++++++----------- functions/src/auto_trainer/item.yaml | 2 +- functions/src/auto_trainer/requirements.txt | 2 +- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/functions/src/auto_trainer/function.yaml b/functions/src/auto_trainer/function.yaml index c879673f6..155e5c58e 100644 --- a/functions/src/auto_trainer/function.yaml +++ b/functions/src/auto_trainer/function.yaml @@ -1,21 +1,13 @@ kind: job -metadata: - name: auto-trainer - categories: - - machine-learning - - model-training - tag: '' verbose: false spec: - image: mlrun/mlrun - description: Automatic train, evaluate and predict functions for the ML frameworks - - Scikit-Learn, XGBoost and LightGBM. disable_auto_mount: false - filename: /Users/Tomer_Weitzman/PycharmProjects/functions/functions/src/auto_trainer/auto_trainer.py + build: + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICAjIENoZWNrIGlmIGRhdGFzZXQgaXMgbGlzdC9kaWN0IGZpcnN0IChiZWZvcmUgdHJ5aW5nIHRvIGFjY2VzcyBhcnRpZmFjdF91cmwpCiAgICBpZiBpc2luc3RhbmNlKGRhdGFzZXQsIChsaXN0LCBkaWN0KSk6CiAgICAgICAgIyBsaXN0L2RpY3QgY2FzZToKICAgICAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICJsYWJlbF9jb2x1bW5zIG5vdCBwcm92aWRlZCwgbWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yIgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICBkYXRhc2V0ID0gcGQuRGF0YUZyYW1lKGRhdGFzZXQpCiAgICAgICAgIyBDaGVja2luZyBpZiBkcm9wX2NvbHVtbnMgcHJvdmlkZWQgYnkgaW50ZWdlciB0eXBlOgogICAgICAgIGlmIGRyb3BfY29sdW1uczoKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIHN0cikgb3IgKAogICAgICAgICAgICAgICAgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIGxpc3QpCiAgICAgICAgICAgICAgICBhbmQgYW55KGlzaW5zdGFuY2UoY29sLCBzdHIpIGZvciBjb2wgaW4gZHJvcF9jb2x1bW5zKQogICAgICAgICAgICApOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyBtdXN0IGJlIGFuIGludGVnZXIvbGlzdCBvZiBpbnRlZ2VycyBpZiBub3QgcHJvdmlkZWQgd2l0aCBhIFVSSS9GZWF0dXJlVmVjdG9yIGRhdGFzZXQiCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICAgICAgICAgIGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSwgaW5wbGFjZT1UcnVlKQogICAgZWxzZToKICAgICAgICAjIERhdGFzZXQgaXMgYSBEYXRhSXRlbSB3aXRoIGFydGlmYWN0X3VybCAoVVJJIG9yIEZlYXR1cmVWZWN0b3IpCiAgICAgICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGF0YXNldC5hcnRpZmFjdF91cmwpCgogICAgICAgICMgR2V0dGluZyB0aGUgZGF0YXNldDoKICAgICAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBsYWJlbF9jb2x1bW5zIG9yIGRhdGFzZXQubWV0YS5zdGF0dXMubGFiZWxfY29sdW1uCiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJsYWJlbCBjb2x1bW5zOiB7bGFiZWxfY29sdW1uc30iKQogICAgICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnYgPSBtbHJ1bi5kYXRhc3RvcmUuZ2V0X3N0b3JlX3Jlc291cmNlKGRhdGFzZXQuYXJ0aWZhY3RfdXJsKQogICAgICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgICAgIGV4Y2VwdCBBdHRyaWJ1dGVFcnJvcjoKICAgICAgICAgICAgICAgICMgTGVhdmUgaGVyZSBmb3IgYmFja3dhcmRzIGNvbXBhdGliaWxpdHkKICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgICAgICBkYXRhc2V0Lm1ldGEudXJpLCBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zCiAgICAgICAgICAgICAgICApLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgIyBzaW1wbGUgVVJMIGNhc2U6CiAgICAgICAgICAgIGlmIG5vdCBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyBub3QgcHJvdmlkZWQsIG1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3RvciIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuYXNfZGYoKQogICAgICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBkYXRhc2V0LmRyb3AoZHJvcF9jb2x1bW5zLCBheGlzPTEpCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgICAgICJub3QgYWxsIG9mIHRoZSBjb2x1bW5zIHRvIGRyb3AgaW4gdGhlIGRhdGFzZXQsIGRyb3AgY29sdW1ucyBwcm9jZXNzIHNraXBwZWQiCiAgICAgICAgICAgICAgICAgICAgKQoKICAgIHJldHVybiBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zCgoKZGVmIHRyYWluKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIG1vZGVsX2NsYXNzOiBzdHIsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIG1vZGVsX25hbWU6IHN0ciA9ICJtb2RlbCIsCiAgICB0YWc6IHN0ciA9ICIiLAogICAgc2FtcGxlX3NldDogRGF0YUl0ZW0gPSBOb25lLAogICAgdGVzdF9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZTogZmxvYXQgPSBOb25lLAogICAgcmFuZG9tX3N0YXRlOiBpbnQgPSBOb25lLAogICAgbGFiZWxzOiBkaWN0ID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBUcmFpbmluZyBhIG1vZGVsIHdpdGggdGhlIGdpdmVuIGRhdGFzZXQuCgogICAgZXhhbXBsZTo6CgogICAgICAgIGltcG9ydCBtbHJ1bgogICAgICAgIHByb2plY3QgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX3Byb2plY3QoIm15LXByb2plY3QiKQogICAgICAgIHByb2plY3Quc2V0X2Z1bmN0aW9uKCJodWI6Ly9hdXRvX3RyYWluZXIiLCAidHJhaW4iKQogICAgICAgIHRyYWluZXJfcnVuID0gcHJvamVjdC5ydW4oCiAgICAgICAgICAgIG5hbWU9InRyYWluIiwKICAgICAgICAgICAgaGFuZGxlcj0idHJhaW4iLAogICAgICAgICAgICBpbnB1dHM9eyJkYXRhc2V0IjogIi4vcGF0aC90by9kYXRhc2V0LmNzdiJ9LAogICAgICAgICAgICBwYXJhbXM9ewogICAgICAgICAgICAgICAgIm1vZGVsX2NsYXNzIjogInNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyI6ICJsYWJlbCIsCiAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIjogImlkIiwKICAgICAgICAgICAgICAgICJtb2RlbF9uYW1lIjogIm15LW1vZGVsIiwKICAgICAgICAgICAgICAgICJ0YWciOiAidjEuMC4wIiwKICAgICAgICAgICAgICAgICJzYW1wbGVfc2V0IjogIi4vcGF0aC90by9zYW1wbGVfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAidGVzdF9zZXQiOiAiLi9wYXRoL3RvL3Rlc3Rfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAiQ0xBU1Nfc29sdmVyIjogImxpYmxpbmVhciIsCiAgICAgICAgICAgIH0sCiAgICAgICAgKQoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dAogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICAgICAgICAgIFRoZSBjbGFzcyBvZiB0aGUgbW9kZWwsIGUuZy4gYHNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbmAKICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcAogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICAgICBUaGUgbW9kZWwncyBuYW1lIHRvIHVzZSBmb3Igc3RvcmluZyB0aGUgbW9kZWwgYXJ0aWZhY3QsIGRlZmF1bHQgdG8gJ21vZGVsJwogICAgOnBhcmFtIHRhZzogICAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwncyB0YWcgdG8gbG9nIHdpdGgKICAgIDpwYXJhbSBzYW1wbGVfc2V0OiAgICAgICAgICAgICAgQSBzYW1wbGUgc2V0IG9mIGlucHV0cyBmb3IgdGhlIG1vZGVsIGZvciBsb2dnaW5nIGl0cyBzdGF0cyBhbG9uZyB0aGUgbW9kZWwgaW4gZmF2b3VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9mIG1vZGVsIG1vbml0b3JpbmcuIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gdGVzdF9zZXQ6ICAgICAgICAgICAgICAgIFRoZSB0ZXN0IHNldCB0byB0cmFpbiB0aGUgbW9kZWwgd2l0aC4KICAgIDpwYXJhbSB0cmFpbl90ZXN0X3NwbGl0X3NpemU6ICAgaWYgdGVzdF9zZXQgd2FzIHByb3ZpZGVkIHRoZW4gdGhpcyBhcmd1bWVudCBpcyBpZ25vcmVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTaG91bGQgYmUgYmV0d2VlbiAwLjAgYW5kIDEuMCBhbmQgcmVwcmVzZW50IHRoZSBwcm9wb3J0aW9uIG9mIHRoZSBkYXRhc2V0IHRvIGluY2x1ZGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaW4gdGhlIHRlc3Qgc3BsaXQuIFRoZSBzaXplIG9mIHRoZSBUcmFpbmluZyBzZXQgaXMgc2V0IHRvIHRoZSBjb21wbGVtZW50IG9mIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWUuIERlZmF1bHQgPSAwLjIKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgUmVsZXZhbnQgb25seSB3aGVuIHVzaW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQSByYW5kb20gc3RhdGUgc2VlZCB0byBzaHVmZmxlIHRoZSBkYXRhLiBGb3IgbW9yZSBpbmZvcm1hdGlvbiwgc2VlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL3NjaWtpdC1sZWFybi5vcmcvc3RhYmxlL2dsb3NzYXJ5Lmh0bWwjdGVybS1yYW5kb21fc3RhdGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTm90aWNlIHRoYXQgaGVyZSB3ZSBvbmx5IHBhc3MgaW50ZWdlciB2YWx1ZXMuCiAgICA6cGFyYW0gbGFiZWxzOiAgICAgICAgICAgICAgICAgIExhYmVscyB0byBsb2cgd2l0aCB0aGUgbW9kZWwKICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgd2l0aCBwcmVmaXhlcywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCB3aWxsIGJlIHBhcnNlZCBhbmQgcGFzc2VkIHRvIHRoZSByZWxldmFudCBmdW5jdGlvbiwgYnkgdGhlIGZvbGxvd2luZyBwcmVmaXhlczoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgQ0xBU1NfYCAtIGZvciB0aGUgbW9kZWwgY2xhc3MgYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYEZJVF9gIC0gZm9yIHRoZSBgZml0YCBmdW5jdGlvbiBhcmd1bWVudHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgVFJBSU5fYCAtIGZvciB0aGUgYHRyYWluYCBmdW5jdGlvbiAoaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24gLSBmdXR1cmUpCgogICAgIiIiCiAgICAjIFZhbGlkYXRlIGlucHV0czoKICAgICMgQ2hlY2sgaWYgZXhhY3RseSBvbmUgb2YgdGhlbSBpcyBzdXBwbGllZDoKICAgIGlmIHRlc3Rfc2V0IGlzIE5vbmU6CiAgICAgICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGlzIE5vbmU6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAidGVzdF9zZXQgb3IgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGFyZSBub3QgcHJvdmlkZWQsIHNldHRpbmcgdHJhaW5fdGVzdF9zcGxpdF9zaXplIHRvIDAuMiIKICAgICAgICAgICAgKQogICAgICAgICAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemUgPSAwLjIKCiAgICBlbGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAidGVzdF9zZXQgcHJvdmlkZWQsIGlnbm9yaW5nIGdpdmVuIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB2YWx1ZSIKICAgICAgICApCiAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gTm9uZQoKICAgICMgR2V0IERhdGFGcmFtZSBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgR2V0dGluZyB0aGUgc2FtcGxlIHNldDoKICAgIGlmIHNhbXBsZV9zZXQgaXMgTm9uZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIlNhbXBsZSBzZXQgbm90IGdpdmVuLCB1c2luZyB0aGUgd2hvbGUgdHJhaW5pbmcgc2V0IGFzIHRoZSBzYW1wbGUgc2V0IgogICAgICAgICkKICAgICAgICBzYW1wbGVfc2V0ID0gZGF0YXNldAogICAgZWxzZToKICAgICAgICBzYW1wbGVfc2V0LCBfID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICAgICAgZGF0YXNldD1zYW1wbGVfc2V0LAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICAgICAgKQoKICAgICMgUGFyc2luZyBrd2FyZ3M6CiAgICAjIFRPRE86IFVzZSBpbiB4Z2Igb3IgbGdibSB0cmFpbiBmdW5jdGlvbi4KICAgIHRyYWluX2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuVFJBSU4pCiAgICBmaXRfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5GSVQpCiAgICBtb2RlbF9jbGFzc19rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeCgKICAgICAgICBzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLk1PREVMX0NMQVNTCiAgICApCgogICAgIyBDaGVjayBpZiBtb2RlbCBvciBmdW5jdGlvbjoKICAgIGlmIGhhc2F0dHIobW9kZWxfY2xhc3MsICJ0cmFpbiIpOgogICAgICAgICMgVE9ETzogTmVlZCB0byBjYWxsOiBtb2RlbCgpLCBhZnRlcndhcmRzIHRvIHN0YXJ0IHRoZSB0cmFpbiBmdW5jdGlvbi4KICAgICAgICAjIG1vZGVsID0gY3JlYXRlX2Z1bmN0aW9uKGYie21vZGVsX2NsYXNzfS50cmFpbiIpCiAgICAgICAgcmFpc2UgTm90SW1wbGVtZW50ZWRFcnJvcgogICAgZWxzZToKICAgICAgICAjIENyZWF0aW5nIG1vZGVsIGluc3RhbmNlOgogICAgICAgIG1vZGVsID0gY3JlYXRlX2NsYXNzKG1vZGVsX2NsYXNzKSgqKm1vZGVsX2NsYXNzX2t3YXJncykKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCiAgICBpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgeF90cmFpbiwgeF90ZXN0LCB5X3RyYWluLCB5X3Rlc3QgPSB0cmFpbl90ZXN0X3NwbGl0KAogICAgICAgICAgICB4LCB5LCB0ZXN0X3NpemU9dHJhaW5fdGVzdF9zcGxpdF9zaXplLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICB4X3RyYWluLCB5X3RyYWluID0geCwgeQoKICAgICAgICB0ZXN0X3NldCA9IHRlc3Rfc2V0LmFzX2RmKCkKICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgIHRlc3Rfc2V0ID0gZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xKQoKICAgICAgICB4X3Rlc3QsIHlfdGVzdCA9IHRlc3Rfc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKSwgdGVzdF9zZXRbbGFiZWxfY29sdW1uc10KCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4oCiAgICAgICAgbW9kZWw9bW9kZWwsCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICB0YWc9dGFnLAogICAgICAgIHNhbXBsZV9zZXQ9c2FtcGxlX3NldCwKICAgICAgICB5X2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICB0ZXN0X3NldD10ZXN0X3NldCwKICAgICAgICB4X3Rlc3Q9eF90ZXN0LAogICAgICAgIHlfdGVzdD15X3Rlc3QsCiAgICAgICAgYXJ0aWZhY3RzPWNvbnRleHQuYXJ0aWZhY3RzLAogICAgICAgIGxhYmVscz1sYWJlbHMsCiAgICApCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidHJhaW5pbmcgJ3ttb2RlbF9uYW1lfSciKQogICAgbW9kZWwuZml0KHhfdHJhaW4sIHlfdHJhaW4sICoqZml0X2t3YXJncykKCgpkZWYgZXZhbHVhdGUoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIEV2YWx1YXRpbmcgYSBtb2RlbC4gQXJ0aWZhY3RzIGdlbmVyYXRlZCBieSB0aGUgTUxIYW5kbGVyLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIGV2YWx1YXRlIHRoZSBtb2RlbCBvbi4gQ2FuIGJlIGVpdGhlciBhIFVSSSBvciBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICAgICAgICAgIHN0ciBvciBhIGxpc3Qgb2Ygc3RyaW5ncyB0aGF0IHJlcHJlc2VudCB0aGUgY29sdW1ucyB0byBkcm9wLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHRvIHRoZSBwcmVkaWN0IGZ1bmN0aW9uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChQUkVESUNUXyBwcmVmaXggaXMgbm90IHJlcXVpcmVkKS4KICAgICIiIgogICAgIyBHZXQgZGF0YXNldCBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgUGFyc2luZyBsYWJlbF9jb2x1bW5zOgogICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMgPSBbXQogICAgaWYgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gKAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zIGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgbGlzdCkgZWxzZSBbbGFiZWxfY29sdW1uc10KICAgICAgICApCiAgICAgICAgZm9yIGxjIGluIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGlmIGZzLmNvbW1vbi5mZWF0dXJlX3NlcGFyYXRvciBpbiBsYzoKICAgICAgICAgICAgICAgIGZlYXR1cmVfc2V0X25hbWUsIGxhYmVsX25hbWUsIGFsaWFzID0gZnMuY29tbW9uLnBhcnNlX2ZlYXR1cmVfc3RyaW5nKGxjKQogICAgICAgICAgICAgICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMuYXBwZW5kKGFsaWFzIG9yIGxhYmVsX25hbWUpCiAgICAgICAgaWYgcGFyc2VkX2xhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBwYXJzZWRfbGFiZWxfY29sdW1ucwoKICAgIHggPSBkYXRhc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKQogICAgeSA9IGRhdGFzZXRbbGFiZWxfY29sdW1uc10KCiAgICAjIExvYWRpbmcgdGhlIG1vZGVsIGFuZCBwcmVkaWN0aW5nOgogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWwsIGNvbnRleHQ9Y29udGV4dCwgbW9kZWxfbmFtZT0ibW9kZWxfTGluZWFyUmVncmVzc2lvbiIKICAgICkKICAgIEF1dG9NTFJ1bi5hcHBseV9tbHJ1bihtb2RlbF9oYW5kbGVyLm1vZGVsLCB5X3Rlc3Q9eSwgbW9kZWxfcGF0aD1tb2RlbCkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiZXZhbHVhdGluZyAne21vZGVsX2hhbmRsZXIubW9kZWxfbmFtZX0nIikKICAgIG1vZGVsX2hhbmRsZXIubW9kZWwucHJlZGljdCh4LCAqKmt3YXJncykKCgpkZWYgcHJlZGljdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWw6IHN0ciwKICAgIGRhdGFzZXQ6IG1scnVuLkRhdGFJdGVtLAogICAgZHJvcF9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXSwgaW50LCBMaXN0W2ludF1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgcmVzdWx0X3NldDogT3B0aW9uYWxbc3RyXSA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgUHJlZGljdGluZyBkYXRhc2V0IGJ5IGEgbW9kZWwuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwgU3RvcmUgcGF0aC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gcHJlZGljdCB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkksIGEgRmVhdHVyZVZlY3RvciBvciBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSBpbiBhIHNoYXBlIG9mIGEgbGlzdC9kaWN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHBhc3NpbmcgYSBzYW1wbGUsIHBhc3MgdGhlIGRhdGFzZXQgYXMgYSBmaWVsZCBpbiBgcGFyYW1zYCBpbnN0ZWFkIG9mIGBpbnB1dHNgLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIvaW50IG9yIGEgbGlzdCBvZiBzdHJpbmdzL2ludHMgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbiBuYW1lcy9pbmRpY2VzIHRvIGRyb3AuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFdoZW4gdGhlIGRhdGFzZXQgaXMgYSBsaXN0L2RpY3QgdGhpcyBwYXJhbWV0ZXIgc2hvdWxkIGJlIHJlcHJlc2VudGVkIGJ5IGludGVnZXJzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gcmVzdWx0X3NldDogICAgICAgICAgICAgIFRoZSBkYiBrZXkgdG8gc2V0IG5hbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0IGFuZCB0aGUgZmlsZW5hbWUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHQgdG8gJ3ByZWRpY3Rpb24nLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIGxvYWRpbmcgdGhlIG1vZGVsLCBhbmQgZ2V0dGluZyB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbChtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQpCgogICAgIyBGaXggZmVhdHVyZSBuYW1lcyBmb3IgbW9kZWxzIHRoYXQgcmVxdWlyZSB0aGVtIChlLmcuLCBYR0Jvb3N0KQogICAgIyBXaGVuIGRhdGFzZXQgY29tZXMgZnJvbSBhIGxpc3QsIHBhbmRhcyBhc3NpZ25zIGRlZmF1bHQgaW50ZWdlciBjb2x1bW4gbmFtZXMKICAgICMgYnV0IHNvbWUgbW9kZWxzIGV4cGVjdCBzcGVjaWZpYyBmZWF0dXJlIG5hbWVzIHRoZXkgd2VyZSB0cmFpbmVkIHdpdGgKICAgIGlmIGhhc2F0dHIobW9kZWxfaGFuZGxlci5tb2RlbCwgJ2ZlYXR1cmVfbmFtZXNfaW5fJyk6CiAgICAgICAgZXhwZWN0ZWRfZmVhdHVyZXMgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLmZlYXR1cmVfbmFtZXNfaW5fCiAgICAgICAgaWYgbGVuKGRhdGFzZXQuY29sdW1ucykgPT0gbGVuKGV4cGVjdGVkX2ZlYXR1cmVzKToKICAgICAgICAgICAgIyBPbmx5IHJlbmFtZSBpZiB0aGUgbnVtYmVyIG9mIGNvbHVtbnMgbWF0Y2hlcwogICAgICAgICAgICAjIFRoaXMgaGFuZGxlcyB0aGUgY2FzZSB3aGVyZSBhIGxpc3Qgd2FzIGNvbnZlcnRlZCB0byBEYXRhRnJhbWUgd2l0aCBkZWZhdWx0IGNvbHVtbiBuYW1lcwogICAgICAgICAgICBpZiBub3QgYWxsKGNvbCA9PSBmZWF0IGZvciBjb2wsIGZlYXQgaW4gemlwKGRhdGFzZXQuY29sdW1ucywgZXhwZWN0ZWRfZmVhdHVyZXMpKToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSZW5hbWluZyBkYXRhc2V0IGNvbHVtbnMgdG8gbWF0Y2ggbW9kZWwncyBleHBlY3RlZCBmZWF0dXJlIG5hbWVzIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZGF0YXNldC5jb2x1bW5zID0gZXhwZWN0ZWRfZmVhdHVyZXMKCiAgICAjIERyb3BwaW5nIGxhYmVsIGNvbHVtbnMgaWYgbmVjZXNzYXJ5OgogICAgaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtdCiAgICBlbGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgc3RyKToKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW2xhYmVsX2NvbHVtbnNdCgogICAgIyBQcmVkaWN0aW5nOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIm1ha2luZyBwcmVkaWN0aW9uIGJ5ICd7bW9kZWxfaGFuZGxlci5tb2RlbF9uYW1lfSciKQogICAgeV9wcmVkID0gbW9kZWxfaGFuZGxlci5tb2RlbC5wcmVkaWN0KGRhdGFzZXQsICoqa3dhcmdzKQoKICAgICMgUHJlcGFyaW5nIGFuZCB2YWxpZGF0aW5nIGxhYmVsIGNvbHVtbnMgZm9yIHRoZSBkYXRhZnJhbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0OgogICAgbnVtX3ByZWRpY3RlZCA9IDEgaWYgbGVuKHlfcHJlZC5zaGFwZSkgPT0gMSBlbHNlIHlfcHJlZC5zaGFwZVsxXQoKICAgIGlmIG51bV9wcmVkaWN0ZWQgPiBsZW4obGFiZWxfY29sdW1ucyk6CiAgICAgICAgaWYgbnVtX3ByZWRpY3RlZCA9PSAxOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWyJwcmVkaWN0ZWQgbGFiZWxzIl0KICAgICAgICBlbHNlOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zLmV4dGVuZCgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICBmInByZWRpY3RlZF9sYWJlbF97aSArIDEgKyBsZW4obGFiZWxfY29sdW1ucyl9IgogICAgICAgICAgICAgICAgICAgIGZvciBpIGluIHJhbmdlKG51bV9wcmVkaWN0ZWQgLSBsZW4obGFiZWxfY29sdW1ucykpCiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgICkKICAgIGVsaWYgbnVtX3ByZWRpY3RlZCA8IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJudW1iZXIgb2YgcHJlZGljdGVkIGxhYmVsczoge251bV9wcmVkaWN0ZWR9IGlzIHNtYWxsZXIgdGhhbiBudW1iZXIgb2YgbGFiZWwgY29sdW1uczoge2xlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBhcnRpZmFjdF9uYW1lID0gcmVzdWx0X3NldCBvciAicHJlZGljdGlvbiIKICAgIGxhYmVsc19pbnNpZGVfZGYgPSBzZXQobGFiZWxfY29sdW1ucykgJiBzZXQoZGF0YXNldC5jb2x1bW5zLnRvbGlzdCgpKQogICAgaWYgbGFiZWxzX2luc2lkZV9kZjoKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJUaGUgbGFiZWxzOiB7bGFiZWxzX2luc2lkZV9kZn0gYXJlIGFscmVhZHkgZXhpc3RlZCBpbiB0aGUgZGF0YWZyYW1lIgogICAgICAgICkKICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICBwcmVkX2RmID0gcGQuY29uY2F0KFtkYXRhc2V0LCBwZC5EYXRhRnJhbWUoeV9wcmVkLCBjb2x1bW5zPWxhYmVsX2NvbHVtbnMpXSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldChhcnRpZmFjdF9uYW1lLCBwcmVkX2RmLCBkYl9rZXk9cmVzdWx0X3NldCkK + origin_filename: '' + code_origin: '' entry_points: train: - name: train - lineno: 126 doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ \ project = mlrun.get_or_create_project(\"my-project\")\n project.set_function(\"\ hub://auto_trainer\", \"train\")\n trainer_run = project.run(\n \ @@ -26,7 +18,6 @@ spec: : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" - has_varargs: false parameters: - name: context type: MLClientCtx @@ -80,12 +71,12 @@ spec: type: dict doc: Labels to log with the model default: null + name: train + lineno: 126 + has_varargs: false has_kwargs: true evaluate: - name: evaluate - lineno: 278 doc: Evaluating a model. Artifacts generated by the MLHandler. - has_varargs: false parameters: - name: context type: MLClientCtx @@ -105,12 +96,12 @@ spec: doc: The target label(s) of the column(s) in the dataset. for Regression or Classification tasks. Mandatory when dataset is not a FeatureVector. default: null + name: evaluate + lineno: 278 + has_varargs: false has_kwargs: true predict: - name: predict - lineno: 332 doc: Predicting dataset by a model. - has_varargs: false parameters: - name: context type: MLClientCtx @@ -139,10 +130,18 @@ spec: doc: The db key to set name of the prediction result and the filename. Default to 'prediction'. default: null + name: predict + lineno: 332 + has_varargs: false has_kwargs: true - command: '' + description: Automatic train, evaluate and predict functions for the ML frameworks + - Scikit-Learn, XGBoost and LightGBM. default_handler: train - build: - origin_filename: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICAjIENoZWNrIGlmIGRhdGFzZXQgaXMgbGlzdC9kaWN0IGZpcnN0IChiZWZvcmUgdHJ5aW5nIHRvIGFjY2VzcyBhcnRpZmFjdF91cmwpCiAgICBpZiBpc2luc3RhbmNlKGRhdGFzZXQsIChsaXN0LCBkaWN0KSk6CiAgICAgICAgIyBsaXN0L2RpY3QgY2FzZToKICAgICAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICJsYWJlbF9jb2x1bW5zIG5vdCBwcm92aWRlZCwgbWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yIgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICBkYXRhc2V0ID0gcGQuRGF0YUZyYW1lKGRhdGFzZXQpCiAgICAgICAgIyBDaGVja2luZyBpZiBkcm9wX2NvbHVtbnMgcHJvdmlkZWQgYnkgaW50ZWdlciB0eXBlOgogICAgICAgIGlmIGRyb3BfY29sdW1uczoKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIHN0cikgb3IgKAogICAgICAgICAgICAgICAgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIGxpc3QpCiAgICAgICAgICAgICAgICBhbmQgYW55KGlzaW5zdGFuY2UoY29sLCBzdHIpIGZvciBjb2wgaW4gZHJvcF9jb2x1bW5zKQogICAgICAgICAgICApOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyBtdXN0IGJlIGFuIGludGVnZXIvbGlzdCBvZiBpbnRlZ2VycyBpZiBub3QgcHJvdmlkZWQgd2l0aCBhIFVSSS9GZWF0dXJlVmVjdG9yIGRhdGFzZXQiCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICAgICAgICAgIGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSwgaW5wbGFjZT1UcnVlKQogICAgZWxzZToKICAgICAgICAjIERhdGFzZXQgaXMgYSBEYXRhSXRlbSB3aXRoIGFydGlmYWN0X3VybCAoVVJJIG9yIEZlYXR1cmVWZWN0b3IpCiAgICAgICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGF0YXNldC5hcnRpZmFjdF91cmwpCgogICAgICAgICMgR2V0dGluZyB0aGUgZGF0YXNldDoKICAgICAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBsYWJlbF9jb2x1bW5zIG9yIGRhdGFzZXQubWV0YS5zdGF0dXMubGFiZWxfY29sdW1uCiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJsYWJlbCBjb2x1bW5zOiB7bGFiZWxfY29sdW1uc30iKQogICAgICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnYgPSBtbHJ1bi5kYXRhc3RvcmUuZ2V0X3N0b3JlX3Jlc291cmNlKGRhdGFzZXQuYXJ0aWZhY3RfdXJsKQogICAgICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgICAgIGV4Y2VwdCBBdHRyaWJ1dGVFcnJvcjoKICAgICAgICAgICAgICAgICMgTGVhdmUgaGVyZSBmb3IgYmFja3dhcmRzIGNvbXBhdGliaWxpdHkKICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgICAgICBkYXRhc2V0Lm1ldGEudXJpLCBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zCiAgICAgICAgICAgICAgICApLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgIyBzaW1wbGUgVVJMIGNhc2U6CiAgICAgICAgICAgIGlmIG5vdCBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyBub3QgcHJvdmlkZWQsIG1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3RvciIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuYXNfZGYoKQogICAgICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBkYXRhc2V0LmRyb3AoZHJvcF9jb2x1bW5zLCBheGlzPTEpCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgICAgICJub3QgYWxsIG9mIHRoZSBjb2x1bW5zIHRvIGRyb3AgaW4gdGhlIGRhdGFzZXQsIGRyb3AgY29sdW1ucyBwcm9jZXNzIHNraXBwZWQiCiAgICAgICAgICAgICAgICAgICAgKQoKICAgIHJldHVybiBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zCgoKZGVmIHRyYWluKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIG1vZGVsX2NsYXNzOiBzdHIsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIG1vZGVsX25hbWU6IHN0ciA9ICJtb2RlbCIsCiAgICB0YWc6IHN0ciA9ICIiLAogICAgc2FtcGxlX3NldDogRGF0YUl0ZW0gPSBOb25lLAogICAgdGVzdF9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZTogZmxvYXQgPSBOb25lLAogICAgcmFuZG9tX3N0YXRlOiBpbnQgPSBOb25lLAogICAgbGFiZWxzOiBkaWN0ID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBUcmFpbmluZyBhIG1vZGVsIHdpdGggdGhlIGdpdmVuIGRhdGFzZXQuCgogICAgZXhhbXBsZTo6CgogICAgICAgIGltcG9ydCBtbHJ1bgogICAgICAgIHByb2plY3QgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX3Byb2plY3QoIm15LXByb2plY3QiKQogICAgICAgIHByb2plY3Quc2V0X2Z1bmN0aW9uKCJodWI6Ly9hdXRvX3RyYWluZXIiLCAidHJhaW4iKQogICAgICAgIHRyYWluZXJfcnVuID0gcHJvamVjdC5ydW4oCiAgICAgICAgICAgIG5hbWU9InRyYWluIiwKICAgICAgICAgICAgaGFuZGxlcj0idHJhaW4iLAogICAgICAgICAgICBpbnB1dHM9eyJkYXRhc2V0IjogIi4vcGF0aC90by9kYXRhc2V0LmNzdiJ9LAogICAgICAgICAgICBwYXJhbXM9ewogICAgICAgICAgICAgICAgIm1vZGVsX2NsYXNzIjogInNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyI6ICJsYWJlbCIsCiAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIjogImlkIiwKICAgICAgICAgICAgICAgICJtb2RlbF9uYW1lIjogIm15LW1vZGVsIiwKICAgICAgICAgICAgICAgICJ0YWciOiAidjEuMC4wIiwKICAgICAgICAgICAgICAgICJzYW1wbGVfc2V0IjogIi4vcGF0aC90by9zYW1wbGVfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAidGVzdF9zZXQiOiAiLi9wYXRoL3RvL3Rlc3Rfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAiQ0xBU1Nfc29sdmVyIjogImxpYmxpbmVhciIsCiAgICAgICAgICAgIH0sCiAgICAgICAgKQoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dAogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICAgICAgICAgIFRoZSBjbGFzcyBvZiB0aGUgbW9kZWwsIGUuZy4gYHNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbmAKICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcAogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICAgICBUaGUgbW9kZWwncyBuYW1lIHRvIHVzZSBmb3Igc3RvcmluZyB0aGUgbW9kZWwgYXJ0aWZhY3QsIGRlZmF1bHQgdG8gJ21vZGVsJwogICAgOnBhcmFtIHRhZzogICAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwncyB0YWcgdG8gbG9nIHdpdGgKICAgIDpwYXJhbSBzYW1wbGVfc2V0OiAgICAgICAgICAgICAgQSBzYW1wbGUgc2V0IG9mIGlucHV0cyBmb3IgdGhlIG1vZGVsIGZvciBsb2dnaW5nIGl0cyBzdGF0cyBhbG9uZyB0aGUgbW9kZWwgaW4gZmF2b3VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9mIG1vZGVsIG1vbml0b3JpbmcuIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gdGVzdF9zZXQ6ICAgICAgICAgICAgICAgIFRoZSB0ZXN0IHNldCB0byB0cmFpbiB0aGUgbW9kZWwgd2l0aC4KICAgIDpwYXJhbSB0cmFpbl90ZXN0X3NwbGl0X3NpemU6ICAgaWYgdGVzdF9zZXQgd2FzIHByb3ZpZGVkIHRoZW4gdGhpcyBhcmd1bWVudCBpcyBpZ25vcmVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTaG91bGQgYmUgYmV0d2VlbiAwLjAgYW5kIDEuMCBhbmQgcmVwcmVzZW50IHRoZSBwcm9wb3J0aW9uIG9mIHRoZSBkYXRhc2V0IHRvIGluY2x1ZGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaW4gdGhlIHRlc3Qgc3BsaXQuIFRoZSBzaXplIG9mIHRoZSBUcmFpbmluZyBzZXQgaXMgc2V0IHRvIHRoZSBjb21wbGVtZW50IG9mIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWUuIERlZmF1bHQgPSAwLjIKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgUmVsZXZhbnQgb25seSB3aGVuIHVzaW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQSByYW5kb20gc3RhdGUgc2VlZCB0byBzaHVmZmxlIHRoZSBkYXRhLiBGb3IgbW9yZSBpbmZvcm1hdGlvbiwgc2VlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL3NjaWtpdC1sZWFybi5vcmcvc3RhYmxlL2dsb3NzYXJ5Lmh0bWwjdGVybS1yYW5kb21fc3RhdGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTm90aWNlIHRoYXQgaGVyZSB3ZSBvbmx5IHBhc3MgaW50ZWdlciB2YWx1ZXMuCiAgICA6cGFyYW0gbGFiZWxzOiAgICAgICAgICAgICAgICAgIExhYmVscyB0byBsb2cgd2l0aCB0aGUgbW9kZWwKICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgd2l0aCBwcmVmaXhlcywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCB3aWxsIGJlIHBhcnNlZCBhbmQgcGFzc2VkIHRvIHRoZSByZWxldmFudCBmdW5jdGlvbiwgYnkgdGhlIGZvbGxvd2luZyBwcmVmaXhlczoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgQ0xBU1NfYCAtIGZvciB0aGUgbW9kZWwgY2xhc3MgYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYEZJVF9gIC0gZm9yIHRoZSBgZml0YCBmdW5jdGlvbiBhcmd1bWVudHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgVFJBSU5fYCAtIGZvciB0aGUgYHRyYWluYCBmdW5jdGlvbiAoaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24gLSBmdXR1cmUpCgogICAgIiIiCiAgICAjIFZhbGlkYXRlIGlucHV0czoKICAgICMgQ2hlY2sgaWYgZXhhY3RseSBvbmUgb2YgdGhlbSBpcyBzdXBwbGllZDoKICAgIGlmIHRlc3Rfc2V0IGlzIE5vbmU6CiAgICAgICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGlzIE5vbmU6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAidGVzdF9zZXQgb3IgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGFyZSBub3QgcHJvdmlkZWQsIHNldHRpbmcgdHJhaW5fdGVzdF9zcGxpdF9zaXplIHRvIDAuMiIKICAgICAgICAgICAgKQogICAgICAgICAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemUgPSAwLjIKCiAgICBlbGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAidGVzdF9zZXQgcHJvdmlkZWQsIGlnbm9yaW5nIGdpdmVuIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB2YWx1ZSIKICAgICAgICApCiAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gTm9uZQoKICAgICMgR2V0IERhdGFGcmFtZSBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgR2V0dGluZyB0aGUgc2FtcGxlIHNldDoKICAgIGlmIHNhbXBsZV9zZXQgaXMgTm9uZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIlNhbXBsZSBzZXQgbm90IGdpdmVuLCB1c2luZyB0aGUgd2hvbGUgdHJhaW5pbmcgc2V0IGFzIHRoZSBzYW1wbGUgc2V0IgogICAgICAgICkKICAgICAgICBzYW1wbGVfc2V0ID0gZGF0YXNldAogICAgZWxzZToKICAgICAgICBzYW1wbGVfc2V0LCBfID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICAgICAgZGF0YXNldD1zYW1wbGVfc2V0LAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICAgICAgKQoKICAgICMgUGFyc2luZyBrd2FyZ3M6CiAgICAjIFRPRE86IFVzZSBpbiB4Z2Igb3IgbGdibSB0cmFpbiBmdW5jdGlvbi4KICAgIHRyYWluX2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuVFJBSU4pCiAgICBmaXRfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5GSVQpCiAgICBtb2RlbF9jbGFzc19rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeCgKICAgICAgICBzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLk1PREVMX0NMQVNTCiAgICApCgogICAgIyBDaGVjayBpZiBtb2RlbCBvciBmdW5jdGlvbjoKICAgIGlmIGhhc2F0dHIobW9kZWxfY2xhc3MsICJ0cmFpbiIpOgogICAgICAgICMgVE9ETzogTmVlZCB0byBjYWxsOiBtb2RlbCgpLCBhZnRlcndhcmRzIHRvIHN0YXJ0IHRoZSB0cmFpbiBmdW5jdGlvbi4KICAgICAgICAjIG1vZGVsID0gY3JlYXRlX2Z1bmN0aW9uKGYie21vZGVsX2NsYXNzfS50cmFpbiIpCiAgICAgICAgcmFpc2UgTm90SW1wbGVtZW50ZWRFcnJvcgogICAgZWxzZToKICAgICAgICAjIENyZWF0aW5nIG1vZGVsIGluc3RhbmNlOgogICAgICAgIG1vZGVsID0gY3JlYXRlX2NsYXNzKG1vZGVsX2NsYXNzKSgqKm1vZGVsX2NsYXNzX2t3YXJncykKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCiAgICBpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgeF90cmFpbiwgeF90ZXN0LCB5X3RyYWluLCB5X3Rlc3QgPSB0cmFpbl90ZXN0X3NwbGl0KAogICAgICAgICAgICB4LCB5LCB0ZXN0X3NpemU9dHJhaW5fdGVzdF9zcGxpdF9zaXplLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICB4X3RyYWluLCB5X3RyYWluID0geCwgeQoKICAgICAgICB0ZXN0X3NldCA9IHRlc3Rfc2V0LmFzX2RmKCkKICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgIHRlc3Rfc2V0ID0gZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xKQoKICAgICAgICB4X3Rlc3QsIHlfdGVzdCA9IHRlc3Rfc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKSwgdGVzdF9zZXRbbGFiZWxfY29sdW1uc10KCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4oCiAgICAgICAgbW9kZWw9bW9kZWwsCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICB0YWc9dGFnLAogICAgICAgIHNhbXBsZV9zZXQ9c2FtcGxlX3NldCwKICAgICAgICB5X2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICB0ZXN0X3NldD10ZXN0X3NldCwKICAgICAgICB4X3Rlc3Q9eF90ZXN0LAogICAgICAgIHlfdGVzdD15X3Rlc3QsCiAgICAgICAgYXJ0aWZhY3RzPWNvbnRleHQuYXJ0aWZhY3RzLAogICAgICAgIGxhYmVscz1sYWJlbHMsCiAgICApCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidHJhaW5pbmcgJ3ttb2RlbF9uYW1lfSciKQogICAgbW9kZWwuZml0KHhfdHJhaW4sIHlfdHJhaW4sICoqZml0X2t3YXJncykKCgpkZWYgZXZhbHVhdGUoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIEV2YWx1YXRpbmcgYSBtb2RlbC4gQXJ0aWZhY3RzIGdlbmVyYXRlZCBieSB0aGUgTUxIYW5kbGVyLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIGV2YWx1YXRlIHRoZSBtb2RlbCBvbi4gQ2FuIGJlIGVpdGhlciBhIFVSSSBvciBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICAgICAgICAgIHN0ciBvciBhIGxpc3Qgb2Ygc3RyaW5ncyB0aGF0IHJlcHJlc2VudCB0aGUgY29sdW1ucyB0byBkcm9wLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHRvIHRoZSBwcmVkaWN0IGZ1bmN0aW9uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChQUkVESUNUXyBwcmVmaXggaXMgbm90IHJlcXVpcmVkKS4KICAgICIiIgogICAgIyBHZXQgZGF0YXNldCBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgUGFyc2luZyBsYWJlbF9jb2x1bW5zOgogICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMgPSBbXQogICAgaWYgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gKAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zIGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgbGlzdCkgZWxzZSBbbGFiZWxfY29sdW1uc10KICAgICAgICApCiAgICAgICAgZm9yIGxjIGluIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGlmIGZzLmNvbW1vbi5mZWF0dXJlX3NlcGFyYXRvciBpbiBsYzoKICAgICAgICAgICAgICAgIGZlYXR1cmVfc2V0X25hbWUsIGxhYmVsX25hbWUsIGFsaWFzID0gZnMuY29tbW9uLnBhcnNlX2ZlYXR1cmVfc3RyaW5nKGxjKQogICAgICAgICAgICAgICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMuYXBwZW5kKGFsaWFzIG9yIGxhYmVsX25hbWUpCiAgICAgICAgaWYgcGFyc2VkX2xhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBwYXJzZWRfbGFiZWxfY29sdW1ucwoKICAgIHggPSBkYXRhc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKQogICAgeSA9IGRhdGFzZXRbbGFiZWxfY29sdW1uc10KCiAgICAjIExvYWRpbmcgdGhlIG1vZGVsIGFuZCBwcmVkaWN0aW5nOgogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWwsIGNvbnRleHQ9Y29udGV4dCwgbW9kZWxfbmFtZT0ibW9kZWxfTGluZWFyUmVncmVzc2lvbiIKICAgICkKICAgIEF1dG9NTFJ1bi5hcHBseV9tbHJ1bihtb2RlbF9oYW5kbGVyLm1vZGVsLCB5X3Rlc3Q9eSwgbW9kZWxfcGF0aD1tb2RlbCkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiZXZhbHVhdGluZyAne21vZGVsX2hhbmRsZXIubW9kZWxfbmFtZX0nIikKICAgIG1vZGVsX2hhbmRsZXIubW9kZWwucHJlZGljdCh4LCAqKmt3YXJncykKCgpkZWYgcHJlZGljdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWw6IHN0ciwKICAgIGRhdGFzZXQ6IG1scnVuLkRhdGFJdGVtLAogICAgZHJvcF9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXSwgaW50LCBMaXN0W2ludF1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgcmVzdWx0X3NldDogT3B0aW9uYWxbc3RyXSA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgUHJlZGljdGluZyBkYXRhc2V0IGJ5IGEgbW9kZWwuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwgU3RvcmUgcGF0aC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gcHJlZGljdCB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkksIGEgRmVhdHVyZVZlY3RvciBvciBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSBpbiBhIHNoYXBlIG9mIGEgbGlzdC9kaWN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHBhc3NpbmcgYSBzYW1wbGUsIHBhc3MgdGhlIGRhdGFzZXQgYXMgYSBmaWVsZCBpbiBgcGFyYW1zYCBpbnN0ZWFkIG9mIGBpbnB1dHNgLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIvaW50IG9yIGEgbGlzdCBvZiBzdHJpbmdzL2ludHMgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbiBuYW1lcy9pbmRpY2VzIHRvIGRyb3AuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFdoZW4gdGhlIGRhdGFzZXQgaXMgYSBsaXN0L2RpY3QgdGhpcyBwYXJhbWV0ZXIgc2hvdWxkIGJlIHJlcHJlc2VudGVkIGJ5IGludGVnZXJzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gcmVzdWx0X3NldDogICAgICAgICAgICAgIFRoZSBkYiBrZXkgdG8gc2V0IG5hbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0IGFuZCB0aGUgZmlsZW5hbWUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHQgdG8gJ3ByZWRpY3Rpb24nLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIGxvYWRpbmcgdGhlIG1vZGVsLCBhbmQgZ2V0dGluZyB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbChtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQpCgogICAgIyBGaXggZmVhdHVyZSBuYW1lcyBmb3IgbW9kZWxzIHRoYXQgcmVxdWlyZSB0aGVtIChlLmcuLCBYR0Jvb3N0KQogICAgIyBXaGVuIGRhdGFzZXQgY29tZXMgZnJvbSBhIGxpc3QsIHBhbmRhcyBhc3NpZ25zIGRlZmF1bHQgaW50ZWdlciBjb2x1bW4gbmFtZXMKICAgICMgYnV0IHNvbWUgbW9kZWxzIGV4cGVjdCBzcGVjaWZpYyBmZWF0dXJlIG5hbWVzIHRoZXkgd2VyZSB0cmFpbmVkIHdpdGgKICAgIGlmIGhhc2F0dHIobW9kZWxfaGFuZGxlci5tb2RlbCwgJ2ZlYXR1cmVfbmFtZXNfaW5fJyk6CiAgICAgICAgZXhwZWN0ZWRfZmVhdHVyZXMgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLmZlYXR1cmVfbmFtZXNfaW5fCiAgICAgICAgaWYgbGVuKGRhdGFzZXQuY29sdW1ucykgPT0gbGVuKGV4cGVjdGVkX2ZlYXR1cmVzKToKICAgICAgICAgICAgIyBPbmx5IHJlbmFtZSBpZiB0aGUgbnVtYmVyIG9mIGNvbHVtbnMgbWF0Y2hlcwogICAgICAgICAgICAjIFRoaXMgaGFuZGxlcyB0aGUgY2FzZSB3aGVyZSBhIGxpc3Qgd2FzIGNvbnZlcnRlZCB0byBEYXRhRnJhbWUgd2l0aCBkZWZhdWx0IGNvbHVtbiBuYW1lcwogICAgICAgICAgICBpZiBub3QgYWxsKGNvbCA9PSBmZWF0IGZvciBjb2wsIGZlYXQgaW4gemlwKGRhdGFzZXQuY29sdW1ucywgZXhwZWN0ZWRfZmVhdHVyZXMpKToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSZW5hbWluZyBkYXRhc2V0IGNvbHVtbnMgdG8gbWF0Y2ggbW9kZWwncyBleHBlY3RlZCBmZWF0dXJlIG5hbWVzIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZGF0YXNldC5jb2x1bW5zID0gZXhwZWN0ZWRfZmVhdHVyZXMKCiAgICAjIERyb3BwaW5nIGxhYmVsIGNvbHVtbnMgaWYgbmVjZXNzYXJ5OgogICAgaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtdCiAgICBlbGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgc3RyKToKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW2xhYmVsX2NvbHVtbnNdCgogICAgIyBQcmVkaWN0aW5nOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIm1ha2luZyBwcmVkaWN0aW9uIGJ5ICd7bW9kZWxfaGFuZGxlci5tb2RlbF9uYW1lfSciKQogICAgeV9wcmVkID0gbW9kZWxfaGFuZGxlci5tb2RlbC5wcmVkaWN0KGRhdGFzZXQsICoqa3dhcmdzKQoKICAgICMgUHJlcGFyaW5nIGFuZCB2YWxpZGF0aW5nIGxhYmVsIGNvbHVtbnMgZm9yIHRoZSBkYXRhZnJhbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0OgogICAgbnVtX3ByZWRpY3RlZCA9IDEgaWYgbGVuKHlfcHJlZC5zaGFwZSkgPT0gMSBlbHNlIHlfcHJlZC5zaGFwZVsxXQoKICAgIGlmIG51bV9wcmVkaWN0ZWQgPiBsZW4obGFiZWxfY29sdW1ucyk6CiAgICAgICAgaWYgbnVtX3ByZWRpY3RlZCA9PSAxOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWyJwcmVkaWN0ZWQgbGFiZWxzIl0KICAgICAgICBlbHNlOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zLmV4dGVuZCgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICBmInByZWRpY3RlZF9sYWJlbF97aSArIDEgKyBsZW4obGFiZWxfY29sdW1ucyl9IgogICAgICAgICAgICAgICAgICAgIGZvciBpIGluIHJhbmdlKG51bV9wcmVkaWN0ZWQgLSBsZW4obGFiZWxfY29sdW1ucykpCiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgICkKICAgIGVsaWYgbnVtX3ByZWRpY3RlZCA8IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJudW1iZXIgb2YgcHJlZGljdGVkIGxhYmVsczoge251bV9wcmVkaWN0ZWR9IGlzIHNtYWxsZXIgdGhhbiBudW1iZXIgb2YgbGFiZWwgY29sdW1uczoge2xlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBhcnRpZmFjdF9uYW1lID0gcmVzdWx0X3NldCBvciAicHJlZGljdGlvbiIKICAgIGxhYmVsc19pbnNpZGVfZGYgPSBzZXQobGFiZWxfY29sdW1ucykgJiBzZXQoZGF0YXNldC5jb2x1bW5zLnRvbGlzdCgpKQogICAgaWYgbGFiZWxzX2luc2lkZV9kZjoKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJUaGUgbGFiZWxzOiB7bGFiZWxzX2luc2lkZV9kZn0gYXJlIGFscmVhZHkgZXhpc3RlZCBpbiB0aGUgZGF0YWZyYW1lIgogICAgICAgICkKICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICBwcmVkX2RmID0gcGQuY29uY2F0KFtkYXRhc2V0LCBwZC5EYXRhRnJhbWUoeV9wcmVkLCBjb2x1bW5zPWxhYmVsX2NvbHVtbnMpXSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldChhcnRpZmFjdF9uYW1lLCBwcmVkX2RmLCBkYl9rZXk9cmVzdWx0X3NldCkK - code_origin: '' + image: mlrun/mlrun + command: '' +metadata: + tag: '' + name: auto-trainer + categories: + - machine-learning + - model-training diff --git a/functions/src/auto_trainer/item.yaml b/functions/src/auto_trainer/item.yaml index 78de92ca0..d397a79d6 100755 --- a/functions/src/auto_trainer/item.yaml +++ b/functions/src/auto_trainer/item.yaml @@ -13,7 +13,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.0 +mlrunVersion: 1.10.0 name: auto_trainer platformVersion: 3.5.0 spec: diff --git a/functions/src/auto_trainer/requirements.txt b/functions/src/auto_trainer/requirements.txt index 4854d84fd..b14a0293c 100644 --- a/functions/src/auto_trainer/requirements.txt +++ b/functions/src/auto_trainer/requirements.txt @@ -1,4 +1,4 @@ pandas -scikit-learn~=1.5 +scikit-learn<1.4.0 xgboost<2.0.0 plotly From ace141e84f2df82048a22d63ab993bdbc381ad11 Mon Sep 17 00:00:00 2001 From: tomerbv Date: Wed, 21 Jan 2026 15:50:10 +0200 Subject: [PATCH 13/15] scikit-learn strict v~=1.5.2 added skip for test_train in test_auto_trainer.py --- functions/src/auto_trainer/function.yaml | 34 +++++++++---------- functions/src/auto_trainer/requirements.txt | 2 +- .../src/auto_trainer/test_auto_trainer.py | 4 +++ functions/src/describe/requirements.txt | 2 +- functions/src/gen_class_data/requirements.txt | 2 +- .../src/sklearn_classifier/requirements.txt | 2 +- 6 files changed, 25 insertions(+), 21 deletions(-) diff --git a/functions/src/auto_trainer/function.yaml b/functions/src/auto_trainer/function.yaml index 155e5c58e..50a36e750 100644 --- a/functions/src/auto_trainer/function.yaml +++ b/functions/src/auto_trainer/function.yaml @@ -1,11 +1,6 @@ -kind: job verbose: false +kind: job spec: - disable_auto_mount: false - build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICAjIENoZWNrIGlmIGRhdGFzZXQgaXMgbGlzdC9kaWN0IGZpcnN0IChiZWZvcmUgdHJ5aW5nIHRvIGFjY2VzcyBhcnRpZmFjdF91cmwpCiAgICBpZiBpc2luc3RhbmNlKGRhdGFzZXQsIChsaXN0LCBkaWN0KSk6CiAgICAgICAgIyBsaXN0L2RpY3QgY2FzZToKICAgICAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICJsYWJlbF9jb2x1bW5zIG5vdCBwcm92aWRlZCwgbWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yIgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICBkYXRhc2V0ID0gcGQuRGF0YUZyYW1lKGRhdGFzZXQpCiAgICAgICAgIyBDaGVja2luZyBpZiBkcm9wX2NvbHVtbnMgcHJvdmlkZWQgYnkgaW50ZWdlciB0eXBlOgogICAgICAgIGlmIGRyb3BfY29sdW1uczoKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIHN0cikgb3IgKAogICAgICAgICAgICAgICAgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIGxpc3QpCiAgICAgICAgICAgICAgICBhbmQgYW55KGlzaW5zdGFuY2UoY29sLCBzdHIpIGZvciBjb2wgaW4gZHJvcF9jb2x1bW5zKQogICAgICAgICAgICApOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyBtdXN0IGJlIGFuIGludGVnZXIvbGlzdCBvZiBpbnRlZ2VycyBpZiBub3QgcHJvdmlkZWQgd2l0aCBhIFVSSS9GZWF0dXJlVmVjdG9yIGRhdGFzZXQiCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICAgICAgICAgIGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSwgaW5wbGFjZT1UcnVlKQogICAgZWxzZToKICAgICAgICAjIERhdGFzZXQgaXMgYSBEYXRhSXRlbSB3aXRoIGFydGlmYWN0X3VybCAoVVJJIG9yIEZlYXR1cmVWZWN0b3IpCiAgICAgICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGF0YXNldC5hcnRpZmFjdF91cmwpCgogICAgICAgICMgR2V0dGluZyB0aGUgZGF0YXNldDoKICAgICAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBsYWJlbF9jb2x1bW5zIG9yIGRhdGFzZXQubWV0YS5zdGF0dXMubGFiZWxfY29sdW1uCiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJsYWJlbCBjb2x1bW5zOiB7bGFiZWxfY29sdW1uc30iKQogICAgICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnYgPSBtbHJ1bi5kYXRhc3RvcmUuZ2V0X3N0b3JlX3Jlc291cmNlKGRhdGFzZXQuYXJ0aWZhY3RfdXJsKQogICAgICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgICAgIGV4Y2VwdCBBdHRyaWJ1dGVFcnJvcjoKICAgICAgICAgICAgICAgICMgTGVhdmUgaGVyZSBmb3IgYmFja3dhcmRzIGNvbXBhdGliaWxpdHkKICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgICAgICBkYXRhc2V0Lm1ldGEudXJpLCBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zCiAgICAgICAgICAgICAgICApLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgIyBzaW1wbGUgVVJMIGNhc2U6CiAgICAgICAgICAgIGlmIG5vdCBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyBub3QgcHJvdmlkZWQsIG1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3RvciIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuYXNfZGYoKQogICAgICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBkYXRhc2V0LmRyb3AoZHJvcF9jb2x1bW5zLCBheGlzPTEpCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgICAgICJub3QgYWxsIG9mIHRoZSBjb2x1bW5zIHRvIGRyb3AgaW4gdGhlIGRhdGFzZXQsIGRyb3AgY29sdW1ucyBwcm9jZXNzIHNraXBwZWQiCiAgICAgICAgICAgICAgICAgICAgKQoKICAgIHJldHVybiBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zCgoKZGVmIHRyYWluKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIG1vZGVsX2NsYXNzOiBzdHIsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIG1vZGVsX25hbWU6IHN0ciA9ICJtb2RlbCIsCiAgICB0YWc6IHN0ciA9ICIiLAogICAgc2FtcGxlX3NldDogRGF0YUl0ZW0gPSBOb25lLAogICAgdGVzdF9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZTogZmxvYXQgPSBOb25lLAogICAgcmFuZG9tX3N0YXRlOiBpbnQgPSBOb25lLAogICAgbGFiZWxzOiBkaWN0ID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBUcmFpbmluZyBhIG1vZGVsIHdpdGggdGhlIGdpdmVuIGRhdGFzZXQuCgogICAgZXhhbXBsZTo6CgogICAgICAgIGltcG9ydCBtbHJ1bgogICAgICAgIHByb2plY3QgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX3Byb2plY3QoIm15LXByb2plY3QiKQogICAgICAgIHByb2plY3Quc2V0X2Z1bmN0aW9uKCJodWI6Ly9hdXRvX3RyYWluZXIiLCAidHJhaW4iKQogICAgICAgIHRyYWluZXJfcnVuID0gcHJvamVjdC5ydW4oCiAgICAgICAgICAgIG5hbWU9InRyYWluIiwKICAgICAgICAgICAgaGFuZGxlcj0idHJhaW4iLAogICAgICAgICAgICBpbnB1dHM9eyJkYXRhc2V0IjogIi4vcGF0aC90by9kYXRhc2V0LmNzdiJ9LAogICAgICAgICAgICBwYXJhbXM9ewogICAgICAgICAgICAgICAgIm1vZGVsX2NsYXNzIjogInNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyI6ICJsYWJlbCIsCiAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIjogImlkIiwKICAgICAgICAgICAgICAgICJtb2RlbF9uYW1lIjogIm15LW1vZGVsIiwKICAgICAgICAgICAgICAgICJ0YWciOiAidjEuMC4wIiwKICAgICAgICAgICAgICAgICJzYW1wbGVfc2V0IjogIi4vcGF0aC90by9zYW1wbGVfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAidGVzdF9zZXQiOiAiLi9wYXRoL3RvL3Rlc3Rfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAiQ0xBU1Nfc29sdmVyIjogImxpYmxpbmVhciIsCiAgICAgICAgICAgIH0sCiAgICAgICAgKQoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dAogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICAgICAgICAgIFRoZSBjbGFzcyBvZiB0aGUgbW9kZWwsIGUuZy4gYHNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbmAKICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcAogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICAgICBUaGUgbW9kZWwncyBuYW1lIHRvIHVzZSBmb3Igc3RvcmluZyB0aGUgbW9kZWwgYXJ0aWZhY3QsIGRlZmF1bHQgdG8gJ21vZGVsJwogICAgOnBhcmFtIHRhZzogICAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwncyB0YWcgdG8gbG9nIHdpdGgKICAgIDpwYXJhbSBzYW1wbGVfc2V0OiAgICAgICAgICAgICAgQSBzYW1wbGUgc2V0IG9mIGlucHV0cyBmb3IgdGhlIG1vZGVsIGZvciBsb2dnaW5nIGl0cyBzdGF0cyBhbG9uZyB0aGUgbW9kZWwgaW4gZmF2b3VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9mIG1vZGVsIG1vbml0b3JpbmcuIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gdGVzdF9zZXQ6ICAgICAgICAgICAgICAgIFRoZSB0ZXN0IHNldCB0byB0cmFpbiB0aGUgbW9kZWwgd2l0aC4KICAgIDpwYXJhbSB0cmFpbl90ZXN0X3NwbGl0X3NpemU6ICAgaWYgdGVzdF9zZXQgd2FzIHByb3ZpZGVkIHRoZW4gdGhpcyBhcmd1bWVudCBpcyBpZ25vcmVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTaG91bGQgYmUgYmV0d2VlbiAwLjAgYW5kIDEuMCBhbmQgcmVwcmVzZW50IHRoZSBwcm9wb3J0aW9uIG9mIHRoZSBkYXRhc2V0IHRvIGluY2x1ZGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaW4gdGhlIHRlc3Qgc3BsaXQuIFRoZSBzaXplIG9mIHRoZSBUcmFpbmluZyBzZXQgaXMgc2V0IHRvIHRoZSBjb21wbGVtZW50IG9mIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWUuIERlZmF1bHQgPSAwLjIKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgUmVsZXZhbnQgb25seSB3aGVuIHVzaW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQSByYW5kb20gc3RhdGUgc2VlZCB0byBzaHVmZmxlIHRoZSBkYXRhLiBGb3IgbW9yZSBpbmZvcm1hdGlvbiwgc2VlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL3NjaWtpdC1sZWFybi5vcmcvc3RhYmxlL2dsb3NzYXJ5Lmh0bWwjdGVybS1yYW5kb21fc3RhdGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTm90aWNlIHRoYXQgaGVyZSB3ZSBvbmx5IHBhc3MgaW50ZWdlciB2YWx1ZXMuCiAgICA6cGFyYW0gbGFiZWxzOiAgICAgICAgICAgICAgICAgIExhYmVscyB0byBsb2cgd2l0aCB0aGUgbW9kZWwKICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgd2l0aCBwcmVmaXhlcywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCB3aWxsIGJlIHBhcnNlZCBhbmQgcGFzc2VkIHRvIHRoZSByZWxldmFudCBmdW5jdGlvbiwgYnkgdGhlIGZvbGxvd2luZyBwcmVmaXhlczoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgQ0xBU1NfYCAtIGZvciB0aGUgbW9kZWwgY2xhc3MgYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYEZJVF9gIC0gZm9yIHRoZSBgZml0YCBmdW5jdGlvbiBhcmd1bWVudHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgVFJBSU5fYCAtIGZvciB0aGUgYHRyYWluYCBmdW5jdGlvbiAoaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24gLSBmdXR1cmUpCgogICAgIiIiCiAgICAjIFZhbGlkYXRlIGlucHV0czoKICAgICMgQ2hlY2sgaWYgZXhhY3RseSBvbmUgb2YgdGhlbSBpcyBzdXBwbGllZDoKICAgIGlmIHRlc3Rfc2V0IGlzIE5vbmU6CiAgICAgICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGlzIE5vbmU6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAidGVzdF9zZXQgb3IgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGFyZSBub3QgcHJvdmlkZWQsIHNldHRpbmcgdHJhaW5fdGVzdF9zcGxpdF9zaXplIHRvIDAuMiIKICAgICAgICAgICAgKQogICAgICAgICAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemUgPSAwLjIKCiAgICBlbGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAidGVzdF9zZXQgcHJvdmlkZWQsIGlnbm9yaW5nIGdpdmVuIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB2YWx1ZSIKICAgICAgICApCiAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gTm9uZQoKICAgICMgR2V0IERhdGFGcmFtZSBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgR2V0dGluZyB0aGUgc2FtcGxlIHNldDoKICAgIGlmIHNhbXBsZV9zZXQgaXMgTm9uZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIlNhbXBsZSBzZXQgbm90IGdpdmVuLCB1c2luZyB0aGUgd2hvbGUgdHJhaW5pbmcgc2V0IGFzIHRoZSBzYW1wbGUgc2V0IgogICAgICAgICkKICAgICAgICBzYW1wbGVfc2V0ID0gZGF0YXNldAogICAgZWxzZToKICAgICAgICBzYW1wbGVfc2V0LCBfID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICAgICAgZGF0YXNldD1zYW1wbGVfc2V0LAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICAgICAgKQoKICAgICMgUGFyc2luZyBrd2FyZ3M6CiAgICAjIFRPRE86IFVzZSBpbiB4Z2Igb3IgbGdibSB0cmFpbiBmdW5jdGlvbi4KICAgIHRyYWluX2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuVFJBSU4pCiAgICBmaXRfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5GSVQpCiAgICBtb2RlbF9jbGFzc19rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeCgKICAgICAgICBzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLk1PREVMX0NMQVNTCiAgICApCgogICAgIyBDaGVjayBpZiBtb2RlbCBvciBmdW5jdGlvbjoKICAgIGlmIGhhc2F0dHIobW9kZWxfY2xhc3MsICJ0cmFpbiIpOgogICAgICAgICMgVE9ETzogTmVlZCB0byBjYWxsOiBtb2RlbCgpLCBhZnRlcndhcmRzIHRvIHN0YXJ0IHRoZSB0cmFpbiBmdW5jdGlvbi4KICAgICAgICAjIG1vZGVsID0gY3JlYXRlX2Z1bmN0aW9uKGYie21vZGVsX2NsYXNzfS50cmFpbiIpCiAgICAgICAgcmFpc2UgTm90SW1wbGVtZW50ZWRFcnJvcgogICAgZWxzZToKICAgICAgICAjIENyZWF0aW5nIG1vZGVsIGluc3RhbmNlOgogICAgICAgIG1vZGVsID0gY3JlYXRlX2NsYXNzKG1vZGVsX2NsYXNzKSgqKm1vZGVsX2NsYXNzX2t3YXJncykKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCiAgICBpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgeF90cmFpbiwgeF90ZXN0LCB5X3RyYWluLCB5X3Rlc3QgPSB0cmFpbl90ZXN0X3NwbGl0KAogICAgICAgICAgICB4LCB5LCB0ZXN0X3NpemU9dHJhaW5fdGVzdF9zcGxpdF9zaXplLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICB4X3RyYWluLCB5X3RyYWluID0geCwgeQoKICAgICAgICB0ZXN0X3NldCA9IHRlc3Rfc2V0LmFzX2RmKCkKICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgIHRlc3Rfc2V0ID0gZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xKQoKICAgICAgICB4X3Rlc3QsIHlfdGVzdCA9IHRlc3Rfc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKSwgdGVzdF9zZXRbbGFiZWxfY29sdW1uc10KCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4oCiAgICAgICAgbW9kZWw9bW9kZWwsCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICB0YWc9dGFnLAogICAgICAgIHNhbXBsZV9zZXQ9c2FtcGxlX3NldCwKICAgICAgICB5X2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICB0ZXN0X3NldD10ZXN0X3NldCwKICAgICAgICB4X3Rlc3Q9eF90ZXN0LAogICAgICAgIHlfdGVzdD15X3Rlc3QsCiAgICAgICAgYXJ0aWZhY3RzPWNvbnRleHQuYXJ0aWZhY3RzLAogICAgICAgIGxhYmVscz1sYWJlbHMsCiAgICApCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidHJhaW5pbmcgJ3ttb2RlbF9uYW1lfSciKQogICAgbW9kZWwuZml0KHhfdHJhaW4sIHlfdHJhaW4sICoqZml0X2t3YXJncykKCgpkZWYgZXZhbHVhdGUoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIEV2YWx1YXRpbmcgYSBtb2RlbC4gQXJ0aWZhY3RzIGdlbmVyYXRlZCBieSB0aGUgTUxIYW5kbGVyLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIGV2YWx1YXRlIHRoZSBtb2RlbCBvbi4gQ2FuIGJlIGVpdGhlciBhIFVSSSBvciBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICAgICAgICAgIHN0ciBvciBhIGxpc3Qgb2Ygc3RyaW5ncyB0aGF0IHJlcHJlc2VudCB0aGUgY29sdW1ucyB0byBkcm9wLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHRvIHRoZSBwcmVkaWN0IGZ1bmN0aW9uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChQUkVESUNUXyBwcmVmaXggaXMgbm90IHJlcXVpcmVkKS4KICAgICIiIgogICAgIyBHZXQgZGF0YXNldCBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgUGFyc2luZyBsYWJlbF9jb2x1bW5zOgogICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMgPSBbXQogICAgaWYgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gKAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zIGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgbGlzdCkgZWxzZSBbbGFiZWxfY29sdW1uc10KICAgICAgICApCiAgICAgICAgZm9yIGxjIGluIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGlmIGZzLmNvbW1vbi5mZWF0dXJlX3NlcGFyYXRvciBpbiBsYzoKICAgICAgICAgICAgICAgIGZlYXR1cmVfc2V0X25hbWUsIGxhYmVsX25hbWUsIGFsaWFzID0gZnMuY29tbW9uLnBhcnNlX2ZlYXR1cmVfc3RyaW5nKGxjKQogICAgICAgICAgICAgICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMuYXBwZW5kKGFsaWFzIG9yIGxhYmVsX25hbWUpCiAgICAgICAgaWYgcGFyc2VkX2xhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBwYXJzZWRfbGFiZWxfY29sdW1ucwoKICAgIHggPSBkYXRhc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKQogICAgeSA9IGRhdGFzZXRbbGFiZWxfY29sdW1uc10KCiAgICAjIExvYWRpbmcgdGhlIG1vZGVsIGFuZCBwcmVkaWN0aW5nOgogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWwsIGNvbnRleHQ9Y29udGV4dCwgbW9kZWxfbmFtZT0ibW9kZWxfTGluZWFyUmVncmVzc2lvbiIKICAgICkKICAgIEF1dG9NTFJ1bi5hcHBseV9tbHJ1bihtb2RlbF9oYW5kbGVyLm1vZGVsLCB5X3Rlc3Q9eSwgbW9kZWxfcGF0aD1tb2RlbCkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiZXZhbHVhdGluZyAne21vZGVsX2hhbmRsZXIubW9kZWxfbmFtZX0nIikKICAgIG1vZGVsX2hhbmRsZXIubW9kZWwucHJlZGljdCh4LCAqKmt3YXJncykKCgpkZWYgcHJlZGljdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWw6IHN0ciwKICAgIGRhdGFzZXQ6IG1scnVuLkRhdGFJdGVtLAogICAgZHJvcF9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXSwgaW50LCBMaXN0W2ludF1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgcmVzdWx0X3NldDogT3B0aW9uYWxbc3RyXSA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgUHJlZGljdGluZyBkYXRhc2V0IGJ5IGEgbW9kZWwuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwgU3RvcmUgcGF0aC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gcHJlZGljdCB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkksIGEgRmVhdHVyZVZlY3RvciBvciBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSBpbiBhIHNoYXBlIG9mIGEgbGlzdC9kaWN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHBhc3NpbmcgYSBzYW1wbGUsIHBhc3MgdGhlIGRhdGFzZXQgYXMgYSBmaWVsZCBpbiBgcGFyYW1zYCBpbnN0ZWFkIG9mIGBpbnB1dHNgLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIvaW50IG9yIGEgbGlzdCBvZiBzdHJpbmdzL2ludHMgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbiBuYW1lcy9pbmRpY2VzIHRvIGRyb3AuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFdoZW4gdGhlIGRhdGFzZXQgaXMgYSBsaXN0L2RpY3QgdGhpcyBwYXJhbWV0ZXIgc2hvdWxkIGJlIHJlcHJlc2VudGVkIGJ5IGludGVnZXJzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gcmVzdWx0X3NldDogICAgICAgICAgICAgIFRoZSBkYiBrZXkgdG8gc2V0IG5hbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0IGFuZCB0aGUgZmlsZW5hbWUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHQgdG8gJ3ByZWRpY3Rpb24nLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIGxvYWRpbmcgdGhlIG1vZGVsLCBhbmQgZ2V0dGluZyB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbChtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQpCgogICAgIyBGaXggZmVhdHVyZSBuYW1lcyBmb3IgbW9kZWxzIHRoYXQgcmVxdWlyZSB0aGVtIChlLmcuLCBYR0Jvb3N0KQogICAgIyBXaGVuIGRhdGFzZXQgY29tZXMgZnJvbSBhIGxpc3QsIHBhbmRhcyBhc3NpZ25zIGRlZmF1bHQgaW50ZWdlciBjb2x1bW4gbmFtZXMKICAgICMgYnV0IHNvbWUgbW9kZWxzIGV4cGVjdCBzcGVjaWZpYyBmZWF0dXJlIG5hbWVzIHRoZXkgd2VyZSB0cmFpbmVkIHdpdGgKICAgIGlmIGhhc2F0dHIobW9kZWxfaGFuZGxlci5tb2RlbCwgJ2ZlYXR1cmVfbmFtZXNfaW5fJyk6CiAgICAgICAgZXhwZWN0ZWRfZmVhdHVyZXMgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLmZlYXR1cmVfbmFtZXNfaW5fCiAgICAgICAgaWYgbGVuKGRhdGFzZXQuY29sdW1ucykgPT0gbGVuKGV4cGVjdGVkX2ZlYXR1cmVzKToKICAgICAgICAgICAgIyBPbmx5IHJlbmFtZSBpZiB0aGUgbnVtYmVyIG9mIGNvbHVtbnMgbWF0Y2hlcwogICAgICAgICAgICAjIFRoaXMgaGFuZGxlcyB0aGUgY2FzZSB3aGVyZSBhIGxpc3Qgd2FzIGNvbnZlcnRlZCB0byBEYXRhRnJhbWUgd2l0aCBkZWZhdWx0IGNvbHVtbiBuYW1lcwogICAgICAgICAgICBpZiBub3QgYWxsKGNvbCA9PSBmZWF0IGZvciBjb2wsIGZlYXQgaW4gemlwKGRhdGFzZXQuY29sdW1ucywgZXhwZWN0ZWRfZmVhdHVyZXMpKToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSZW5hbWluZyBkYXRhc2V0IGNvbHVtbnMgdG8gbWF0Y2ggbW9kZWwncyBleHBlY3RlZCBmZWF0dXJlIG5hbWVzIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZGF0YXNldC5jb2x1bW5zID0gZXhwZWN0ZWRfZmVhdHVyZXMKCiAgICAjIERyb3BwaW5nIGxhYmVsIGNvbHVtbnMgaWYgbmVjZXNzYXJ5OgogICAgaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtdCiAgICBlbGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgc3RyKToKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW2xhYmVsX2NvbHVtbnNdCgogICAgIyBQcmVkaWN0aW5nOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIm1ha2luZyBwcmVkaWN0aW9uIGJ5ICd7bW9kZWxfaGFuZGxlci5tb2RlbF9uYW1lfSciKQogICAgeV9wcmVkID0gbW9kZWxfaGFuZGxlci5tb2RlbC5wcmVkaWN0KGRhdGFzZXQsICoqa3dhcmdzKQoKICAgICMgUHJlcGFyaW5nIGFuZCB2YWxpZGF0aW5nIGxhYmVsIGNvbHVtbnMgZm9yIHRoZSBkYXRhZnJhbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0OgogICAgbnVtX3ByZWRpY3RlZCA9IDEgaWYgbGVuKHlfcHJlZC5zaGFwZSkgPT0gMSBlbHNlIHlfcHJlZC5zaGFwZVsxXQoKICAgIGlmIG51bV9wcmVkaWN0ZWQgPiBsZW4obGFiZWxfY29sdW1ucyk6CiAgICAgICAgaWYgbnVtX3ByZWRpY3RlZCA9PSAxOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWyJwcmVkaWN0ZWQgbGFiZWxzIl0KICAgICAgICBlbHNlOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zLmV4dGVuZCgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICBmInByZWRpY3RlZF9sYWJlbF97aSArIDEgKyBsZW4obGFiZWxfY29sdW1ucyl9IgogICAgICAgICAgICAgICAgICAgIGZvciBpIGluIHJhbmdlKG51bV9wcmVkaWN0ZWQgLSBsZW4obGFiZWxfY29sdW1ucykpCiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgICkKICAgIGVsaWYgbnVtX3ByZWRpY3RlZCA8IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJudW1iZXIgb2YgcHJlZGljdGVkIGxhYmVsczoge251bV9wcmVkaWN0ZWR9IGlzIHNtYWxsZXIgdGhhbiBudW1iZXIgb2YgbGFiZWwgY29sdW1uczoge2xlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBhcnRpZmFjdF9uYW1lID0gcmVzdWx0X3NldCBvciAicHJlZGljdGlvbiIKICAgIGxhYmVsc19pbnNpZGVfZGYgPSBzZXQobGFiZWxfY29sdW1ucykgJiBzZXQoZGF0YXNldC5jb2x1bW5zLnRvbGlzdCgpKQogICAgaWYgbGFiZWxzX2luc2lkZV9kZjoKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJUaGUgbGFiZWxzOiB7bGFiZWxzX2luc2lkZV9kZn0gYXJlIGFscmVhZHkgZXhpc3RlZCBpbiB0aGUgZGF0YWZyYW1lIgogICAgICAgICkKICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICBwcmVkX2RmID0gcGQuY29uY2F0KFtkYXRhc2V0LCBwZC5EYXRhRnJhbWUoeV9wcmVkLCBjb2x1bW5zPWxhYmVsX2NvbHVtbnMpXSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldChhcnRpZmFjdF9uYW1lLCBwcmVkX2RmLCBkYl9rZXk9cmVzdWx0X3NldCkK - origin_filename: '' - code_origin: '' entry_points: train: doc: "Training a model with the given dataset.\n\nexample::\n\n import mlrun\n\ @@ -18,6 +13,7 @@ spec: : \"my-model\",\n \"tag\": \"v1.0.0\",\n \"sample_set\"\ : \"./path/to/sample_set.csv\",\n \"test_set\": \"./path/to/test_set.csv\"\ ,\n \"CLASS_solver\": \"liblinear\",\n },\n )" + has_kwargs: true parameters: - name: context type: MLClientCtx @@ -71,12 +67,12 @@ spec: type: dict doc: Labels to log with the model default: null - name: train lineno: 126 + name: train has_varargs: false - has_kwargs: true evaluate: doc: Evaluating a model. Artifacts generated by the MLHandler. + has_kwargs: true parameters: - name: context type: MLClientCtx @@ -96,12 +92,12 @@ spec: doc: The target label(s) of the column(s) in the dataset. for Regression or Classification tasks. Mandatory when dataset is not a FeatureVector. default: null - name: evaluate lineno: 278 + name: evaluate has_varargs: false - has_kwargs: true predict: doc: Predicting dataset by a model. + has_kwargs: true parameters: - name: context type: MLClientCtx @@ -130,18 +126,22 @@ spec: doc: The db key to set name of the prediction result and the filename. Default to 'prediction'. default: null - name: predict lineno: 332 + name: predict has_varargs: false - has_kwargs: true - description: Automatic train, evaluate and predict functions for the ML frameworks - - Scikit-Learn, XGBoost and LightGBM. + build: + code_origin: '' + origin_filename: '' + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSBwYXRobGliIGltcG9ydCBQYXRoCmZyb20gdHlwaW5nIGltcG9ydCBBbnksIERpY3QsIExpc3QsIE9wdGlvbmFsLCBUdXBsZSwgVW5pb24KCmltcG9ydCBtbHJ1bgppbXBvcnQgbWxydW4uZGF0YXN0b3JlCmltcG9ydCBtbHJ1bi51dGlscwppbXBvcnQgcGFuZGFzIGFzIHBkCmZyb20gbWxydW4gaW1wb3J0IGZlYXR1cmVfc3RvcmUgYXMgZnMKZnJvbSBtbHJ1bi5kYXRhc3RvcmUgaW1wb3J0IERhdGFJdGVtCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmZyYW1ld29ya3MuYXV0b19tbHJ1biBpbXBvcnQgQXV0b01MUnVuCmZyb20gbWxydW4udXRpbHMuaGVscGVycyBpbXBvcnQgY3JlYXRlX2NsYXNzLCBjcmVhdGVfZnVuY3Rpb24KZnJvbSBza2xlYXJuLm1vZGVsX3NlbGVjdGlvbiBpbXBvcnQgdHJhaW5fdGVzdF9zcGxpdAoKUGF0aFR5cGUgPSBVbmlvbltzdHIsIFBhdGhdCgoKY2xhc3MgS1dBcmdzUHJlZml4ZXM6CiAgICBNT0RFTF9DTEFTUyA9ICJDTEFTU18iCiAgICBGSVQgPSAiRklUXyIKICAgIFRSQUlOID0gIlRSQUlOXyIKCgpkZWYgX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjOiBEaWN0LCBwcmVmaXhfa2V5OiBzdHIpIC0+IERpY3Rbc3RyLCBBbnldOgogICAgIiIiCiAgICBDb2xsZWN0IGFsbCB0aGUga2V5cyBmcm9tIHRoZSBnaXZlbiBkaWN0IHRoYXQgc3RhcnRzIHdpdGggdGhlIGdpdmVuIHByZWZpeCBhbmQgY3JlYXRlcyBhIG5ldyBkaWN0aW9uYXJ5IHdpdGggdGhlc2UKICAgIGtleXMuCgogICAgOnBhcmFtIHNyYzogICAgICAgICBUaGUgc291cmNlIGRpY3QgdG8gZXh0cmFjdCB0aGUgdmFsdWVzIGZyb20uCiAgICA6cGFyYW0gcHJlZml4X2tleTogIE9ubHkga2V5cyB3aXRoIHRoaXMgcHJlZml4IHdpbGwgYmUgcmV0dXJuZWQuIFRoZSBrZXlzIGluIHRoZSByZXN1bHQgZGljdCB3aWxsIGJlIHdpdGhvdXQgdGhpcwogICAgICAgICAgICAgICAgICAgICAgICBwcmVmaXguCiAgICAiIiIKICAgIHJldHVybiB7CiAgICAgICAga2V5LnJlcGxhY2UocHJlZml4X2tleSwgIiIpOiB2YWwKICAgICAgICBmb3Iga2V5LCB2YWwgaW4gc3JjLml0ZW1zKCkKICAgICAgICBpZiBrZXkuc3RhcnRzd2l0aChwcmVmaXhfa2V5KQogICAgfQoKCmRlZiBfZ2V0X2RhdGFmcmFtZSgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgZGF0YXNldDogRGF0YUl0ZW0sCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogVW5pb25bc3RyLCBMaXN0W3N0cl0sIGludCwgTGlzdFtpbnRdXSA9IE5vbmUsCikgLT4gVHVwbGVbcGQuRGF0YUZyYW1lLCBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dXToKICAgICIiIgogICAgR2V0dGluZyB0aGUgRGF0YUZyYW1lIG9mIHRoZSBkYXRhc2V0IGFuZCBkcm9wIHRoZSBjb2x1bW5zIGFjY29yZGluZ2x5LgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgIE1MUnVuIGNvbnRleHQuCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICBDYW4gYmUgZWl0aGVyIGEgbGlzdCBvZiBsaXN0cywgZGljdCwgVVJJIG9yIGEgRmVhdHVyZVZlY3Rvci4KICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgIFRoZSB0YXJnZXQgbGFiZWwocykgb2YgdGhlIGNvbHVtbihzKSBpbiB0aGUgZGF0YXNldC4gZm9yIFJlZ3Jlc3Npb24gb3IKICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgc3RyL2ludCBvciBhIGxpc3Qgb2Ygc3RyaW5ncy9pbnRzIHRoYXQgcmVwcmVzZW50IHRoZSBjb2x1bW4gbmFtZXMvaW5kaWNlcyB0byBkcm9wLgogICAgIiIiCiAgICAjIENoZWNrIGlmIGRhdGFzZXQgaXMgbGlzdC9kaWN0IGZpcnN0IChiZWZvcmUgdHJ5aW5nIHRvIGFjY2VzcyBhcnRpZmFjdF91cmwpCiAgICBpZiBpc2luc3RhbmNlKGRhdGFzZXQsIChsaXN0LCBkaWN0KSk6CiAgICAgICAgIyBsaXN0L2RpY3QgY2FzZToKICAgICAgICBpZiBub3QgbGFiZWxfY29sdW1uczoKICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICJsYWJlbF9jb2x1bW5zIG5vdCBwcm92aWRlZCwgbWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yIgogICAgICAgICAgICApCiAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICBkYXRhc2V0ID0gcGQuRGF0YUZyYW1lKGRhdGFzZXQpCiAgICAgICAgIyBDaGVja2luZyBpZiBkcm9wX2NvbHVtbnMgcHJvdmlkZWQgYnkgaW50ZWdlciB0eXBlOgogICAgICAgIGlmIGRyb3BfY29sdW1uczoKICAgICAgICAgICAgaWYgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIHN0cikgb3IgKAogICAgICAgICAgICAgICAgaXNpbnN0YW5jZShkcm9wX2NvbHVtbnMsIGxpc3QpCiAgICAgICAgICAgICAgICBhbmQgYW55KGlzaW5zdGFuY2UoY29sLCBzdHIpIGZvciBjb2wgaW4gZHJvcF9jb2x1bW5zKQogICAgICAgICAgICApOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuZXJyb3IoCiAgICAgICAgICAgICAgICAgICAgImRyb3BfY29sdW1ucyBtdXN0IGJlIGFuIGludGVnZXIvbGlzdCBvZiBpbnRlZ2VycyBpZiBub3QgcHJvdmlkZWQgd2l0aCBhIFVSSS9GZWF0dXJlVmVjdG9yIGRhdGFzZXQiCiAgICAgICAgICAgICAgICApCiAgICAgICAgICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICAgICAgICAgIGRhdGFzZXQuZHJvcChkcm9wX2NvbHVtbnMsIGF4aXM9MSwgaW5wbGFjZT1UcnVlKQogICAgZWxzZToKICAgICAgICAjIERhdGFzZXQgaXMgYSBEYXRhSXRlbSB3aXRoIGFydGlmYWN0X3VybCAoVVJJIG9yIEZlYXR1cmVWZWN0b3IpCiAgICAgICAgc3RvcmVfdXJpX3ByZWZpeCwgXyA9IG1scnVuLmRhdGFzdG9yZS5wYXJzZV9zdG9yZV91cmkoZGF0YXNldC5hcnRpZmFjdF91cmwpCgogICAgICAgICMgR2V0dGluZyB0aGUgZGF0YXNldDoKICAgICAgICBpZiBtbHJ1bi51dGlscy5TdG9yZVByZWZpeC5GZWF0dXJlVmVjdG9yID09IHN0b3JlX3VyaV9wcmVmaXg6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBsYWJlbF9jb2x1bW5zIG9yIGRhdGFzZXQubWV0YS5zdGF0dXMubGFiZWxfY29sdW1uCiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oZiJsYWJlbCBjb2x1bW5zOiB7bGFiZWxfY29sdW1uc30iKQogICAgICAgICAgICAjIEZlYXR1cmVWZWN0b3IgY2FzZToKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgZnYgPSBtbHJ1bi5kYXRhc3RvcmUuZ2V0X3N0b3JlX3Jlc291cmNlKGRhdGFzZXQuYXJ0aWZhY3RfdXJsKQogICAgICAgICAgICAgICAgZGF0YXNldCA9IGZ2LmdldF9vZmZsaW5lX2ZlYXR1cmVzKGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMpLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgICAgIGV4Y2VwdCBBdHRyaWJ1dGVFcnJvcjoKICAgICAgICAgICAgICAgICMgTGVhdmUgaGVyZSBmb3IgYmFja3dhcmRzIGNvbXBhdGliaWxpdHkKICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBmcy5nZXRfb2ZmbGluZV9mZWF0dXJlcygKICAgICAgICAgICAgICAgICAgICBkYXRhc2V0Lm1ldGEudXJpLCBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zCiAgICAgICAgICAgICAgICApLnRvX2RhdGFmcmFtZSgpCiAgICAgICAgZWxzZToKICAgICAgICAgICAgIyBzaW1wbGUgVVJMIGNhc2U6CiAgICAgICAgICAgIGlmIG5vdCBsYWJlbF9jb2x1bW5zOgogICAgICAgICAgICAgICAgY29udGV4dC5sb2dnZXIuaW5mbygKICAgICAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyBub3QgcHJvdmlkZWQsIG1hbmRhdG9yeSB3aGVuIGRhdGFzZXQgaXMgbm90IGEgRmVhdHVyZVZlY3RvciIKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKICAgICAgICAgICAgZGF0YXNldCA9IGRhdGFzZXQuYXNfZGYoKQogICAgICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgICAgICBpZiBhbGwoY29sIGluIGRhdGFzZXQgZm9yIGNvbCBpbiBkcm9wX2NvbHVtbnMpOgogICAgICAgICAgICAgICAgICAgIGRhdGFzZXQgPSBkYXRhc2V0LmRyb3AoZHJvcF9jb2x1bW5zLCBheGlzPTEpCiAgICAgICAgICAgICAgICBlbHNlOgogICAgICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgICAgICJub3QgYWxsIG9mIHRoZSBjb2x1bW5zIHRvIGRyb3AgaW4gdGhlIGRhdGFzZXQsIGRyb3AgY29sdW1ucyBwcm9jZXNzIHNraXBwZWQiCiAgICAgICAgICAgICAgICAgICAgKQoKICAgIHJldHVybiBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zCgoKZGVmIHRyYWluKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIG1vZGVsX2NsYXNzOiBzdHIsCiAgICBsYWJlbF9jb2x1bW5zOiBPcHRpb25hbFtVbmlvbltzdHIsIExpc3Rbc3RyXV1dID0gTm9uZSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIG1vZGVsX25hbWU6IHN0ciA9ICJtb2RlbCIsCiAgICB0YWc6IHN0ciA9ICIiLAogICAgc2FtcGxlX3NldDogRGF0YUl0ZW0gPSBOb25lLAogICAgdGVzdF9zZXQ6IERhdGFJdGVtID0gTm9uZSwKICAgIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZTogZmxvYXQgPSBOb25lLAogICAgcmFuZG9tX3N0YXRlOiBpbnQgPSBOb25lLAogICAgbGFiZWxzOiBkaWN0ID0gTm9uZSwKICAgICoqa3dhcmdzLAopOgogICAgIiIiCiAgICBUcmFpbmluZyBhIG1vZGVsIHdpdGggdGhlIGdpdmVuIGRhdGFzZXQuCgogICAgZXhhbXBsZTo6CgogICAgICAgIGltcG9ydCBtbHJ1bgogICAgICAgIHByb2plY3QgPSBtbHJ1bi5nZXRfb3JfY3JlYXRlX3Byb2plY3QoIm15LXByb2plY3QiKQogICAgICAgIHByb2plY3Quc2V0X2Z1bmN0aW9uKCJodWI6Ly9hdXRvX3RyYWluZXIiLCAidHJhaW4iKQogICAgICAgIHRyYWluZXJfcnVuID0gcHJvamVjdC5ydW4oCiAgICAgICAgICAgIG5hbWU9InRyYWluIiwKICAgICAgICAgICAgaGFuZGxlcj0idHJhaW4iLAogICAgICAgICAgICBpbnB1dHM9eyJkYXRhc2V0IjogIi4vcGF0aC90by9kYXRhc2V0LmNzdiJ9LAogICAgICAgICAgICBwYXJhbXM9ewogICAgICAgICAgICAgICAgIm1vZGVsX2NsYXNzIjogInNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbiIsCiAgICAgICAgICAgICAgICAibGFiZWxfY29sdW1ucyI6ICJsYWJlbCIsCiAgICAgICAgICAgICAgICAiZHJvcF9jb2x1bW5zIjogImlkIiwKICAgICAgICAgICAgICAgICJtb2RlbF9uYW1lIjogIm15LW1vZGVsIiwKICAgICAgICAgICAgICAgICJ0YWciOiAidjEuMC4wIiwKICAgICAgICAgICAgICAgICJzYW1wbGVfc2V0IjogIi4vcGF0aC90by9zYW1wbGVfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAidGVzdF9zZXQiOiAiLi9wYXRoL3RvL3Rlc3Rfc2V0LmNzdiIsCiAgICAgICAgICAgICAgICAiQ0xBU1Nfc29sdmVyIjogImxpYmxpbmVhciIsCiAgICAgICAgICAgIH0sCiAgICAgICAgKQoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dAogICAgOnBhcmFtIGRhdGFzZXQ6ICAgICAgICAgICAgICAgICBUaGUgZGF0YXNldCB0byB0cmFpbiB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gbW9kZWxfY2xhc3M6ICAgICAgICAgICAgIFRoZSBjbGFzcyBvZiB0aGUgbW9kZWwsIGUuZy4gYHNrbGVhcm4ubGluZWFyX21vZGVsLkxvZ2lzdGljUmVncmVzc2lvbmAKICAgIDpwYXJhbSBsYWJlbF9jb2x1bW5zOiAgICAgICAgICAgVGhlIHRhcmdldCBsYWJlbChzKSBvZiB0aGUgY29sdW1uKHMpIGluIHRoZSBkYXRhc2V0LiBmb3IgUmVncmVzc2lvbiBvcgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBDbGFzc2lmaWNhdGlvbiB0YXNrcy4gTWFuZGF0b3J5IHdoZW4gZGF0YXNldCBpcyBub3QgYSBGZWF0dXJlVmVjdG9yLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIgb3IgYSBsaXN0IG9mIHN0cmluZ3MgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbnMgdG8gZHJvcAogICAgOnBhcmFtIG1vZGVsX25hbWU6ICAgICAgICAgICAgICBUaGUgbW9kZWwncyBuYW1lIHRvIHVzZSBmb3Igc3RvcmluZyB0aGUgbW9kZWwgYXJ0aWZhY3QsIGRlZmF1bHQgdG8gJ21vZGVsJwogICAgOnBhcmFtIHRhZzogICAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwncyB0YWcgdG8gbG9nIHdpdGgKICAgIDpwYXJhbSBzYW1wbGVfc2V0OiAgICAgICAgICAgICAgQSBzYW1wbGUgc2V0IG9mIGlucHV0cyBmb3IgdGhlIG1vZGVsIGZvciBsb2dnaW5nIGl0cyBzdGF0cyBhbG9uZyB0aGUgbW9kZWwgaW4gZmF2b3VyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9mIG1vZGVsIG1vbml0b3JpbmcuIENhbiBiZSBlaXRoZXIgYSBVUkkgb3IgYSBGZWF0dXJlVmVjdG9yCiAgICA6cGFyYW0gdGVzdF9zZXQ6ICAgICAgICAgICAgICAgIFRoZSB0ZXN0IHNldCB0byB0cmFpbiB0aGUgbW9kZWwgd2l0aC4KICAgIDpwYXJhbSB0cmFpbl90ZXN0X3NwbGl0X3NpemU6ICAgaWYgdGVzdF9zZXQgd2FzIHByb3ZpZGVkIHRoZW4gdGhpcyBhcmd1bWVudCBpcyBpZ25vcmVkLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBTaG91bGQgYmUgYmV0d2VlbiAwLjAgYW5kIDEuMCBhbmQgcmVwcmVzZW50IHRoZSBwcm9wb3J0aW9uIG9mIHRoZSBkYXRhc2V0IHRvIGluY2x1ZGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgaW4gdGhlIHRlc3Qgc3BsaXQuIFRoZSBzaXplIG9mIHRoZSBUcmFpbmluZyBzZXQgaXMgc2V0IHRvIHRoZSBjb21wbGVtZW50IG9mIHRoaXMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWUuIERlZmF1bHQgPSAwLjIKICAgIDpwYXJhbSByYW5kb21fc3RhdGU6ICAgICAgICAgICAgUmVsZXZhbnQgb25seSB3aGVuIHVzaW5nIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZS4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgQSByYW5kb20gc3RhdGUgc2VlZCB0byBzaHVmZmxlIHRoZSBkYXRhLiBGb3IgbW9yZSBpbmZvcm1hdGlvbiwgc2VlOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBodHRwczovL3NjaWtpdC1sZWFybi5vcmcvc3RhYmxlL2dsb3NzYXJ5Lmh0bWwjdGVybS1yYW5kb21fc3RhdGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgTm90aWNlIHRoYXQgaGVyZSB3ZSBvbmx5IHBhc3MgaW50ZWdlciB2YWx1ZXMuCiAgICA6cGFyYW0gbGFiZWxzOiAgICAgICAgICAgICAgICAgIExhYmVscyB0byBsb2cgd2l0aCB0aGUgbW9kZWwKICAgIDpwYXJhbSBrd2FyZ3M6ICAgICAgICAgICAgICAgICAgSGVyZSB5b3UgY2FuIHBhc3Mga2V5d29yZCBhcmd1bWVudHMgd2l0aCBwcmVmaXhlcywKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdGhhdCB3aWxsIGJlIHBhcnNlZCBhbmQgcGFzc2VkIHRvIHRoZSByZWxldmFudCBmdW5jdGlvbiwgYnkgdGhlIGZvbGxvd2luZyBwcmVmaXhlczoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgQ0xBU1NfYCAtIGZvciB0aGUgbW9kZWwgY2xhc3MgYXJndW1lbnRzCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIC0gYEZJVF9gIC0gZm9yIHRoZSBgZml0YCBmdW5jdGlvbiBhcmd1bWVudHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgLSBgVFJBSU5fYCAtIGZvciB0aGUgYHRyYWluYCBmdW5jdGlvbiAoaW4geGdiIG9yIGxnYm0gdHJhaW4gZnVuY3Rpb24gLSBmdXR1cmUpCgogICAgIiIiCiAgICAjIFZhbGlkYXRlIGlucHV0czoKICAgICMgQ2hlY2sgaWYgZXhhY3RseSBvbmUgb2YgdGhlbSBpcyBzdXBwbGllZDoKICAgIGlmIHRlc3Rfc2V0IGlzIE5vbmU6CiAgICAgICAgaWYgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGlzIE5vbmU6CiAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAidGVzdF9zZXQgb3IgdHJhaW5fdGVzdF9zcGxpdF9zaXplIGFyZSBub3QgcHJvdmlkZWQsIHNldHRpbmcgdHJhaW5fdGVzdF9zcGxpdF9zaXplIHRvIDAuMiIKICAgICAgICAgICAgKQogICAgICAgICAgICB0cmFpbl90ZXN0X3NwbGl0X3NpemUgPSAwLjIKCiAgICBlbGlmIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICAidGVzdF9zZXQgcHJvdmlkZWQsIGlnbm9yaW5nIGdpdmVuIHRyYWluX3Rlc3Rfc3BsaXRfc2l6ZSB2YWx1ZSIKICAgICAgICApCiAgICAgICAgdHJhaW5fdGVzdF9zcGxpdF9zaXplID0gTm9uZQoKICAgICMgR2V0IERhdGFGcmFtZSBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgR2V0dGluZyB0aGUgc2FtcGxlIHNldDoKICAgIGlmIHNhbXBsZV9zZXQgaXMgTm9uZToKICAgICAgICBjb250ZXh0LmxvZ2dlci5pbmZvKAogICAgICAgICAgICBmIlNhbXBsZSBzZXQgbm90IGdpdmVuLCB1c2luZyB0aGUgd2hvbGUgdHJhaW5pbmcgc2V0IGFzIHRoZSBzYW1wbGUgc2V0IgogICAgICAgICkKICAgICAgICBzYW1wbGVfc2V0ID0gZGF0YXNldAogICAgZWxzZToKICAgICAgICBzYW1wbGVfc2V0LCBfID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICAgICAgZGF0YXNldD1zYW1wbGVfc2V0LAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgICAgIGRyb3BfY29sdW1ucz1kcm9wX2NvbHVtbnMsCiAgICAgICAgKQoKICAgICMgUGFyc2luZyBrd2FyZ3M6CiAgICAjIFRPRE86IFVzZSBpbiB4Z2Igb3IgbGdibSB0cmFpbiBmdW5jdGlvbi4KICAgIHRyYWluX2t3YXJncyA9IF9nZXRfc3ViX2RpY3RfYnlfcHJlZml4KHNyYz1rd2FyZ3MsIHByZWZpeF9rZXk9S1dBcmdzUHJlZml4ZXMuVFJBSU4pCiAgICBmaXRfa3dhcmdzID0gX2dldF9zdWJfZGljdF9ieV9wcmVmaXgoc3JjPWt3YXJncywgcHJlZml4X2tleT1LV0FyZ3NQcmVmaXhlcy5GSVQpCiAgICBtb2RlbF9jbGFzc19rd2FyZ3MgPSBfZ2V0X3N1Yl9kaWN0X2J5X3ByZWZpeCgKICAgICAgICBzcmM9a3dhcmdzLCBwcmVmaXhfa2V5PUtXQXJnc1ByZWZpeGVzLk1PREVMX0NMQVNTCiAgICApCgogICAgIyBDaGVjayBpZiBtb2RlbCBvciBmdW5jdGlvbjoKICAgIGlmIGhhc2F0dHIobW9kZWxfY2xhc3MsICJ0cmFpbiIpOgogICAgICAgICMgVE9ETzogTmVlZCB0byBjYWxsOiBtb2RlbCgpLCBhZnRlcndhcmRzIHRvIHN0YXJ0IHRoZSB0cmFpbiBmdW5jdGlvbi4KICAgICAgICAjIG1vZGVsID0gY3JlYXRlX2Z1bmN0aW9uKGYie21vZGVsX2NsYXNzfS50cmFpbiIpCiAgICAgICAgcmFpc2UgTm90SW1wbGVtZW50ZWRFcnJvcgogICAgZWxzZToKICAgICAgICAjIENyZWF0aW5nIG1vZGVsIGluc3RhbmNlOgogICAgICAgIG1vZGVsID0gY3JlYXRlX2NsYXNzKG1vZGVsX2NsYXNzKSgqKm1vZGVsX2NsYXNzX2t3YXJncykKCiAgICB4ID0gZGF0YXNldC5kcm9wKGxhYmVsX2NvbHVtbnMsIGF4aXM9MSkKICAgIHkgPSBkYXRhc2V0W2xhYmVsX2NvbHVtbnNdCiAgICBpZiB0cmFpbl90ZXN0X3NwbGl0X3NpemU6CiAgICAgICAgeF90cmFpbiwgeF90ZXN0LCB5X3RyYWluLCB5X3Rlc3QgPSB0cmFpbl90ZXN0X3NwbGl0KAogICAgICAgICAgICB4LCB5LCB0ZXN0X3NpemU9dHJhaW5fdGVzdF9zcGxpdF9zaXplLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICB4X3RyYWluLCB5X3RyYWluID0geCwgeQoKICAgICAgICB0ZXN0X3NldCA9IHRlc3Rfc2V0LmFzX2RmKCkKICAgICAgICBpZiBkcm9wX2NvbHVtbnM6CiAgICAgICAgICAgIHRlc3Rfc2V0ID0gZGF0YXNldC5kcm9wKGRyb3BfY29sdW1ucywgYXhpcz0xKQoKICAgICAgICB4X3Rlc3QsIHlfdGVzdCA9IHRlc3Rfc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKSwgdGVzdF9zZXRbbGFiZWxfY29sdW1uc10KCiAgICBBdXRvTUxSdW4uYXBwbHlfbWxydW4oCiAgICAgICAgbW9kZWw9bW9kZWwsCiAgICAgICAgbW9kZWxfbmFtZT1tb2RlbF9uYW1lLAogICAgICAgIGNvbnRleHQ9Y29udGV4dCwKICAgICAgICB0YWc9dGFnLAogICAgICAgIHNhbXBsZV9zZXQ9c2FtcGxlX3NldCwKICAgICAgICB5X2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICB0ZXN0X3NldD10ZXN0X3NldCwKICAgICAgICB4X3Rlc3Q9eF90ZXN0LAogICAgICAgIHlfdGVzdD15X3Rlc3QsCiAgICAgICAgYXJ0aWZhY3RzPWNvbnRleHQuYXJ0aWZhY3RzLAogICAgICAgIGxhYmVscz1sYWJlbHMsCiAgICApCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYidHJhaW5pbmcgJ3ttb2RlbF9uYW1lfSciKQogICAgbW9kZWwuZml0KHhfdHJhaW4sIHlfdHJhaW4sICoqZml0X2t3YXJncykKCgpkZWYgZXZhbHVhdGUoCiAgICBjb250ZXh0OiBNTENsaWVudEN0eCwKICAgIG1vZGVsOiBzdHIsCiAgICBkYXRhc2V0OiBtbHJ1bi5EYXRhSXRlbSwKICAgIGRyb3BfY29sdW1uczogTGlzdFtzdHJdID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgKiprd2FyZ3MsCik6CiAgICAiIiIKICAgIEV2YWx1YXRpbmcgYSBtb2RlbC4gQXJ0aWZhY3RzIGdlbmVyYXRlZCBieSB0aGUgTUxIYW5kbGVyLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgICAgTUxSdW4gY29udGV4dC4KICAgIDpwYXJhbSBtb2RlbDogICAgICAgICAgICAgICAgICAgVGhlIG1vZGVsIFN0b3JlIHBhdGguCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICAgICAgIFRoZSBkYXRhc2V0IHRvIGV2YWx1YXRlIHRoZSBtb2RlbCBvbi4gQ2FuIGJlIGVpdGhlciBhIFVSSSBvciBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gZHJvcF9jb2x1bW5zOiAgICAgICAgICAgIHN0ciBvciBhIGxpc3Qgb2Ygc3RyaW5ncyB0aGF0IHJlcHJlc2VudCB0aGUgY29sdW1ucyB0byBkcm9wLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0ga3dhcmdzOiAgICAgICAgICAgICAgICAgIEhlcmUgeW91IGNhbiBwYXNzIGtleXdvcmQgYXJndW1lbnRzIHRvIHRoZSBwcmVkaWN0IGZ1bmN0aW9uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIChQUkVESUNUXyBwcmVmaXggaXMgbm90IHJlcXVpcmVkKS4KICAgICIiIgogICAgIyBHZXQgZGF0YXNldCBieSBVUkwgb3IgYnkgRmVhdHVyZVZlY3RvcjoKICAgIGRhdGFzZXQsIGxhYmVsX2NvbHVtbnMgPSBfZ2V0X2RhdGFmcmFtZSgKICAgICAgICBjb250ZXh0PWNvbnRleHQsCiAgICAgICAgZGF0YXNldD1kYXRhc2V0LAogICAgICAgIGxhYmVsX2NvbHVtbnM9bGFiZWxfY29sdW1ucywKICAgICAgICBkcm9wX2NvbHVtbnM9ZHJvcF9jb2x1bW5zLAogICAgKQoKICAgICMgUGFyc2luZyBsYWJlbF9jb2x1bW5zOgogICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMgPSBbXQogICAgaWYgbGFiZWxfY29sdW1uczoKICAgICAgICBsYWJlbF9jb2x1bW5zID0gKAogICAgICAgICAgICBsYWJlbF9jb2x1bW5zIGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgbGlzdCkgZWxzZSBbbGFiZWxfY29sdW1uc10KICAgICAgICApCiAgICAgICAgZm9yIGxjIGluIGxhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGlmIGZzLmNvbW1vbi5mZWF0dXJlX3NlcGFyYXRvciBpbiBsYzoKICAgICAgICAgICAgICAgIGZlYXR1cmVfc2V0X25hbWUsIGxhYmVsX25hbWUsIGFsaWFzID0gZnMuY29tbW9uLnBhcnNlX2ZlYXR1cmVfc3RyaW5nKGxjKQogICAgICAgICAgICAgICAgcGFyc2VkX2xhYmVsX2NvbHVtbnMuYXBwZW5kKGFsaWFzIG9yIGxhYmVsX25hbWUpCiAgICAgICAgaWYgcGFyc2VkX2xhYmVsX2NvbHVtbnM6CiAgICAgICAgICAgIGxhYmVsX2NvbHVtbnMgPSBwYXJzZWRfbGFiZWxfY29sdW1ucwoKICAgIHggPSBkYXRhc2V0LmRyb3AobGFiZWxfY29sdW1ucywgYXhpcz0xKQogICAgeSA9IGRhdGFzZXRbbGFiZWxfY29sdW1uc10KCiAgICAjIExvYWRpbmcgdGhlIG1vZGVsIGFuZCBwcmVkaWN0aW5nOgogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWwsIGNvbnRleHQ9Y29udGV4dCwgbW9kZWxfbmFtZT0ibW9kZWxfTGluZWFyUmVncmVzc2lvbiIKICAgICkKICAgIEF1dG9NTFJ1bi5hcHBseV9tbHJ1bihtb2RlbF9oYW5kbGVyLm1vZGVsLCB5X3Rlc3Q9eSwgbW9kZWxfcGF0aD1tb2RlbCkKCiAgICBjb250ZXh0LmxvZ2dlci5pbmZvKGYiZXZhbHVhdGluZyAne21vZGVsX2hhbmRsZXIubW9kZWxfbmFtZX0nIikKICAgIG1vZGVsX2hhbmRsZXIubW9kZWwucHJlZGljdCh4LCAqKmt3YXJncykKCgpkZWYgcHJlZGljdCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWw6IHN0ciwKICAgIGRhdGFzZXQ6IG1scnVuLkRhdGFJdGVtLAogICAgZHJvcF9jb2x1bW5zOiBVbmlvbltzdHIsIExpc3Rbc3RyXSwgaW50LCBMaXN0W2ludF1dID0gTm9uZSwKICAgIGxhYmVsX2NvbHVtbnM6IE9wdGlvbmFsW1VuaW9uW3N0ciwgTGlzdFtzdHJdXV0gPSBOb25lLAogICAgcmVzdWx0X3NldDogT3B0aW9uYWxbc3RyXSA9IE5vbmUsCiAgICAqKmt3YXJncywKKToKICAgICIiIgogICAgUHJlZGljdGluZyBkYXRhc2V0IGJ5IGEgbW9kZWwuCgogICAgOnBhcmFtIGNvbnRleHQ6ICAgICAgICAgICAgICAgICBNTFJ1biBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsOiAgICAgICAgICAgICAgICAgICBUaGUgbW9kZWwgU3RvcmUgcGF0aC4KICAgIDpwYXJhbSBkYXRhc2V0OiAgICAgICAgICAgICAgICAgVGhlIGRhdGFzZXQgdG8gcHJlZGljdCB0aGUgbW9kZWwgb24uIENhbiBiZSBlaXRoZXIgYSBVUkksIGEgRmVhdHVyZVZlY3RvciBvciBhCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNhbXBsZSBpbiBhIHNoYXBlIG9mIGEgbGlzdC9kaWN0LgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBXaGVuIHBhc3NpbmcgYSBzYW1wbGUsIHBhc3MgdGhlIGRhdGFzZXQgYXMgYSBmaWVsZCBpbiBgcGFyYW1zYCBpbnN0ZWFkIG9mIGBpbnB1dHNgLgogICAgOnBhcmFtIGRyb3BfY29sdW1uczogICAgICAgICAgICBzdHIvaW50IG9yIGEgbGlzdCBvZiBzdHJpbmdzL2ludHMgdGhhdCByZXByZXNlbnQgdGhlIGNvbHVtbiBuYW1lcy9pbmRpY2VzIHRvIGRyb3AuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFdoZW4gdGhlIGRhdGFzZXQgaXMgYSBsaXN0L2RpY3QgdGhpcyBwYXJhbWV0ZXIgc2hvdWxkIGJlIHJlcHJlc2VudGVkIGJ5IGludGVnZXJzLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbnM6ICAgICAgICAgICBUaGUgdGFyZ2V0IGxhYmVsKHMpIG9mIHRoZSBjb2x1bW4ocykgaW4gdGhlIGRhdGFzZXQuIGZvciBSZWdyZXNzaW9uIG9yCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIENsYXNzaWZpY2F0aW9uIHRhc2tzLiBNYW5kYXRvcnkgd2hlbiBkYXRhc2V0IGlzIG5vdCBhIEZlYXR1cmVWZWN0b3IuCiAgICA6cGFyYW0gcmVzdWx0X3NldDogICAgICAgICAgICAgIFRoZSBkYiBrZXkgdG8gc2V0IG5hbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0IGFuZCB0aGUgZmlsZW5hbWUuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHQgdG8gJ3ByZWRpY3Rpb24nLgogICAgOnBhcmFtIGt3YXJnczogICAgICAgICAgICAgICAgICBIZXJlIHlvdSBjYW4gcGFzcyBrZXl3b3JkIGFyZ3VtZW50cyB0byB0aGUgcHJlZGljdCBmdW5jdGlvbgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAoUFJFRElDVF8gcHJlZml4IGlzIG5vdCByZXF1aXJlZCkuCiAgICAiIiIKICAgICMgR2V0IGRhdGFzZXQgYnkgVVJMIG9yIGJ5IEZlYXR1cmVWZWN0b3I6CiAgICBkYXRhc2V0LCBsYWJlbF9jb2x1bW5zID0gX2dldF9kYXRhZnJhbWUoCiAgICAgICAgY29udGV4dD1jb250ZXh0LAogICAgICAgIGRhdGFzZXQ9ZGF0YXNldCwKICAgICAgICBsYWJlbF9jb2x1bW5zPWxhYmVsX2NvbHVtbnMsCiAgICAgICAgZHJvcF9jb2x1bW5zPWRyb3BfY29sdW1ucywKICAgICkKCiAgICAjIGxvYWRpbmcgdGhlIG1vZGVsLCBhbmQgZ2V0dGluZyB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIG1vZGVsX2hhbmRsZXIgPSBBdXRvTUxSdW4ubG9hZF9tb2RlbChtb2RlbF9wYXRoPW1vZGVsLCBjb250ZXh0PWNvbnRleHQpCgogICAgIyBGaXggZmVhdHVyZSBuYW1lcyBmb3IgbW9kZWxzIHRoYXQgcmVxdWlyZSB0aGVtIChlLmcuLCBYR0Jvb3N0KQogICAgIyBXaGVuIGRhdGFzZXQgY29tZXMgZnJvbSBhIGxpc3QsIHBhbmRhcyBhc3NpZ25zIGRlZmF1bHQgaW50ZWdlciBjb2x1bW4gbmFtZXMKICAgICMgYnV0IHNvbWUgbW9kZWxzIGV4cGVjdCBzcGVjaWZpYyBmZWF0dXJlIG5hbWVzIHRoZXkgd2VyZSB0cmFpbmVkIHdpdGgKICAgIGlmIGhhc2F0dHIobW9kZWxfaGFuZGxlci5tb2RlbCwgJ2ZlYXR1cmVfbmFtZXNfaW5fJyk6CiAgICAgICAgZXhwZWN0ZWRfZmVhdHVyZXMgPSBtb2RlbF9oYW5kbGVyLm1vZGVsLmZlYXR1cmVfbmFtZXNfaW5fCiAgICAgICAgaWYgbGVuKGRhdGFzZXQuY29sdW1ucykgPT0gbGVuKGV4cGVjdGVkX2ZlYXR1cmVzKToKICAgICAgICAgICAgIyBPbmx5IHJlbmFtZSBpZiB0aGUgbnVtYmVyIG9mIGNvbHVtbnMgbWF0Y2hlcwogICAgICAgICAgICAjIFRoaXMgaGFuZGxlcyB0aGUgY2FzZSB3aGVyZSBhIGxpc3Qgd2FzIGNvbnZlcnRlZCB0byBEYXRhRnJhbWUgd2l0aCBkZWZhdWx0IGNvbHVtbiBuYW1lcwogICAgICAgICAgICBpZiBub3QgYWxsKGNvbCA9PSBmZWF0IGZvciBjb2wsIGZlYXQgaW4gemlwKGRhdGFzZXQuY29sdW1ucywgZXhwZWN0ZWRfZmVhdHVyZXMpKToKICAgICAgICAgICAgICAgIGNvbnRleHQubG9nZ2VyLmluZm8oCiAgICAgICAgICAgICAgICAgICAgZiJSZW5hbWluZyBkYXRhc2V0IGNvbHVtbnMgdG8gbWF0Y2ggbW9kZWwncyBleHBlY3RlZCBmZWF0dXJlIG5hbWVzIgogICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgZGF0YXNldC5jb2x1bW5zID0gZXhwZWN0ZWRfZmVhdHVyZXMKCiAgICAjIERyb3BwaW5nIGxhYmVsIGNvbHVtbnMgaWYgbmVjZXNzYXJ5OgogICAgaWYgbm90IGxhYmVsX2NvbHVtbnM6CiAgICAgICAgbGFiZWxfY29sdW1ucyA9IFtdCiAgICBlbGlmIGlzaW5zdGFuY2UobGFiZWxfY29sdW1ucywgc3RyKToKICAgICAgICBsYWJlbF9jb2x1bW5zID0gW2xhYmVsX2NvbHVtbnNdCgogICAgIyBQcmVkaWN0aW5nOgogICAgY29udGV4dC5sb2dnZXIuaW5mbyhmIm1ha2luZyBwcmVkaWN0aW9uIGJ5ICd7bW9kZWxfaGFuZGxlci5tb2RlbF9uYW1lfSciKQogICAgeV9wcmVkID0gbW9kZWxfaGFuZGxlci5tb2RlbC5wcmVkaWN0KGRhdGFzZXQsICoqa3dhcmdzKQoKICAgICMgUHJlcGFyaW5nIGFuZCB2YWxpZGF0aW5nIGxhYmVsIGNvbHVtbnMgZm9yIHRoZSBkYXRhZnJhbWUgb2YgdGhlIHByZWRpY3Rpb24gcmVzdWx0OgogICAgbnVtX3ByZWRpY3RlZCA9IDEgaWYgbGVuKHlfcHJlZC5zaGFwZSkgPT0gMSBlbHNlIHlfcHJlZC5zaGFwZVsxXQoKICAgIGlmIG51bV9wcmVkaWN0ZWQgPiBsZW4obGFiZWxfY29sdW1ucyk6CiAgICAgICAgaWYgbnVtX3ByZWRpY3RlZCA9PSAxOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zID0gWyJwcmVkaWN0ZWQgbGFiZWxzIl0KICAgICAgICBlbHNlOgogICAgICAgICAgICBsYWJlbF9jb2x1bW5zLmV4dGVuZCgKICAgICAgICAgICAgICAgIFsKICAgICAgICAgICAgICAgICAgICBmInByZWRpY3RlZF9sYWJlbF97aSArIDEgKyBsZW4obGFiZWxfY29sdW1ucyl9IgogICAgICAgICAgICAgICAgICAgIGZvciBpIGluIHJhbmdlKG51bV9wcmVkaWN0ZWQgLSBsZW4obGFiZWxfY29sdW1ucykpCiAgICAgICAgICAgICAgICBdCiAgICAgICAgICAgICkKICAgIGVsaWYgbnVtX3ByZWRpY3RlZCA8IGxlbihsYWJlbF9jb2x1bW5zKToKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJudW1iZXIgb2YgcHJlZGljdGVkIGxhYmVsczoge251bV9wcmVkaWN0ZWR9IGlzIHNtYWxsZXIgdGhhbiBudW1iZXIgb2YgbGFiZWwgY29sdW1uczoge2xlbihsYWJlbF9jb2x1bW5zKX0iCiAgICAgICAgKQogICAgICAgIHJhaXNlIFZhbHVlRXJyb3IKCiAgICBhcnRpZmFjdF9uYW1lID0gcmVzdWx0X3NldCBvciAicHJlZGljdGlvbiIKICAgIGxhYmVsc19pbnNpZGVfZGYgPSBzZXQobGFiZWxfY29sdW1ucykgJiBzZXQoZGF0YXNldC5jb2x1bW5zLnRvbGlzdCgpKQogICAgaWYgbGFiZWxzX2luc2lkZV9kZjoKICAgICAgICBjb250ZXh0LmxvZ2dlci5lcnJvcigKICAgICAgICAgICAgZiJUaGUgbGFiZWxzOiB7bGFiZWxzX2luc2lkZV9kZn0gYXJlIGFscmVhZHkgZXhpc3RlZCBpbiB0aGUgZGF0YWZyYW1lIgogICAgICAgICkKICAgICAgICByYWlzZSBWYWx1ZUVycm9yCiAgICBwcmVkX2RmID0gcGQuY29uY2F0KFtkYXRhc2V0LCBwZC5EYXRhRnJhbWUoeV9wcmVkLCBjb2x1bW5zPWxhYmVsX2NvbHVtbnMpXSwgYXhpcz0xKQogICAgY29udGV4dC5sb2dfZGF0YXNldChhcnRpZmFjdF9uYW1lLCBwcmVkX2RmLCBkYl9rZXk9cmVzdWx0X3NldCkK + command: '' default_handler: train image: mlrun/mlrun - command: '' + disable_auto_mount: false + description: Automatic train, evaluate and predict functions for the ML frameworks + - Scikit-Learn, XGBoost and LightGBM. metadata: - tag: '' - name: auto-trainer categories: - machine-learning - model-training + tag: '' + name: auto-trainer diff --git a/functions/src/auto_trainer/requirements.txt b/functions/src/auto_trainer/requirements.txt index b14a0293c..1f735026c 100644 --- a/functions/src/auto_trainer/requirements.txt +++ b/functions/src/auto_trainer/requirements.txt @@ -1,4 +1,4 @@ pandas -scikit-learn<1.4.0 +scikit-learn~=1.5.2 xgboost<2.0.0 plotly diff --git a/functions/src/auto_trainer/test_auto_trainer.py b/functions/src/auto_trainer/test_auto_trainer.py index 06b553a35..2874f3368 100644 --- a/functions/src/auto_trainer/test_auto_trainer.py +++ b/functions/src/auto_trainer/test_auto_trainer.py @@ -78,6 +78,10 @@ def _assert_train_handler(train_run): @pytest.mark.parametrize("model", MODELS) +@pytest.mark.skipif( + condition=not _validate_environment_variables(), + reason="Project's environment variables are not set", +) def test_train(model: Tuple[str, str]): dataset, label_columns = _get_dataset(model[1]) is_test_passed = True diff --git a/functions/src/describe/requirements.txt b/functions/src/describe/requirements.txt index 7a15c8465..ac445e6d6 100644 --- a/functions/src/describe/requirements.txt +++ b/functions/src/describe/requirements.txt @@ -1,4 +1,4 @@ -scikit-learn~=1.5 +scikit-learn~=1.5.2 plotly~=5.23 pytest~=7.0.1 matplotlib~=3.5.1 diff --git a/functions/src/gen_class_data/requirements.txt b/functions/src/gen_class_data/requirements.txt index fc53d535f..e265290f6 100644 --- a/functions/src/gen_class_data/requirements.txt +++ b/functions/src/gen_class_data/requirements.txt @@ -1,2 +1,2 @@ pandas -scikit-learn~=1.5 \ No newline at end of file +scikit-learn~=1.5.2 \ No newline at end of file diff --git a/functions/src/sklearn_classifier/requirements.txt b/functions/src/sklearn_classifier/requirements.txt index 97a565a9e..113d4a02a 100644 --- a/functions/src/sklearn_classifier/requirements.txt +++ b/functions/src/sklearn_classifier/requirements.txt @@ -1,5 +1,5 @@ pandas -scikit-learn~=1.5 +scikit-learn~=1.5.2 matplotlib seaborn scikit-plot From 94ff0d5d1758858fa58aa85be397be8bd8fff6f4 Mon Sep 17 00:00:00 2001 From: tomerbv Date: Wed, 11 Feb 2026 18:22:39 +0200 Subject: [PATCH 14/15] revert sklearn_classifier.py changes change XGBRegressor to LGBMRegressor --- functions/src/auto_trainer/auto_trainer.py | 14 -- functions/src/auto_trainer/requirements.txt | 2 +- .../src/auto_trainer/test_auto_trainer.py | 2 +- .../src/sklearn_classifier/function.yaml | 109 ++-------------- functions/src/sklearn_classifier/item.yaml | 4 +- .../src/sklearn_classifier/requirements.txt | 2 +- .../sklearn_classifier/sklearn_classifier.py | 121 +----------------- .../test_sklearn_classifier.py | 28 ++-- 8 files changed, 29 insertions(+), 253 deletions(-) diff --git a/functions/src/auto_trainer/auto_trainer.py b/functions/src/auto_trainer/auto_trainer.py index ab2c6ee88..4e53e5b7e 100755 --- a/functions/src/auto_trainer/auto_trainer.py +++ b/functions/src/auto_trainer/auto_trainer.py @@ -366,20 +366,6 @@ def predict( # loading the model, and getting the model handler: model_handler = AutoMLRun.load_model(model_path=model, context=context) - # Fix feature names for models that require them (e.g., XGBoost) - # When dataset comes from a list, pandas assigns default integer column names - # but some models expect specific feature names they were trained with - if hasattr(model_handler.model, 'feature_names_in_'): - expected_features = model_handler.model.feature_names_in_ - if len(dataset.columns) == len(expected_features): - # Only rename if the number of columns matches - # This handles the case where a list was converted to DataFrame with default column names - if not all(col == feat for col, feat in zip(dataset.columns, expected_features)): - context.logger.info( - f"Renaming dataset columns to match model's expected feature names" - ) - dataset.columns = expected_features - # Dropping label columns if necessary: if not label_columns: label_columns = [] diff --git a/functions/src/auto_trainer/requirements.txt b/functions/src/auto_trainer/requirements.txt index 1f735026c..262f5e9f2 100644 --- a/functions/src/auto_trainer/requirements.txt +++ b/functions/src/auto_trainer/requirements.txt @@ -1,4 +1,4 @@ pandas scikit-learn~=1.5.2 -xgboost<2.0.0 +lightgbm plotly diff --git a/functions/src/auto_trainer/test_auto_trainer.py b/functions/src/auto_trainer/test_auto_trainer.py index 2874f3368..fe5c051a4 100644 --- a/functions/src/auto_trainer/test_auto_trainer.py +++ b/functions/src/auto_trainer/test_auto_trainer.py @@ -28,7 +28,7 @@ MODELS = [ ("sklearn.linear_model.LinearRegression", "regression"), ("sklearn.ensemble.RandomForestClassifier", "classification"), - ("xgboost.XGBRegressor", "regression"), + ("lightgbm.LGBMRegressor", "regression"), ] REQUIRED_ENV_VARS = [ diff --git a/functions/src/sklearn_classifier/function.yaml b/functions/src/sklearn_classifier/function.yaml index 208497ecc..205df697d 100644 --- a/functions/src/sklearn_classifier/function.yaml +++ b/functions/src/sklearn_classifier/function.yaml @@ -1,98 +1,10 @@ -kind: job spec: - default_handler: train_model - command: '' image: mlrun/mlrun + description: train any classifier using scikit-learn's API + default_handler: train_model entry_points: - get_sample: - has_kwargs: false - has_varargs: false - lineno: 33 - parameters: - - name: dataset - type: DataItem - doc: DataItem containing the dataset - - name: sample - type: int - doc: Number of samples to take. If -1, use all. If < -1, take random sample. - - name: label_column - type: str - doc: Name of the label column - outputs: - - type: Tuple[pd.DataFrame, pd.Series, list] - name: get_sample - doc: Get a sample of the dataset with labels separated. - get_splits: - has_kwargs: false - has_varargs: false - lineno: 56 - parameters: - - name: features - type: DataFrame - doc: Feature DataFrame - - name: labels - type: Series - doc: Labels Series - - name: num_splits - type: int - doc: Number of splits (3 for train/val/test) - - name: test_size - type: float - doc: Proportion for test set - - name: val_size - type: float - doc: Proportion of remaining data for validation - - name: random_state - type: int - doc: Random seed - default: 1 - outputs: - - type: List[Tuple[pd.DataFrame, pd.Series]] - name: get_splits - doc: Split data into train, validation, and test sets. - gen_sklearn_model: - has_kwargs: false - has_varargs: false - lineno: 86 - parameters: - - name: model_pkg_class - type: str - doc: Full class path (e.g., "sklearn.ensemble.RandomForestClassifier") - - name: parameters - type: list - doc: List of (key, value) parameter tuples - outputs: - - type: dict - name: gen_sklearn_model - doc: Generate sklearn model configuration from class name and parameters. - eval_model_v2: - has_kwargs: false - has_varargs: false - lineno: 117 - parameters: - - name: context - type: MLClientCtx - doc: MLRun context - - name: xvalid - type: DataFrame - doc: Validation features - - name: yvalid - type: Series - doc: Validation labels - - name: model - doc: Trained sklearn model - - name: plots_artifact_path - type: str - doc: Path for plots (not used in this simplified version) - default: null - outputs: - - type: dict - name: eval_model_v2 - doc: Evaluate a sklearn classifier model. train_model: - has_kwargs: false has_varargs: false - lineno: 148 parameters: - name: context type: MLClientCtx @@ -116,7 +28,7 @@ spec: type: int doc: Selects the first n rows, or select a sample starting from the first. If negative <-1, select a random sample - default: + default: - name: test_size type: float doc: (0.05) test set size @@ -152,8 +64,6 @@ spec: type: int doc: (1) sklearn rng seed default: 1 - outputs: - - type: None name: train_model doc: 'train a classifier @@ -166,16 +76,21 @@ spec: scalar "results", a "plots" keys with a list of PlotArtifacts, and and "tables" key containing a returned list of TableArtifacts.' + outputs: + - type: None + lineno: 32 + has_kwargs: false + disable_auto_mount: false build: - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgp3YXJuaW5ncy5zaW1wbGVmaWx0ZXIoYWN0aW9uPSJpZ25vcmUiLCBjYXRlZ29yeT1GdXR1cmVXYXJuaW5nKQoKCmZyb20gY2xvdWRwaWNrbGUgaW1wb3J0IGR1bXBzCmltcG9ydCBwYW5kYXMgYXMgcGQKaW1wb3J0IG51bXB5IGFzIG5wCmZyb20gdHlwaW5nIGltcG9ydCBMaXN0LCBUdXBsZQpmcm9tIHNrbGVhcm4ubW9kZWxfc2VsZWN0aW9uIGltcG9ydCB0cmFpbl90ZXN0X3NwbGl0CmZyb20gc2tsZWFybi5tZXRyaWNzIGltcG9ydCBhY2N1cmFjeV9zY29yZSwgcHJlY2lzaW9uX3Njb3JlLCByZWNhbGxfc2NvcmUsIGYxX3Njb3JlCmZyb20gbWxydW4uZXhlY3V0aW9uIGltcG9ydCBNTENsaWVudEN0eApmcm9tIG1scnVuLmRhdGFzdG9yZSBpbXBvcnQgRGF0YUl0ZW0KZnJvbSBtbHJ1bi51dGlscy5oZWxwZXJzIGltcG9ydCBjcmVhdGVfY2xhc3MKCgpkZWYgZ2V0X3NhbXBsZShkYXRhc2V0OiBEYXRhSXRlbSwgc2FtcGxlOiBpbnQsIGxhYmVsX2NvbHVtbjogc3RyKSAtPiBUdXBsZVtwZC5EYXRhRnJhbWUsIHBkLlNlcmllcywgbGlzdF06CiAgICAiIiJHZXQgYSBzYW1wbGUgb2YgdGhlIGRhdGFzZXQgd2l0aCBsYWJlbHMgc2VwYXJhdGVkLgoKICAgIDpwYXJhbSBkYXRhc2V0OiBEYXRhSXRlbSBjb250YWluaW5nIHRoZSBkYXRhc2V0CiAgICA6cGFyYW0gc2FtcGxlOiBOdW1iZXIgb2Ygc2FtcGxlcyB0byB0YWtlLiBJZiAtMSwgdXNlIGFsbC4gSWYgPCAtMSwgdGFrZSByYW5kb20gc2FtcGxlLgogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogTmFtZSBvZiB0aGUgbGFiZWwgY29sdW1uCiAgICAiIiIKICAgIGRmID0gZGF0YXNldC5hc19kZigpCgogICAgaWYgc2FtcGxlID09IC0xOgogICAgICAgIHNhbXBsZWRfZGYgPSBkZgogICAgZWxpZiBzYW1wbGUgPCAtMToKICAgICAgICBzYW1wbGVkX2RmID0gZGYuc2FtcGxlKG49YWJzKHNhbXBsZSksIHJhbmRvbV9zdGF0ZT0xKQogICAgZWxzZToKICAgICAgICBzYW1wbGVkX2RmID0gZGYuaGVhZChzYW1wbGUpCgogICAgbGFiZWxzID0gc2FtcGxlZF9kZltsYWJlbF9jb2x1bW5dCiAgICBmZWF0dXJlcyA9IHNhbXBsZWRfZGYuZHJvcChsYWJlbF9jb2x1bW4sIGF4aXM9MSkKICAgIGhlYWRlciA9IGxpc3QoZmVhdHVyZXMuY29sdW1ucykKCiAgICByZXR1cm4gZmVhdHVyZXMsIGxhYmVscywgaGVhZGVyCgoKZGVmIGdldF9zcGxpdHMoCiAgICBmZWF0dXJlczogcGQuRGF0YUZyYW1lLAogICAgbGFiZWxzOiBwZC5TZXJpZXMsCiAgICBudW1fc3BsaXRzOiBpbnQsCiAgICB0ZXN0X3NpemU6IGZsb2F0LAogICAgdmFsX3NpemU6IGZsb2F0LAogICAgcmFuZG9tX3N0YXRlOiBpbnQgPSAxCikgLT4gTGlzdFtUdXBsZVtwZC5EYXRhRnJhbWUsIHBkLlNlcmllc11dOgogICAgIiIiU3BsaXQgZGF0YSBpbnRvIHRyYWluLCB2YWxpZGF0aW9uLCBhbmQgdGVzdCBzZXRzLgoKICAgIDpwYXJhbSBmZWF0dXJlczogRmVhdHVyZSBEYXRhRnJhbWUKICAgIDpwYXJhbSBsYWJlbHM6IExhYmVscyBTZXJpZXMKICAgIDpwYXJhbSBudW1fc3BsaXRzOiBOdW1iZXIgb2Ygc3BsaXRzICgzIGZvciB0cmFpbi92YWwvdGVzdCkKICAgIDpwYXJhbSB0ZXN0X3NpemU6IFByb3BvcnRpb24gZm9yIHRlc3Qgc2V0CiAgICA6cGFyYW0gdmFsX3NpemU6IFByb3BvcnRpb24gb2YgcmVtYWluaW5nIGRhdGEgZm9yIHZhbGlkYXRpb24KICAgIDpwYXJhbSByYW5kb21fc3RhdGU6IFJhbmRvbSBzZWVkCiAgICAiIiIKICAgICMgRmlyc3Qgc3BsaXQ6IHNlcGFyYXRlIHRlc3Qgc2V0CiAgICBYX3RlbXAsIFhfdGVzdCwgeV90ZW1wLCB5X3Rlc3QgPSB0cmFpbl90ZXN0X3NwbGl0KAogICAgICAgIGZlYXR1cmVzLCBsYWJlbHMsIHRlc3Rfc2l6ZT10ZXN0X3NpemUsIHJhbmRvbV9zdGF0ZT1yYW5kb21fc3RhdGUKICAgICkKCiAgICAjIFNlY29uZCBzcGxpdDogc2VwYXJhdGUgdHJhaW4gYW5kIHZhbGlkYXRpb24gZnJvbSByZW1haW5pbmcgZGF0YQogICAgWF90cmFpbiwgWF92YWwsIHlfdHJhaW4sIHlfdmFsID0gdHJhaW5fdGVzdF9zcGxpdCgKICAgICAgICBYX3RlbXAsIHlfdGVtcCwgdGVzdF9zaXplPXZhbF9zaXplLCByYW5kb21fc3RhdGU9cmFuZG9tX3N0YXRlCiAgICApCgogICAgcmV0dXJuIFsoWF90cmFpbiwgeV90cmFpbiksIChYX3ZhbCwgeV92YWwpLCAoWF90ZXN0LCB5X3Rlc3QpXQoKCmRlZiBnZW5fc2tsZWFybl9tb2RlbChtb2RlbF9wa2dfY2xhc3M6IHN0ciwgcGFyYW1ldGVyczogbGlzdCkgLT4gZGljdDoKICAgICIiIkdlbmVyYXRlIHNrbGVhcm4gbW9kZWwgY29uZmlndXJhdGlvbiBmcm9tIGNsYXNzIG5hbWUgYW5kIHBhcmFtZXRlcnMuCgogICAgOnBhcmFtIG1vZGVsX3BrZ19jbGFzczogRnVsbCBjbGFzcyBwYXRoIChlLmcuLCAic2tsZWFybi5lbnNlbWJsZS5SYW5kb21Gb3Jlc3RDbGFzc2lmaWVyIikKICAgIDpwYXJhbSBwYXJhbWV0ZXJzOiBMaXN0IG9mIChrZXksIHZhbHVlKSBwYXJhbWV0ZXIgdHVwbGVzCiAgICAiIiIKICAgIGNvbmZpZyA9IHsKICAgICAgICAiTUVUQSI6IHsiY2xhc3MiOiBtb2RlbF9wa2dfY2xhc3N9LAogICAgICAgICJDTEFTUyI6IHt9LAogICAgICAgICJGSVQiOiB7fQogICAgfQoKICAgICMgUGFyYW1ldGVycyB0aGF0IHNob3VsZCBub3QgYmUgcGFzc2VkIHRvIHNrbGVhcm4gbW9kZWwKICAgIGV4Y2x1ZGVkX3BhcmFtcyA9IHsKICAgICAgICAnbW9kZWxfcGtnX2NsYXNzJywgJ2RhdGFzZXQnLCAnbGFiZWxfY29sdW1uJywgJ2VuY29kZV9jb2xzJywKICAgICAgICAnc2FtcGxlJywgJ3Rlc3Rfc2l6ZScsICd0cmFpbl92YWxfc3BsaXQnLCAndGVzdF9zZXRfa2V5JywKICAgICAgICAnbW9kZWxfZXZhbHVhdG9yJywgJ21vZGVsc19kZXN0JywgJ3Bsb3RzX2Rlc3QnLCAnZmlsZV9leHQnLAogICAgICAgICdtb2RlbF9wa2dfZmlsZScsICdjb250ZXh0JwogICAgfQoKICAgICMgU2VwYXJhdGUgcGFyYW1ldGVycyBpbnRvIG1vZGVsIGluaXQgcGFyYW1zIGFuZCBmaXQgcGFyYW1zCiAgICBmb3Iga2V5LCB2YWx1ZSBpbiBwYXJhbWV0ZXJzOgogICAgICAgIGlmIGtleSBpbiBbJ1gnLCAneScsICdzYW1wbGVfd2VpZ2h0J106CiAgICAgICAgICAgIGNvbmZpZ1siRklUIl1ba2V5XSA9IHZhbHVlCiAgICAgICAgZWxpZiBrZXkgbm90IGluIGV4Y2x1ZGVkX3BhcmFtczoKICAgICAgICAgICAgIyBPbmx5IGFkZCBwYXJhbWV0ZXJzIHRoYXQgYXJlIG5vdCBmdW5jdGlvbi1zcGVjaWZpYwogICAgICAgICAgICBjb25maWdbIkNMQVNTIl1ba2V5XSA9IHZhbHVlCgogICAgcmV0dXJuIGNvbmZpZwoKCmRlZiBldmFsX21vZGVsX3YyKAogICAgY29udGV4dDogTUxDbGllbnRDdHgsCiAgICB4dmFsaWQ6IHBkLkRhdGFGcmFtZSwKICAgIHl2YWxpZDogcGQuU2VyaWVzLAogICAgbW9kZWwsCiAgICBwbG90c19hcnRpZmFjdF9wYXRoOiBzdHIgPSBOb25lCikgLT4gZGljdDoKICAgICIiIkV2YWx1YXRlIGEgc2tsZWFybiBjbGFzc2lmaWVyIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiBNTFJ1biBjb250ZXh0CiAgICA6cGFyYW0geHZhbGlkOiBWYWxpZGF0aW9uIGZlYXR1cmVzCiAgICA6cGFyYW0geXZhbGlkOiBWYWxpZGF0aW9uIGxhYmVscwogICAgOnBhcmFtIG1vZGVsOiBUcmFpbmVkIHNrbGVhcm4gbW9kZWwKICAgIDpwYXJhbSBwbG90c19hcnRpZmFjdF9wYXRoOiBQYXRoIGZvciBwbG90cyAobm90IHVzZWQgaW4gdGhpcyBzaW1wbGlmaWVkIHZlcnNpb24pCiAgICAiIiIKICAgIHlfcHJlZCA9IG1vZGVsLnByZWRpY3QoeHZhbGlkKQoKICAgIG1ldHJpY3MgPSB7CiAgICAgICAgImFjY3VyYWN5IjogYWNjdXJhY3lfc2NvcmUoeXZhbGlkLCB5X3ByZWQpLAogICAgICAgICJwcmVjaXNpb24iOiBwcmVjaXNpb25fc2NvcmUoeXZhbGlkLCB5X3ByZWQsIGF2ZXJhZ2U9J3dlaWdodGVkJywgemVyb19kaXZpc2lvbj0wKSwKICAgICAgICAicmVjYWxsIjogcmVjYWxsX3Njb3JlKHl2YWxpZCwgeV9wcmVkLCBhdmVyYWdlPSd3ZWlnaHRlZCcsIHplcm9fZGl2aXNpb249MCksCiAgICAgICAgImYxX3Njb3JlIjogZjFfc2NvcmUoeXZhbGlkLCB5X3ByZWQsIGF2ZXJhZ2U9J3dlaWdodGVkJywgemVyb19kaXZpc2lvbj0wKQogICAgfQoKICAgICMgTG9nIG1ldHJpY3MgdG8gY29udGV4dAogICAgZm9yIGtleSwgdmFsdWUgaW4gbWV0cmljcy5pdGVtcygpOgogICAgICAgIGNvbnRleHQubG9nX3Jlc3VsdChrZXksIHZhbHVlKQoKICAgIHJldHVybiB7fQoKCmRlZiB0cmFpbl9tb2RlbCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWxfcGtnX2NsYXNzOiBzdHIsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gImxhYmVscyIsCiAgICBlbmNvZGVfY29sczogTGlzdFtzdHJdID0gW10sCiAgICBzYW1wbGU6IGludCA9IC0xLAogICAgdGVzdF9zaXplOiBmbG9hdCA9IDAuMzAsCiAgICB0cmFpbl92YWxfc3BsaXQ6IGZsb2F0ID0gMC43MCwKICAgIHRlc3Rfc2V0X2tleTogc3RyID0gInRlc3Rfc2V0IiwKICAgIG1vZGVsX2V2YWx1YXRvcj1Ob25lLAogICAgbW9kZWxzX2Rlc3Q6IHN0ciA9ICIiLAogICAgcGxvdHNfZGVzdDogc3RyID0gInBsb3RzIiwKICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICBtb2RlbF9wa2dfZmlsZTogc3RyID0gIiIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDEsCikgLT4gTm9uZToKICAgICIiInRyYWluIGEgY2xhc3NpZmllcgoKICAgIEFuIG9wdGlvbmFsIGN1dG9tIG1vZGVsIGV2YWx1YXRvciBjYW4gYmUgc3VwcGxpZWQgdGhhdCBzaG91bGQgaGF2ZSB0aGUgc2lnbmF0dXJlOgogICAgYG15X2N1c3RvbV9ldmFsdWF0b3IoY29udGV4dCwgeHZhbGlkLCB5dmFsaWQsIG1vZGVsKWAgYW5kIHJldHVybiBhIGRpY3Rpb25hcnkgb2YKICAgIHNjYWxhciAicmVzdWx0cyIsIGEgInBsb3RzIiBrZXlzIHdpdGggYSBsaXN0IG9mIFBsb3RBcnRpZmFjdHMsIGFuZAogICAgYW5kICJ0YWJsZXMiIGtleSBjb250YWluaW5nIGEgcmV0dXJuZWQgbGlzdCBvZiBUYWJsZUFydGlmYWN0cy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgIHRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxfcGtnX2NsYXNzOiAgIHRoZSBtb2RlbCB0byB0cmFpbiwgZS5nLCAic2tsZWFybi5uZXVyYWxfbmV0d29ya3MuTUxQQ2xhc3NpZmllciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9yIGpzb24gbW9kZWwgY29uZmlnCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICgiZGF0YSIpIG5hbWUgb2YgcmF3IGRhdGEgZmlsZQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICBncm91bmQtdHJ1dGggKHkpIGxhYmVscwogICAgOnBhcmFtIGVuY29kZV9jb2xzOiAgICAgICBkaWN0aW9uYXJ5IG9mIG5hbWVzIGFuZCBwcmVmaXhlcyBmb3IgY29sdW1ucyB0aGF0IGFyZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0byBob3QgYmUgZW5jb2RlZC4KICAgIDpwYXJhbSBzYW1wbGU6ICAgICAgICAgICAgU2VsZWN0cyB0aGUgZmlyc3QgbiByb3dzLCBvciBzZWxlY3QgYSBzYW1wbGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RhcnRpbmcgZnJvbSB0aGUgZmlyc3QuIElmIG5lZ2F0aXZlIDwtMSwgc2VsZWN0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGEgcmFuZG9tIHNhbXBsZQogICAgOnBhcmFtIHRlc3Rfc2l6ZTogICAgICAgICAoMC4wNSkgdGVzdCBzZXQgc2l6ZQogICAgOnBhcmFtIHRyYWluX3ZhbF9zcGxpdDogICAoMC43NSkgT25jZSB0aGUgdGVzdCBzZXQgaGFzIGJlZW4gcmVtb3ZlZCB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJhaW5pbmcgc2V0IGdldHMgdGhpcyBwcm9wb3J0aW9uLgogICAgOnBhcmFtIHRlc3Rfc2V0X2tleTogICAgICBrZXkgb2YgaGVsZCBvdXQgZGF0YSBpbiBhcnRpZmFjdCBzdG9yZQogICAgOnBhcmFtIG1vZGVsX2V2YWx1YXRvcjogICAoTm9uZSkgYSBjdXN0b20gbW9kZWwgZXZhbHVhdG9yIGNhbiBiZSBzcGVjaWZpZWQKICAgIDpwYXJhbSBtb2RlbHNfZGVzdDogICAgICAgKCIiKSBtb2RlbHMgc3ViZm9sZGVyIG9uIGFydGlmYWN0IHBhdGgKICAgIDpwYXJhbSBwbG90c19kZXN0OiAgICAgICAgcGxvdCBzdWJmb2xkZXIgb24gYXJ0aWZhY3QgcGF0aAogICAgOnBhcmFtIGZpbGVfZXh0OiAgICAgICAgICAoInBhcnF1ZXQiKSBmb3JtYXQgZm9yIHRlc3Rfc2V0X2tleSBob2xkIG91dCBkYXRhCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICgxKSBza2xlYXJuIHJuZyBzZWVkCgogICAgIiIiCiAgICBtb2RlbHNfZGVzdCA9IG1vZGVsc19kZXN0IG9yICJtb2RlbCIKCiAgICByYXcsIGxhYmVscywgaGVhZGVyID0gZ2V0X3NhbXBsZShkYXRhc2V0LCBzYW1wbGUsIGxhYmVsX2NvbHVtbikKCiAgICBpZiBlbmNvZGVfY29sczoKICAgICAgICByYXcgPSBwZC5nZXRfZHVtbWllcygKICAgICAgICAgICAgcmF3LAogICAgICAgICAgICBjb2x1bW5zPWxpc3QoZW5jb2RlX2NvbHMua2V5cygpKSwKICAgICAgICAgICAgcHJlZml4PWxpc3QoZW5jb2RlX2NvbHMudmFsdWVzKCkpLAogICAgICAgICAgICBkcm9wX2ZpcnN0PVRydWUsCiAgICAgICAgKQoKICAgICh4dHJhaW4sIHl0cmFpbiksICh4dmFsaWQsIHl2YWxpZCksICh4dGVzdCwgeXRlc3QpID0gZ2V0X3NwbGl0cygKICAgICAgICByYXcsIGxhYmVscywgMywgdGVzdF9zaXplLCAxIC0gdHJhaW5fdmFsX3NwbGl0LCByYW5kb21fc3RhdGUKICAgICkKCiAgICB0ZXN0X3NldCA9IHBkLmNvbmNhdChbeHRlc3QsIHl0ZXN0LnRvX2ZyYW1lKCldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgIHRlc3Rfc2V0X2tleSwKICAgICAgICBkZj10ZXN0X3NldCwKICAgICAgICBmb3JtYXQ9ZmlsZV9leHQsCiAgICAgICAgaW5kZXg9RmFsc2UsCiAgICAgICAgbGFiZWxzPXsiZGF0YS10eXBlIjogImhlbGQtb3V0In0sCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1jb250ZXh0LmFydGlmYWN0X3N1YnBhdGgoImRhdGEiKSwKICAgICkKCiAgICBtb2RlbF9jb25maWcgPSBnZW5fc2tsZWFybl9tb2RlbChtb2RlbF9wa2dfY2xhc3MsIGNvbnRleHQucGFyYW1ldGVycy5pdGVtcygpKQoKICAgIG1vZGVsX2NvbmZpZ1siRklUIl0udXBkYXRlKHsiWCI6IHh0cmFpbiwgInkiOiB5dHJhaW4udmFsdWVzfSkKCiAgICBDbGFzc2lmaWVyQ2xhc3MgPSBjcmVhdGVfY2xhc3MobW9kZWxfY29uZmlnWyJNRVRBIl1bImNsYXNzIl0pCgogICAgbW9kZWwgPSBDbGFzc2lmaWVyQ2xhc3MoKiptb2RlbF9jb25maWdbIkNMQVNTIl0pCgogICAgbW9kZWwuZml0KCoqbW9kZWxfY29uZmlnWyJGSVQiXSkKCiAgICBhcnRpZmFjdF9wYXRoID0gY29udGV4dC5hcnRpZmFjdF9zdWJwYXRoKG1vZGVsc19kZXN0KQogICAgcGxvdHNfcGF0aCA9IGNvbnRleHQuYXJ0aWZhY3Rfc3VicGF0aChtb2RlbHNfZGVzdCwgcGxvdHNfZGVzdCkKICAgIGlmIG1vZGVsX2V2YWx1YXRvcjoKICAgICAgICBldmFsX21ldHJpY3MgPSBtb2RlbF9ldmFsdWF0b3IoCiAgICAgICAgICAgIGNvbnRleHQsIHh2YWxpZCwgeXZhbGlkLCBtb2RlbCwgcGxvdHNfYXJ0aWZhY3RfcGF0aD1wbG90c19wYXRoCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICBldmFsX21ldHJpY3MgPSBldmFsX21vZGVsX3YyKAogICAgICAgICAgICBjb250ZXh0LCB4dmFsaWQsIHl2YWxpZCwgbW9kZWwsIHBsb3RzX2FydGlmYWN0X3BhdGg9cGxvdHNfcGF0aAogICAgICAgICkKCiAgICBrd2FyZ3MgPSB7InRyYWluaW5nX3NldCI6IHRlc3Rfc2V0LCAibGFiZWxfY29sdW1uIjogbGFiZWxfY29sdW1ufQogICAgc3BsaXQgPSBtb2RlbF9wa2dfY2xhc3MucnNwbGl0KCIuIiwgMSkKICAgIGlmIHNwbGl0IGFuZCBsZW4oc3BsaXQpID09IDI6CiAgICAgICAga3dhcmdzWyJhbGdvcml0aG0iXSA9IHNwbGl0WzFdCgogICAgaWYgZGF0YXNldC5tZXRhIGFuZCBkYXRhc2V0Lm1ldGEua2luZCA9PSAiRmVhdHVyZVZlY3RvciI6CiAgICAgICAga3dhcmdzWyJmZWF0dXJlX3ZlY3RvciJdID0gZGF0YXNldC5tZXRhLnVyaQoKICAgIGNvbnRleHQuc2V0X2xhYmVsKCJjbGFzcyIsIG1vZGVsX3BrZ19jbGFzcykKICAgIGNvbnRleHQubG9nX21vZGVsKAogICAgICAgICJtb2RlbCIsCiAgICAgICAgYm9keT1kdW1wcyhtb2RlbCksCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1hcnRpZmFjdF9wYXRoLAogICAgICAgIGV4dHJhX2RhdGE9ZXZhbF9tZXRyaWNzLAogICAgICAgIG1vZGVsX2ZpbGU9Im1vZGVsLnBrbCIsCiAgICAgICAgbWV0cmljcz1jb250ZXh0LnJlc3VsdHMsCiAgICAgICAgbGFiZWxzPXsiY2xhc3MiOiBtb2RlbF9wa2dfY2xhc3N9LAogICAgICAgIGZyYW1ld29yaz0ic2tsZWFybiIsCiAgICAgICAgKiprd2FyZ3MKICAgICkK + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKIyBHZW5lcmF0ZWQgYnkgbnVjbGlvLmV4cG9ydC5OdWNsaW9FeHBvcnRlcgoKaW1wb3J0IHdhcm5pbmdzCgp3YXJuaW5ncy5zaW1wbGVmaWx0ZXIoYWN0aW9uPSJpZ25vcmUiLCBjYXRlZ29yeT1GdXR1cmVXYXJuaW5nKQoKCmZyb20gY2xvdWRwaWNrbGUgaW1wb3J0IGR1bXBzCmltcG9ydCBwYW5kYXMgYXMgcGQKZnJvbSB0eXBpbmcgaW1wb3J0IExpc3QKZnJvbSBtbHJ1bi5leGVjdXRpb24gaW1wb3J0IE1MQ2xpZW50Q3R4CmZyb20gbWxydW4uZGF0YXN0b3JlIGltcG9ydCBEYXRhSXRlbQpmcm9tIG1scnVuLm1sdXRpbHMuZGF0YSBpbXBvcnQgZ2V0X3NhbXBsZSwgZ2V0X3NwbGl0cwpmcm9tIG1scnVuLm1sdXRpbHMubW9kZWxzIGltcG9ydCBnZW5fc2tsZWFybl9tb2RlbCwgZXZhbF9tb2RlbF92Mgpmcm9tIG1scnVuLnV0aWxzLmhlbHBlcnMgaW1wb3J0IGNyZWF0ZV9jbGFzcwoKCmRlZiB0cmFpbl9tb2RlbCgKICAgIGNvbnRleHQ6IE1MQ2xpZW50Q3R4LAogICAgbW9kZWxfcGtnX2NsYXNzOiBzdHIsCiAgICBkYXRhc2V0OiBEYXRhSXRlbSwKICAgIGxhYmVsX2NvbHVtbjogc3RyID0gImxhYmVscyIsCiAgICBlbmNvZGVfY29sczogTGlzdFtzdHJdID0gW10sCiAgICBzYW1wbGU6IGludCA9IC0xLAogICAgdGVzdF9zaXplOiBmbG9hdCA9IDAuMzAsCiAgICB0cmFpbl92YWxfc3BsaXQ6IGZsb2F0ID0gMC43MCwKICAgIHRlc3Rfc2V0X2tleTogc3RyID0gInRlc3Rfc2V0IiwKICAgIG1vZGVsX2V2YWx1YXRvcj1Ob25lLAogICAgbW9kZWxzX2Rlc3Q6IHN0ciA9ICIiLAogICAgcGxvdHNfZGVzdDogc3RyID0gInBsb3RzIiwKICAgIGZpbGVfZXh0OiBzdHIgPSAicGFycXVldCIsCiAgICBtb2RlbF9wa2dfZmlsZTogc3RyID0gIiIsCiAgICByYW5kb21fc3RhdGU6IGludCA9IDEsCikgLT4gTm9uZToKICAgICIiInRyYWluIGEgY2xhc3NpZmllcgoKICAgIEFuIG9wdGlvbmFsIGN1dG9tIG1vZGVsIGV2YWx1YXRvciBjYW4gYmUgc3VwcGxpZWQgdGhhdCBzaG91bGQgaGF2ZSB0aGUgc2lnbmF0dXJlOgogICAgYG15X2N1c3RvbV9ldmFsdWF0b3IoY29udGV4dCwgeHZhbGlkLCB5dmFsaWQsIG1vZGVsKWAgYW5kIHJldHVybiBhIGRpY3Rpb25hcnkgb2YKICAgIHNjYWxhciAicmVzdWx0cyIsIGEgInBsb3RzIiBrZXlzIHdpdGggYSBsaXN0IG9mIFBsb3RBcnRpZmFjdHMsIGFuZAogICAgYW5kICJ0YWJsZXMiIGtleSBjb250YWluaW5nIGEgcmV0dXJuZWQgbGlzdCBvZiBUYWJsZUFydGlmYWN0cy4KCiAgICA6cGFyYW0gY29udGV4dDogICAgICAgICAgIHRoZSBmdW5jdGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxfcGtnX2NsYXNzOiAgIHRoZSBtb2RlbCB0byB0cmFpbiwgZS5nLCAic2tsZWFybi5uZXVyYWxfbmV0d29ya3MuTUxQQ2xhc3NpZmllciIsCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG9yIGpzb24gbW9kZWwgY29uZmlnCiAgICA6cGFyYW0gZGF0YXNldDogICAgICAgICAgICgiZGF0YSIpIG5hbWUgb2YgcmF3IGRhdGEgZmlsZQogICAgOnBhcmFtIGxhYmVsX2NvbHVtbjogICAgICBncm91bmQtdHJ1dGggKHkpIGxhYmVscwogICAgOnBhcmFtIGVuY29kZV9jb2xzOiAgICAgICBkaWN0aW9uYXJ5IG9mIG5hbWVzIGFuZCBwcmVmaXhlcyBmb3IgY29sdW1ucyB0aGF0IGFyZQogICAgICAgICAgICAgICAgICAgICAgICAgICAgICB0byBob3QgYmUgZW5jb2RlZC4KICAgIDpwYXJhbSBzYW1wbGU6ICAgICAgICAgICAgU2VsZWN0cyB0aGUgZmlyc3QgbiByb3dzLCBvciBzZWxlY3QgYSBzYW1wbGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgc3RhcnRpbmcgZnJvbSB0aGUgZmlyc3QuIElmIG5lZ2F0aXZlIDwtMSwgc2VsZWN0CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGEgcmFuZG9tIHNhbXBsZQogICAgOnBhcmFtIHRlc3Rfc2l6ZTogICAgICAgICAoMC4wNSkgdGVzdCBzZXQgc2l6ZQogICAgOnBhcmFtIHRyYWluX3ZhbF9zcGxpdDogICAoMC43NSkgT25jZSB0aGUgdGVzdCBzZXQgaGFzIGJlZW4gcmVtb3ZlZCB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdHJhaW5pbmcgc2V0IGdldHMgdGhpcyBwcm9wb3J0aW9uLgogICAgOnBhcmFtIHRlc3Rfc2V0X2tleTogICAgICBrZXkgb2YgaGVsZCBvdXQgZGF0YSBpbiBhcnRpZmFjdCBzdG9yZQogICAgOnBhcmFtIG1vZGVsX2V2YWx1YXRvcjogICAoTm9uZSkgYSBjdXN0b20gbW9kZWwgZXZhbHVhdG9yIGNhbiBiZSBzcGVjaWZpZWQKICAgIDpwYXJhbSBtb2RlbHNfZGVzdDogICAgICAgKCIiKSBtb2RlbHMgc3ViZm9sZGVyIG9uIGFydGlmYWN0IHBhdGgKICAgIDpwYXJhbSBwbG90c19kZXN0OiAgICAgICAgcGxvdCBzdWJmb2xkZXIgb24gYXJ0aWZhY3QgcGF0aAogICAgOnBhcmFtIGZpbGVfZXh0OiAgICAgICAgICAoInBhcnF1ZXQiKSBmb3JtYXQgZm9yIHRlc3Rfc2V0X2tleSBob2xkIG91dCBkYXRhCiAgICA6cGFyYW0gcmFuZG9tX3N0YXRlOiAgICAgICgxKSBza2xlYXJuIHJuZyBzZWVkCgogICAgIiIiCiAgICBtb2RlbHNfZGVzdCA9IG1vZGVsc19kZXN0IG9yICJtb2RlbCIKCiAgICByYXcsIGxhYmVscywgaGVhZGVyID0gZ2V0X3NhbXBsZShkYXRhc2V0LCBzYW1wbGUsIGxhYmVsX2NvbHVtbikKCiAgICBpZiBlbmNvZGVfY29sczoKICAgICAgICByYXcgPSBwZC5nZXRfZHVtbWllcygKICAgICAgICAgICAgcmF3LAogICAgICAgICAgICBjb2x1bW5zPWxpc3QoZW5jb2RlX2NvbHMua2V5cygpKSwKICAgICAgICAgICAgcHJlZml4PWxpc3QoZW5jb2RlX2NvbHMudmFsdWVzKCkpLAogICAgICAgICAgICBkcm9wX2ZpcnN0PVRydWUsCiAgICAgICAgKQoKICAgICh4dHJhaW4sIHl0cmFpbiksICh4dmFsaWQsIHl2YWxpZCksICh4dGVzdCwgeXRlc3QpID0gZ2V0X3NwbGl0cygKICAgICAgICByYXcsIGxhYmVscywgMywgdGVzdF9zaXplLCAxIC0gdHJhaW5fdmFsX3NwbGl0LCByYW5kb21fc3RhdGUKICAgICkKCiAgICB0ZXN0X3NldCA9IHBkLmNvbmNhdChbeHRlc3QsIHl0ZXN0LnRvX2ZyYW1lKCldLCBheGlzPTEpCiAgICBjb250ZXh0LmxvZ19kYXRhc2V0KAogICAgICAgIHRlc3Rfc2V0X2tleSwKICAgICAgICBkZj10ZXN0X3NldCwKICAgICAgICBmb3JtYXQ9ZmlsZV9leHQsCiAgICAgICAgaW5kZXg9RmFsc2UsCiAgICAgICAgbGFiZWxzPXsiZGF0YS10eXBlIjogImhlbGQtb3V0In0sCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1jb250ZXh0LmFydGlmYWN0X3N1YnBhdGgoImRhdGEiKSwKICAgICkKCiAgICBtb2RlbF9jb25maWcgPSBnZW5fc2tsZWFybl9tb2RlbChtb2RlbF9wa2dfY2xhc3MsIGNvbnRleHQucGFyYW1ldGVycy5pdGVtcygpKQoKICAgIG1vZGVsX2NvbmZpZ1siRklUIl0udXBkYXRlKHsiWCI6IHh0cmFpbiwgInkiOiB5dHJhaW4udmFsdWVzfSkKCiAgICBDbGFzc2lmaWVyQ2xhc3MgPSBjcmVhdGVfY2xhc3MobW9kZWxfY29uZmlnWyJNRVRBIl1bImNsYXNzIl0pCgogICAgbW9kZWwgPSBDbGFzc2lmaWVyQ2xhc3MoKiptb2RlbF9jb25maWdbIkNMQVNTIl0pCgogICAgbW9kZWwuZml0KCoqbW9kZWxfY29uZmlnWyJGSVQiXSkKCiAgICBhcnRpZmFjdF9wYXRoID0gY29udGV4dC5hcnRpZmFjdF9zdWJwYXRoKG1vZGVsc19kZXN0KQogICAgcGxvdHNfcGF0aCA9IGNvbnRleHQuYXJ0aWZhY3Rfc3VicGF0aChtb2RlbHNfZGVzdCwgcGxvdHNfZGVzdCkKICAgIGlmIG1vZGVsX2V2YWx1YXRvcjoKICAgICAgICBldmFsX21ldHJpY3MgPSBtb2RlbF9ldmFsdWF0b3IoCiAgICAgICAgICAgIGNvbnRleHQsIHh2YWxpZCwgeXZhbGlkLCBtb2RlbCwgcGxvdHNfYXJ0aWZhY3RfcGF0aD1wbG90c19wYXRoCiAgICAgICAgKQogICAgZWxzZToKICAgICAgICBldmFsX21ldHJpY3MgPSBldmFsX21vZGVsX3YyKAogICAgICAgICAgICBjb250ZXh0LCB4dmFsaWQsIHl2YWxpZCwgbW9kZWwsIHBsb3RzX2FydGlmYWN0X3BhdGg9cGxvdHNfcGF0aAogICAgICAgICkKCiAgICBrd2FyZ3MgPSB7InRyYWluaW5nX3NldCI6IHRlc3Rfc2V0LCAibGFiZWxfY29sdW1uIjogbGFiZWxfY29sdW1ufQogICAgc3BsaXQgPSBtb2RlbF9wa2dfY2xhc3MucnNwbGl0KCIuIiwgMSkKICAgIGlmIHNwbGl0IGFuZCBsZW4oc3BsaXQpID09IDI6CiAgICAgICAga3dhcmdzWyJhbGdvcml0aG0iXSA9IHNwbGl0WzFdCgogICAgaWYgZGF0YXNldC5tZXRhIGFuZCBkYXRhc2V0Lm1ldGEua2luZCA9PSAiRmVhdHVyZVZlY3RvciI6CiAgICAgICAga3dhcmdzWyJmZWF0dXJlX3ZlY3RvciJdID0gZGF0YXNldC5tZXRhLnVyaQoKICAgIGNvbnRleHQuc2V0X2xhYmVsKCJjbGFzcyIsIG1vZGVsX3BrZ19jbGFzcykKICAgIGNvbnRleHQubG9nX21vZGVsKAogICAgICAgICJtb2RlbCIsCiAgICAgICAgYm9keT1kdW1wcyhtb2RlbCksCiAgICAgICAgYXJ0aWZhY3RfcGF0aD1hcnRpZmFjdF9wYXRoLAogICAgICAgIGV4dHJhX2RhdGE9ZXZhbF9tZXRyaWNzLAogICAgICAgIG1vZGVsX2ZpbGU9Im1vZGVsLnBrbCIsCiAgICAgICAgbWV0cmljcz1jb250ZXh0LnJlc3VsdHMsCiAgICAgICAgbGFiZWxzPXsiY2xhc3MiOiBtb2RlbF9wa2dfY2xhc3N9LAogICAgICAgIGZyYW1ld29yaz0ic2tsZWFybiIsCiAgICAgICAgKiprd2FyZ3MKICAgICkK origin_filename: '' code_origin: '' - disable_auto_mount: false - description: train any classifier using scikit-learn's API + command: '' metadata: + tag: '' name: sklearn-classifier categories: - machine-learning - model-training - tag: '' verbose: false +kind: job diff --git a/functions/src/sklearn_classifier/item.yaml b/functions/src/sklearn_classifier/item.yaml index 4fa374938..b9726fb79 100644 --- a/functions/src/sklearn_classifier/item.yaml +++ b/functions/src/sklearn_classifier/item.yaml @@ -13,7 +13,7 @@ labels: framework: sklearn maintainers: [] marketplaceType: '' -mlrunVersion: 1.10.0 +mlrunVersion: 1.7.0 name: sklearn-classifier platformVersion: 3.5.3 spec: @@ -23,5 +23,5 @@ spec: kind: job requirements: [] url: '' -version: 1.3.0 +version: 1.2.0 test_valid: false diff --git a/functions/src/sklearn_classifier/requirements.txt b/functions/src/sklearn_classifier/requirements.txt index 113d4a02a..4d9e097f9 100644 --- a/functions/src/sklearn_classifier/requirements.txt +++ b/functions/src/sklearn_classifier/requirements.txt @@ -1,5 +1,5 @@ pandas -scikit-learn~=1.5.2 +scikit-learn==1.0.2 matplotlib seaborn scikit-plot diff --git a/functions/src/sklearn_classifier/sklearn_classifier.py b/functions/src/sklearn_classifier/sklearn_classifier.py index 2fc60a102..1a73d4045 100644 --- a/functions/src/sklearn_classifier/sklearn_classifier.py +++ b/functions/src/sklearn_classifier/sklearn_classifier.py @@ -21,129 +21,14 @@ from cloudpickle import dumps import pandas as pd -from typing import List, Tuple -from sklearn.model_selection import train_test_split -from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score +from typing import List from mlrun.execution import MLClientCtx from mlrun.datastore import DataItem +from mlrun.mlutils.data import get_sample, get_splits +from mlrun.mlutils.models import gen_sklearn_model, eval_model_v2 from mlrun.utils.helpers import create_class -def get_sample(dataset: DataItem, sample: int, label_column: str) -> Tuple[pd.DataFrame, pd.Series, list]: - """Get a sample of the dataset with labels separated. - - :param dataset: DataItem containing the dataset - :param sample: Number of samples to take. If -1, use all. If < -1, take random sample. - :param label_column: Name of the label column - """ - df = dataset.as_df() - - if sample == -1: - sampled_df = df - elif sample < -1: - sampled_df = df.sample(n=abs(sample), random_state=1) - else: - sampled_df = df.head(sample) - - labels = sampled_df[label_column] - features = sampled_df.drop(label_column, axis=1) - header = list(features.columns) - - return features, labels, header - - -def get_splits( - features: pd.DataFrame, - labels: pd.Series, - num_splits: int, - test_size: float, - val_size: float, - random_state: int = 1 -) -> List[Tuple[pd.DataFrame, pd.Series]]: - """Split data into train, validation, and test sets. - - :param features: Feature DataFrame - :param labels: Labels Series - :param num_splits: Number of splits (3 for train/val/test) - :param test_size: Proportion for test set - :param val_size: Proportion of remaining data for validation - :param random_state: Random seed - """ - # First split: separate test set - X_temp, X_test, y_temp, y_test = train_test_split( - features, labels, test_size=test_size, random_state=random_state - ) - - # Second split: separate train and validation from remaining data - X_train, X_val, y_train, y_val = train_test_split( - X_temp, y_temp, test_size=val_size, random_state=random_state - ) - - return [(X_train, y_train), (X_val, y_val), (X_test, y_test)] - - -def gen_sklearn_model(model_pkg_class: str, parameters: list) -> dict: - """Generate sklearn model configuration from class name and parameters. - - :param model_pkg_class: Full class path (e.g., "sklearn.ensemble.RandomForestClassifier") - :param parameters: List of (key, value) parameter tuples - """ - config = { - "META": {"class": model_pkg_class}, - "CLASS": {}, - "FIT": {} - } - - # Parameters that should not be passed to sklearn model - excluded_params = { - 'model_pkg_class', 'dataset', 'label_column', 'encode_cols', - 'sample', 'test_size', 'train_val_split', 'test_set_key', - 'model_evaluator', 'models_dest', 'plots_dest', 'file_ext', - 'model_pkg_file', 'context' - } - - # Separate parameters into model init params and fit params - for key, value in parameters: - if key in ['X', 'y', 'sample_weight']: - config["FIT"][key] = value - elif key not in excluded_params: - # Only add parameters that are not function-specific - config["CLASS"][key] = value - - return config - - -def eval_model_v2( - context: MLClientCtx, - xvalid: pd.DataFrame, - yvalid: pd.Series, - model, - plots_artifact_path: str = None -) -> dict: - """Evaluate a sklearn classifier model. - - :param context: MLRun context - :param xvalid: Validation features - :param yvalid: Validation labels - :param model: Trained sklearn model - :param plots_artifact_path: Path for plots (not used in this simplified version) - """ - y_pred = model.predict(xvalid) - - metrics = { - "accuracy": accuracy_score(yvalid, y_pred), - "precision": precision_score(yvalid, y_pred, average='weighted', zero_division=0), - "recall": recall_score(yvalid, y_pred, average='weighted', zero_division=0), - "f1_score": f1_score(yvalid, y_pred, average='weighted', zero_division=0) - } - - # Log metrics to context - for key, value in metrics.items(): - context.log_result(key, value) - - return {} - - def train_model( context: MLClientCtx, model_pkg_class: str, diff --git a/functions/src/sklearn_classifier/test_sklearn_classifier.py b/functions/src/sklearn_classifier/test_sklearn_classifier.py index 78afd623b..5c29e85b3 100644 --- a/functions/src/sklearn_classifier/test_sklearn_classifier.py +++ b/functions/src/sklearn_classifier/test_sklearn_classifier.py @@ -38,29 +38,19 @@ def test_import_sklearn_classifier(): params = {"model_pkg_class": "sklearn.ensemble.RandomForestClassifier", "label_column": "labels"} - # In local mode, artifacts are in function-name/iteration subdirectory - dataset_path = "./artifacts/gen-class-data-gen-class-data/0/classifier-data.csv" - assert os.path.exists(dataset_path), f"Dataset not found at {dataset_path}" - train_run = fn.run(params=params, - inputs={"dataset": dataset_path}, + inputs={"dataset": acquire_run.status.artifacts[0]['spec']['target_path']}, local=True, - artifact_path="./artifacts") - - # Check that the run completed successfully - assert train_run.status.state == "completed", f"Run failed with state: {train_run.status.state}" - - # In local mode, check if model metrics were logged - assert "accuracy" in train_run.status.results or len(train_run.status.results) > 0, \ - "No metrics were logged" + artifact_path="./") - # In local mode, the model is saved to artifacts/model/function-name/iteration/model/model.pkl - model_path = "./artifacts/model/sklearn-classifier-train-model/0/model/model.pkl" - assert os.path.exists(model_path), f'Could not find model file at {model_path}' + for artifact in train_run.status.artifacts: + if artifact['kind'] == 'model': + assert os.path.exists(artifact['spec']['target_path']), 'Could not find model dir' + break - # Load the model and verify it can make predictions - model = pickle.load(open(model_path, 'rb')) - df = pd.read_csv(dataset_path) + assert os.path.exists(train_run.status.artifacts[0]['spec']['target_path']) + model = pickle.load(open(artifact['spec']['target_path'] + artifact['spec']['model_file'], 'rb')) + df = pd.read_csv(acquire_run.status.artifacts[0]['spec']['target_path']) x = df.drop(['labels'], axis=1).iloc[0:1] y_true = df['labels'][0] y_pred = model.predict_proba(x).argmax() From 2078e86f530dd0b6338da8afa2b1aeb90f2cba02 Mon Sep 17 00:00:00 2001 From: tomerbv Date: Thu, 12 Feb 2026 11:06:09 +0200 Subject: [PATCH 15/15] added xgboost.XGBRegressor, xgboost.XGBClassifier and lightgbm.LGBMClassifier models to test --- functions/src/auto_trainer/requirements.txt | 1 + functions/src/auto_trainer/test_auto_trainer.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/functions/src/auto_trainer/requirements.txt b/functions/src/auto_trainer/requirements.txt index 262f5e9f2..b23f9b9dd 100644 --- a/functions/src/auto_trainer/requirements.txt +++ b/functions/src/auto_trainer/requirements.txt @@ -1,4 +1,5 @@ pandas scikit-learn~=1.5.2 lightgbm +xgboost<2.0.0 plotly diff --git a/functions/src/auto_trainer/test_auto_trainer.py b/functions/src/auto_trainer/test_auto_trainer.py index fe5c051a4..ac95109f8 100644 --- a/functions/src/auto_trainer/test_auto_trainer.py +++ b/functions/src/auto_trainer/test_auto_trainer.py @@ -28,7 +28,10 @@ MODELS = [ ("sklearn.linear_model.LinearRegression", "regression"), ("sklearn.ensemble.RandomForestClassifier", "classification"), + ("xgboost.XGBRegressor", "regression"), + ("xgboost.XGBClassifier", "classification"), ("lightgbm.LGBMRegressor", "regression"), + ("lightgbm.LGBMClassifier", "classification") ] REQUIRED_ENV_VARS = [