From 4f07d57b94fde752bdd4da8c0614b251a57cfd3c Mon Sep 17 00:00:00 2001 From: Omer_Mimon Date: Tue, 20 Jan 2026 15:43:57 +0200 Subject: [PATCH 1/4] Update onnx_utils dependencies and improve test robustness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Upgrade PyTorch (2.6.0 → 2.8.0) and TorchVision (0.21.0 → 0.23.0) for better compatibility and performance - Update MLRun version requirement to 1.10.0 in item.yaml - Bump function version to 1.4.0 Test improvements: - Add environment variable validation (MLRUN_DBPATH, MLRUN_ARTIFACT_PATH) - Add conditional test skipping based on tf2onnx availability - Fix cleanup function to properly remove test artifacts (model.pt, model_modules_map.json, onnx_model.onnx, etc.) - Update deprecated artifact_path parameter to output_path - Add explicit project context to all MLRun function calls - Fix PyTorch test artifact path construction --- functions/src/onnx_utils/function.yaml | 74 +++++++------- functions/src/onnx_utils/item.yaml | 8 +- functions/src/onnx_utils/requirements.txt | 7 +- functions/src/onnx_utils/test_onnx_utils.py | 108 +++++++++++++++++--- 4 files changed, 135 insertions(+), 62 deletions(-) diff --git a/functions/src/onnx_utils/function.yaml b/functions/src/onnx_utils/function.yaml index 05a0f0bc2..091002cdc 100644 --- a/functions/src/onnx_utils/function.yaml +++ b/functions/src/onnx_utils/function.yaml @@ -1,39 +1,13 @@ -kind: job metadata: + name: onnx-utils + tag: '' categories: - utilities - deep-learning - name: onnx-utils - tag: '' -verbose: false +kind: job spec: - build: - code_origin: '' - base_image: mlrun/mlrun - origin_filename: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgQ2FsbGFibGUsIERpY3QsIExpc3QsIFR1cGxlCgppbXBvcnQgbWxydW4KCgpjbGFzcyBfVG9PTk5YQ29udmVyc2lvbnM6CiAgICAiIiIKICAgIEFuIE9OTlggY29udmVyc2lvbiBmdW5jdGlvbnMgbGlicmFyeSBjbGFzcy4KICAgICIiIgoKICAgIEBzdGF0aWNtZXRob2QKICAgIGRlZiB0Zl9rZXJhc190b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50XSwgc3RyXV0gPSBOb25lLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBDb252ZXJ0IGEgVEYuS2VyYXMgbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICBBbiBpbml0aWFsaXplZCBURktlcmFzTW9kZWxIYW5kbGVyIHdpdGggYSBsb2FkZWQgbW9kZWwgdG8gY29udmVydCB0byBPTk5YLgogICAgICAgIDpwYXJhbSBvbm54X21vZGVsX25hbWU6IFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbiBgbW9kZWxfbmFtZWAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICBXaGV0aGVyIG9yIG5vdCB0byBvcHRpbWl6ZSB0aGUgT05OWCBtb2RlbCB1c2luZyAnb25ueG9wdGltaXplcicgYmVmb3JlIHNhdmluZyB0aGUgbW9kZWwuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgRGVmYXVsdGVkIHRvIFRydWUuCiAgICAgICAgOnBhcmFtIGlucHV0X3NpZ25hdHVyZTogQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEgbGlzdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHdoZXJlIGVhY2ggZWxlbWVudCBpcyBhbiBpbnB1dCBsYXllciB0dXBsZS4gQW4gaW5wdXQgbGF5ZXIgdHVwbGUgaXMgYSB0dXBsZSBvZjoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBbMF0gPSBMYXllcidzIHNoYXBlLCBhIHR1cGxlIG9mIGludGVnZXJzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBOb25lLCB0aGUgaW5wdXQgc2lnbmF0dXJlIHdpbGwgYmUgdHJpZWQgdG8gYmUgcmVhZCBmcm9tIHRoZSBtb2RlbCBhcnRpZmFjdC4gRGVmYXVsdGVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG8gTm9uZS4KICAgICAgICAiIiIKICAgICAgICAjIEltcG9ydCB0aGUgZnJhbWV3b3JrIGFuZCBoYW5kbGVyOgogICAgICAgIGltcG9ydCB0ZW5zb3JmbG93IGFzIHRmCiAgICAgICAgZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLnRmX2tlcmFzIGltcG9ydCBURktlcmFzVXRpbHMKCiAgICAgICAgIyBDaGVjayB0aGUgZ2l2ZW4gJ2lucHV0X3NpZ25hdHVyZScgcGFyYW1ldGVyOgogICAgICAgIGlmIGlucHV0X3NpZ25hdHVyZSBpcyBOb25lOgogICAgICAgICAgICAjIFJlYWQgdGhlIGlucHV0cyBmcm9tIHRoZSBtb2RlbDoKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgbW9kZWxfaGFuZGxlci5yZWFkX2lucHV0c19mcm9tX21vZGVsKCkKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlcnJvcjoKICAgICAgICAgICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1blJ1bnRpbWVFcnJvcigKICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSBwcm92aWRlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXIuIFRoZSBmdW5jdGlvbiB0cmllZCByZWFkaW5nIHRoZSBpbnB1dCBsYXllcnMgIgogICAgICAgICAgICAgICAgICAgIGYiaW5mb3JtYXRpb24gYXV0b21hdGljYWxseSBidXQgZmFpbGVkIHdpdGggdGhlIGZvbGxvd2luZyBlcnJvcjoge2Vycm9yfSIKICAgICAgICAgICAgICAgICkKICAgICAgICBlbHNlOgogICAgICAgICAgICAjIFBhcnNlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXI6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IFsKICAgICAgICAgICAgICAgIHRmLlRlbnNvclNwZWMoCiAgICAgICAgICAgICAgICAgICAgc2hhcGU9c2hhcGUsCiAgICAgICAgICAgICAgICAgICAgZHR5cGU9VEZLZXJhc1V0aWxzLmNvbnZlcnRfdmFsdWVfdHlwZV90b190Zl9kdHlwZSgKICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICBdCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICkKCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgcHl0b3JjaF90b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50LCAuLi5dLCBzdHJdXSA9IE5vbmUsCiAgICAgICAgaW5wdXRfbGF5ZXJzX25hbWVzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgICAgIG91dHB1dF9sYXllcnNfbmFtZXM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICAgICAgZHluYW1pY19heGVzOiBEaWN0W3N0ciwgRGljdFtpbnQsIHN0cl1dID0gTm9uZSwKICAgICAgICBpc19iYXRjaGVkOiBib29sID0gVHJ1ZSwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCBhIFB5VG9yY2ggbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICAgICAgQW4gaW5pdGlhbGl6ZWQgUHlUb3JjaE1vZGVsSGFuZGxlciB3aXRoIGEgbG9hZGVkIG1vZGVsIHRvIGNvbnZlcnQgdG8gT05OWC4KICAgICAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiAgICAgVGhlIG5hbWUgdG8gdXNlIHRvIGxvZyB0aGUgY29udmVydGVkIE9OTlggbW9kZWwuIElmIG5vdCBnaXZlbiwgdGhlIGdpdmVuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtb2RlbF9uYW1lYCB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgICAgV2hldGhlciBvciBub3QgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsLiBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgICAgICA6cGFyYW0gaW5wdXRfc2lnbmF0dXJlOiAgICAgQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdCB3aGVyZSBlYWNoIGVsZW1lbnQgaXMgYW4gaW5wdXQgbGF5ZXIgdHVwbGUuIEFuIGlucHV0IGxheWVyIHR1cGxlIGlzIGEgdHVwbGUgb2Y6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFswXSA9IExheWVyJ3Mgc2hhcGUsIGEgdHVwbGUgb2YgaW50ZWdlcnMuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgdGhlIGlucHV0IHNpZ25hdHVyZSB3aWxsIGJlIHRyaWVkIHRvIGJlIHJlYWQgZnJvbSB0aGUgbW9kZWwgYXJ0aWZhY3QuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpbnB1dF9sYXllcnNfbmFtZXM6ICBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgaW5wdXQgbm9kZXMgb2YgdGhlIGdyYXBoIGluIG9yZGVyLiBBbGwgb2YgdGhlIG90aGVyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHBhcmFtZXRlcnMgKGlubmVyIGxheWVycykgY2FuIGJlIHNldCBhcyB3ZWxsIGJ5IHBhc3NpbmcgYWRkaXRpb25hbCBuYW1lcyBpbiB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdC4gVGhlIG9yZGVyIGlzIGJ5IHRoZSBvcmRlciBvZiB0aGUgcGFyYW1ldGVycyBpbiB0aGUgbW9kZWwuIElmIE5vbmUsIHRoZSBpbnB1dHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBpbnB1dHMuIElmIGl0cyBhbHNvIE5vbmUsIGl0IGlzIGRlZmF1bHRlZCB0bzoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImlucHV0XzAiLCAiaW5wdXRfMSIsIC4uLgogICAgICAgIDpwYXJhbSBvdXRwdXRfbGF5ZXJzX25hbWVzOiBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgb3V0cHV0IG5vZGVzIG9mIHRoZSBncmFwaCBpbiBvcmRlci4gSWYgTm9uZSwgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG91dHB1dHMgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBvdXRwdXRzLiBJZiBpdHMgYWxzbyBOb25lLCBpdCBpcyBkZWZhdWx0ZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG86ICJvdXRwdXRfMCIgKGZvciBtdWx0aXBsZSBvdXRwdXRzLCB0aGlzIHBhcmFtZXRlciBtdXN0IGJlIHByb3ZpZGVkKS4KICAgICAgICA6cGFyYW0gZHluYW1pY19heGVzOiAgICAgICAgSWYgcGFydCBvZiB0aGUgaW5wdXQgLyBvdXRwdXQgc2hhcGUgaXMgZHluYW1pYywgbGlrZSAoYmF0Y2hfc2l6ZSwgMywgMzIsIDMyKSB5b3UgY2FuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNwZWNpZnkgaXQgYnkgZ2l2aW5nIGEgZHluYW1pYyBheGlzIHRvIHRoZSBpbnB1dCAvIG91dHB1dCBsYXllciBieSBpdHMgbmFtZSBhcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb2xsb3dzOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiaW5wdXQgbGF5ZXIgbmFtZSI6IHswOiAiYmF0Y2hfc2l6ZSJ9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm91dHB1dCBsYXllciBuYW1lIjogezA6ICJiYXRjaF9zaXplIn0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgcHJvdmlkZWQsIHRoZSAnaXNfYmF0Y2hlZCcgZmxhZyB3aWxsIGJlIGlnbm9yZWQuIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpc19iYXRjaGVkOiAgICAgICAgICBXaGV0aGVyIHRvIGluY2x1ZGUgYSBiYXRjaCBzaXplIGFzIHRoZSBmaXJzdCBheGlzIGluIGV2ZXJ5IGlucHV0IGFuZCBvdXRwdXQgbGF5ZXIuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBUcnVlLiBXaWxsIGJlIGlnbm9yZWQgaWYgJ2R5bmFtaWNfYXhlcycgaXMgcHJvdmlkZWQuCiAgICAgICAgIiIiCiAgICAgICAgIyBJbXBvcnQgdGhlIGZyYW1ld29yayBhbmQgaGFuZGxlcjoKICAgICAgICBpbXBvcnQgdG9yY2gKICAgICAgICBmcm9tIG1scnVuLmZyYW1ld29ya3MucHl0b3JjaCBpbXBvcnQgUHlUb3JjaFV0aWxzCgogICAgICAgICMgUGFyc2UgdGhlICdpbnB1dF9zaWduYXR1cmUnIHBhcmFtZXRlcjoKICAgICAgICBpZiBpbnB1dF9zaWduYXR1cmUgaXMgbm90IE5vbmU6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IHR1cGxlKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIHRvcmNoLnplcm9zKAogICAgICAgICAgICAgICAgICAgICAgICBzaXplPXNoYXBlLAogICAgICAgICAgICAgICAgICAgICAgICBkdHlwZT1QeVRvcmNoVXRpbHMuY29udmVydF92YWx1ZV90eXBlX3RvX3RvcmNoX2R0eXBlKAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgICAgICksCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICAgICAgXQogICAgICAgICAgICApCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NhbXBsZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICAgICBpbnB1dF9sYXllcnNfbmFtZXM9aW5wdXRfbGF5ZXJzX25hbWVzLAogICAgICAgICAgICBvdXRwdXRfbGF5ZXJzX25hbWVzPW91dHB1dF9sYXllcnNfbmFtZXMsCiAgICAgICAgICAgIGR5bmFtaWNfYXhlcz1keW5hbWljX2F4ZXMsCiAgICAgICAgICAgIGlzX2JhdGNoZWQ9aXNfYmF0Y2hlZCwKICAgICAgICApCgoKIyBNYXAgZm9yIGdldHRpbmcgdGhlIGNvbnZlcnNpb24gZnVuY3Rpb24gYWNjb3JkaW5nIHRvIHRoZSBwcm92aWRlZCBmcmFtZXdvcms6Cl9DT05WRVJTSU9OX01BUCA9IHsKICAgICJ0ZW5zb3JmbG93LmtlcmFzIjogX1RvT05OWENvbnZlcnNpb25zLnRmX2tlcmFzX3RvX29ubngsCiAgICAidG9yY2giOiBfVG9PTk5YQ29udmVyc2lvbnMucHl0b3JjaF90b19vbm54LAp9ICAjIHR5cGU6IERpY3Rbc3RyLCBDYWxsYWJsZV0KCgpkZWYgdG9fb25ueCgKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgbW9kZWxfcGF0aDogc3RyLAogICAgbG9hZF9tb2RlbF9rd2FyZ3M6IGRpY3QgPSBOb25lLAogICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgb3B0aW1pemVfbW9kZWw6IGJvb2wgPSBUcnVlLAogICAgZnJhbWV3b3JrX2t3YXJnczogRGljdFtzdHIsIEFueV0gPSBOb25lLAopOgogICAgIiIiCiAgICBDb252ZXJ0IHRoZSBnaXZlbiBtb2RlbCB0byBhbiBPTk5YIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgVGhlIE1MUnVuIGZ1bmN0aW9uIGV4ZWN1dGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxfcGF0aDogICAgICAgIFRoZSBtb2RlbCBwYXRoIHN0b3JlIG9iamVjdC4KICAgIDpwYXJhbSBsb2FkX21vZGVsX2t3YXJnczogS2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUgYEF1dG9NTFJ1bi5sb2FkX21vZGVsYCBtZXRob2QuCiAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiAgIFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbiBgbW9kZWxfbmFtZWAgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSB1c2VkIHdpdGggYW4gYWRkaXRpb25hbCBzdWZmaXggYF9vbm54YC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgIFdoZXRoZXIgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlIG1vZGVsLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgIDpwYXJhbSBmcmFtZXdvcmtfa3dhcmdzOiAgQWRkaXRpb25hbCBhcmd1bWVudHMgZWFjaCBmcmFtZXdvcmsgbWF5IHJlcXVpcmUgdG8gY29udmVydCB0byBPTk5YLiBUbyBnZXQgdGhlIGRvYyBzdHJpbmcKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb2YgdGhlIGRlc2lyZWQgZnJhbWV3b3JrIG9ubnggY29udmVyc2lvbiBmdW5jdGlvbiwgcGFzcyAiaGVscCIuCiAgICAiIiIKICAgIGZyb20gbWxydW4uZnJhbWV3b3Jrcy5hdXRvX21scnVuLmF1dG9fbWxydW4gaW1wb3J0IEF1dG9NTFJ1bgoKICAgICMgR2V0IGEgbW9kZWwgaGFuZGxlciBvZiB0aGUgcmVxdWlyZWQgZnJhbWV3b3JrOgogICAgbG9hZF9tb2RlbF9rd2FyZ3MgPSBsb2FkX21vZGVsX2t3YXJncyBvciB7fQogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWxfcGF0aCwgY29udGV4dD1jb250ZXh0LCAqKmxvYWRfbW9kZWxfa3dhcmdzCiAgICApCgogICAgIyBHZXQgdGhlIG1vZGVsJ3MgZnJhbWV3b3JrOgogICAgZnJhbWV3b3JrID0gbW9kZWxfaGFuZGxlci5GUkFNRVdPUktfTkFNRQoKICAgICMgVXNlIHRoZSBjb252ZXJzaW9uIG1hcCB0byBnZXQgdGhlIHNwZWNpZmljIGZyYW1ld29yayB0byBvbm54IGNvbnZlcnNpb246CiAgICBpZiBmcmFtZXdvcmsgbm90IGluIF9DT05WRVJTSU9OX01BUDoKICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgZiJUaGUgZm9sbG93aW5nIGZyYW1ld29yazogJ3tmcmFtZXdvcmt9JywgaGFzIG5vIE9OTlggY29udmVyc2lvbi4iCiAgICAgICAgKQogICAgY29udmVyc2lvbl9mdW5jdGlvbiA9IF9DT05WRVJTSU9OX01BUFtmcmFtZXdvcmtdCgogICAgIyBDaGVjayBpZiBuZWVkZWQgdG8gcHJpbnQgdGhlIGZ1bmN0aW9uJ3MgZG9jIHN0cmluZyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzID09ICJoZWxwIjoKICAgICAgICBwcmludChjb252ZXJzaW9uX2Z1bmN0aW9uLl9fZG9jX18pCiAgICAgICAgcmV0dXJuCgogICAgIyBTZXQgdGhlIGRlZmF1bHQgZW1wdHkgZnJhbWV3b3JrIGt3YXJncyBpZiBuZWVkZWQ6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzIGlzIE5vbmU6CiAgICAgICAgZnJhbWV3b3JrX2t3YXJncyA9IHt9CgogICAgIyBSdW4gdGhlIGNvbnZlcnNpb246CiAgICB0cnk6CiAgICAgICAgY29udmVyc2lvbl9mdW5jdGlvbigKICAgICAgICAgICAgbW9kZWxfaGFuZGxlcj1tb2RlbF9oYW5kbGVyLAogICAgICAgICAgICBvbm54X21vZGVsX25hbWU9b25ueF9tb2RlbF9uYW1lLAogICAgICAgICAgICBvcHRpbWl6ZV9tb2RlbD1vcHRpbWl6ZV9tb2RlbCwKICAgICAgICAgICAgKipmcmFtZXdvcmtfa3dhcmdzLAogICAgICAgICkKICAgIGV4Y2VwdCBUeXBlRXJyb3IgYXMgZXhjZXB0aW9uOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIkVSUk9SOiBBIFR5cGVFcnJvciBleGNlcHRpb24gd2FzIHJhaXNlZCBkdXJpbmcgdGhlIGNvbnZlcnNpb246XG57ZXhjZXB0aW9ufS4gIgogICAgICAgICAgICBmIlBsZWFzZSByZWFkIHRoZSB7ZnJhbWV3b3JrfSBmcmFtZXdvcmsgY29udmVyc2lvbiBmdW5jdGlvbiBkb2Mgc3RyaW5nIGJ5IHBhc3NpbmcgJ2hlbHAnIGluIHRoZSAiCiAgICAgICAgICAgIGYiJ2ZyYW1ld29ya19rd2FyZ3MnIGRpY3Rpb25hcnkgcGFyYW1ldGVyLiIKICAgICAgICApCgoKZGVmIG9wdGltaXplKAogICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICBtb2RlbF9wYXRoOiBzdHIsCiAgICBoYW5kbGVyX2luaXRfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIG9wdGltaXphdGlvbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBmaXhlZF9wb2ludDogYm9vbCA9IEZhbHNlLAogICAgb3B0aW1pemVkX21vZGVsX25hbWU6IHN0ciA9IE5vbmUsCik6CiAgICAiIiIKICAgIE9wdGltaXplIHRoZSBnaXZlbiBPTk5YIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgVGhlIE1MUnVuIGZ1bmN0aW9uIGV4ZWN1dGlvbiBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsX3BhdGg6ICAgICAgICAgICBQYXRoIHRvIHRoZSBPTk5YIG1vZGVsIG9iamVjdC4KICAgIDpwYXJhbSBoYW5kbGVyX2luaXRfa3dhcmdzOiAgS2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUgYE9OTlhNb2RlbEhhbmRsZXJgIGluaXQgbWV0aG9kIHByZWxvYWRpbmcuCiAgICA6cGFyYW0gb3B0aW1pemF0aW9uczogICAgICAgIExpc3Qgb2YgcG9zc2libGUgb3B0aW1pemF0aW9ucy4gVG8gc2VlIHdoYXQgb3B0aW1pemF0aW9ucyBhcmUgYXZhaWxhYmxlLCBwYXNzICJoZWxwIi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgYWxsIHRoZSBvcHRpbWl6YXRpb25zIHdpbGwgYmUgdXNlZC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gZml4ZWRfcG9pbnQ6ICAgICAgICAgIE9wdGltaXplIHRoZSB3ZWlnaHRzIHVzaW5nIGZpeGVkIHBvaW50LiBEZWZhdWx0ZWQgdG8gRmFsc2UuCiAgICA6cGFyYW0gb3B0aW1pemVkX21vZGVsX25hbWU6IFRoZSBuYW1lIG9mIHRoZSBvcHRpbWl6ZWQgbW9kZWwuIElmIE5vbmUsIHRoZSBvcmlnaW5hbCBtb2RlbCB3aWxsIGJlIG92ZXJyaWRkZW4uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgIiIiCiAgICAjIEltcG9ydCB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIGltcG9ydCBvbm54b3B0aW1pemVyCiAgICBmcm9tIG1scnVuLmZyYW1ld29ya3Mub25ueCBpbXBvcnQgT05OWE1vZGVsSGFuZGxlcgoKICAgICMgQ2hlY2sgaWYgbmVlZGVkIHRvIHByaW50IHRoZSBhdmFpbGFibGUgb3B0aW1pemF0aW9ucyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBvcHRpbWl6YXRpb25zID09ICJoZWxwIjoKICAgICAgICBhdmFpbGFibGVfcGFzc2VzID0gIlxuKiAiLmpvaW4ob25ueG9wdGltaXplci5nZXRfYXZhaWxhYmxlX3Bhc3NlcygpKQogICAgICAgIHByaW50KGYiVGhlIGF2YWlsYWJsZSBvcHRpbWl6YXRpb25zIGFyZTpcbioge2F2YWlsYWJsZV9wYXNzZXN9IikKICAgICAgICByZXR1cm4KCiAgICAjIENyZWF0ZSB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIGhhbmRsZXJfaW5pdF9rd2FyZ3MgPSBoYW5kbGVyX2luaXRfa3dhcmdzIG9yIHt9CiAgICBtb2RlbF9oYW5kbGVyID0gT05OWE1vZGVsSGFuZGxlcigKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsX3BhdGgsIGNvbnRleHQ9Y29udGV4dCwgKipoYW5kbGVyX2luaXRfa3dhcmdzCiAgICApCgogICAgIyBMb2FkIHRoZSBPTk5YIG1vZGVsOgogICAgbW9kZWxfaGFuZGxlci5sb2FkKCkKCiAgICAjIE9wdGltaXplIHRoZSBtb2RlbCB1c2luZyB0aGUgZ2l2ZW4gY29uZmlndXJhdGlvbnM6CiAgICBtb2RlbF9oYW5kbGVyLm9wdGltaXplKG9wdGltaXphdGlvbnM9b3B0aW1pemF0aW9ucywgZml4ZWRfcG9pbnQ9Zml4ZWRfcG9pbnQpCgogICAgIyBSZW5hbWUgaWYgbmVlZGVkOgogICAgaWYgb3B0aW1pemVkX21vZGVsX25hbWUgaXMgbm90IE5vbmU6CiAgICAgICAgbW9kZWxfaGFuZGxlci5zZXRfbW9kZWxfbmFtZShtb2RlbF9uYW1lPW9wdGltaXplZF9tb2RlbF9uYW1lKQoKICAgICMgTG9nIHRoZSBvcHRpbWl6ZWQgbW9kZWw6CiAgICBtb2RlbF9oYW5kbGVyLmxvZygpCg== - requirements: - - tqdm~=4.67.1 - - tensorflow~=2.19.0 - - tf_keras~=2.19.0 - - torch~=2.6.0 - - torchvision~=0.21.0 - - onnx~=1.17.0 - - onnxruntime~=1.19.2 - - onnxoptimizer~=0.3.13 - - onnxmltools~=1.13.0 - - tf2onnx~=1.16.1 - - plotly~=5.23 - with_mlrun: false - auto_build: true - disable_auto_mount: false - description: ONNX intigration in MLRun, some utils functions for the ONNX framework, - optimizing and converting models from different framework to ONNX using MLRun. - image: '' entry_points: tf_keras_to_onnx: - doc: Convert a TF.Keras model to an ONNX model and log it back to MLRun as a - new model object. name: tf_keras_to_onnx parameters: - name: model_handler @@ -58,12 +32,12 @@ spec: data type, a mlrun.data_types.ValueType string. If None, the input signature will be tried to be read from the model artifact. Defaulted to None.' default: null + doc: Convert a TF.Keras model to an ONNX model and log it back to MLRun as a + new model object. + lineno: 26 has_varargs: false has_kwargs: false - lineno: 26 pytorch_to_onnx: - doc: Convert a PyTorch model to an ONNX model and log it back to MLRun as a - new model object. name: pytorch_to_onnx parameters: - name: model_handler @@ -116,11 +90,12 @@ spec: doc: Whether to include a batch size as the first axis in every input and output layer. Defaulted to True. Will be ignored if 'dynamic_axes' is provided. default: true + doc: Convert a PyTorch model to an ONNX model and log it back to MLRun as a + new model object. + lineno: 81 has_varargs: false has_kwargs: false - lineno: 81 to_onnx: - doc: Convert the given model to an ONNX model. name: to_onnx parameters: - name: context @@ -150,11 +125,11 @@ spec: get the doc string of the desired framework onnx conversion function, pass "help". default: null + doc: Convert the given model to an ONNX model. + lineno: 160 has_varargs: false has_kwargs: false - lineno: 160 optimize: - doc: Optimize the given ONNX model. name: optimize parameters: - name: context @@ -181,9 +156,34 @@ spec: doc: The name of the optimized model. If None, the original model will be overridden. Defaulted to None. default: null + doc: Optimize the given ONNX model. + lineno: 224 has_varargs: false has_kwargs: false - lineno: 224 + image: '' default_handler: to_onnx allow_empty_resources: true command: '' + disable_auto_mount: false + description: ONNX intigration in MLRun, some utils functions for the ONNX framework, + optimizing and converting models from different framework to ONNX using MLRun. + build: + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgQ2FsbGFibGUsIERpY3QsIExpc3QsIFR1cGxlCgppbXBvcnQgbWxydW4KCgpjbGFzcyBfVG9PTk5YQ29udmVyc2lvbnM6CiAgICAiIiIKICAgIEFuIE9OTlggY29udmVyc2lvbiBmdW5jdGlvbnMgbGlicmFyeSBjbGFzcy4KICAgICIiIgoKICAgIEBzdGF0aWNtZXRob2QKICAgIGRlZiB0Zl9rZXJhc190b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50XSwgc3RyXV0gPSBOb25lLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBDb252ZXJ0IGEgVEYuS2VyYXMgbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICBBbiBpbml0aWFsaXplZCBURktlcmFzTW9kZWxIYW5kbGVyIHdpdGggYSBsb2FkZWQgbW9kZWwgdG8gY29udmVydCB0byBPTk5YLgogICAgICAgIDpwYXJhbSBvbm54X21vZGVsX25hbWU6IFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbiBgbW9kZWxfbmFtZWAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICBXaGV0aGVyIG9yIG5vdCB0byBvcHRpbWl6ZSB0aGUgT05OWCBtb2RlbCB1c2luZyAnb25ueG9wdGltaXplcicgYmVmb3JlIHNhdmluZyB0aGUgbW9kZWwuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgRGVmYXVsdGVkIHRvIFRydWUuCiAgICAgICAgOnBhcmFtIGlucHV0X3NpZ25hdHVyZTogQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEgbGlzdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHdoZXJlIGVhY2ggZWxlbWVudCBpcyBhbiBpbnB1dCBsYXllciB0dXBsZS4gQW4gaW5wdXQgbGF5ZXIgdHVwbGUgaXMgYSB0dXBsZSBvZjoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBbMF0gPSBMYXllcidzIHNoYXBlLCBhIHR1cGxlIG9mIGludGVnZXJzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBOb25lLCB0aGUgaW5wdXQgc2lnbmF0dXJlIHdpbGwgYmUgdHJpZWQgdG8gYmUgcmVhZCBmcm9tIHRoZSBtb2RlbCBhcnRpZmFjdC4gRGVmYXVsdGVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG8gTm9uZS4KICAgICAgICAiIiIKICAgICAgICAjIEltcG9ydCB0aGUgZnJhbWV3b3JrIGFuZCBoYW5kbGVyOgogICAgICAgIGltcG9ydCB0ZW5zb3JmbG93IGFzIHRmCiAgICAgICAgZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLnRmX2tlcmFzIGltcG9ydCBURktlcmFzVXRpbHMKCiAgICAgICAgIyBDaGVjayB0aGUgZ2l2ZW4gJ2lucHV0X3NpZ25hdHVyZScgcGFyYW1ldGVyOgogICAgICAgIGlmIGlucHV0X3NpZ25hdHVyZSBpcyBOb25lOgogICAgICAgICAgICAjIFJlYWQgdGhlIGlucHV0cyBmcm9tIHRoZSBtb2RlbDoKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgbW9kZWxfaGFuZGxlci5yZWFkX2lucHV0c19mcm9tX21vZGVsKCkKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlcnJvcjoKICAgICAgICAgICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1blJ1bnRpbWVFcnJvcigKICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSBwcm92aWRlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXIuIFRoZSBmdW5jdGlvbiB0cmllZCByZWFkaW5nIHRoZSBpbnB1dCBsYXllcnMgIgogICAgICAgICAgICAgICAgICAgIGYiaW5mb3JtYXRpb24gYXV0b21hdGljYWxseSBidXQgZmFpbGVkIHdpdGggdGhlIGZvbGxvd2luZyBlcnJvcjoge2Vycm9yfSIKICAgICAgICAgICAgICAgICkKICAgICAgICBlbHNlOgogICAgICAgICAgICAjIFBhcnNlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXI6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IFsKICAgICAgICAgICAgICAgIHRmLlRlbnNvclNwZWMoCiAgICAgICAgICAgICAgICAgICAgc2hhcGU9c2hhcGUsCiAgICAgICAgICAgICAgICAgICAgZHR5cGU9VEZLZXJhc1V0aWxzLmNvbnZlcnRfdmFsdWVfdHlwZV90b190Zl9kdHlwZSgKICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICBdCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICkKCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgcHl0b3JjaF90b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50LCAuLi5dLCBzdHJdXSA9IE5vbmUsCiAgICAgICAgaW5wdXRfbGF5ZXJzX25hbWVzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgICAgIG91dHB1dF9sYXllcnNfbmFtZXM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICAgICAgZHluYW1pY19heGVzOiBEaWN0W3N0ciwgRGljdFtpbnQsIHN0cl1dID0gTm9uZSwKICAgICAgICBpc19iYXRjaGVkOiBib29sID0gVHJ1ZSwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCBhIFB5VG9yY2ggbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICAgICAgQW4gaW5pdGlhbGl6ZWQgUHlUb3JjaE1vZGVsSGFuZGxlciB3aXRoIGEgbG9hZGVkIG1vZGVsIHRvIGNvbnZlcnQgdG8gT05OWC4KICAgICAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiAgICAgVGhlIG5hbWUgdG8gdXNlIHRvIGxvZyB0aGUgY29udmVydGVkIE9OTlggbW9kZWwuIElmIG5vdCBnaXZlbiwgdGhlIGdpdmVuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtb2RlbF9uYW1lYCB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgICAgV2hldGhlciBvciBub3QgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsLiBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgICAgICA6cGFyYW0gaW5wdXRfc2lnbmF0dXJlOiAgICAgQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdCB3aGVyZSBlYWNoIGVsZW1lbnQgaXMgYW4gaW5wdXQgbGF5ZXIgdHVwbGUuIEFuIGlucHV0IGxheWVyIHR1cGxlIGlzIGEgdHVwbGUgb2Y6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFswXSA9IExheWVyJ3Mgc2hhcGUsIGEgdHVwbGUgb2YgaW50ZWdlcnMuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgdGhlIGlucHV0IHNpZ25hdHVyZSB3aWxsIGJlIHRyaWVkIHRvIGJlIHJlYWQgZnJvbSB0aGUgbW9kZWwgYXJ0aWZhY3QuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpbnB1dF9sYXllcnNfbmFtZXM6ICBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgaW5wdXQgbm9kZXMgb2YgdGhlIGdyYXBoIGluIG9yZGVyLiBBbGwgb2YgdGhlIG90aGVyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHBhcmFtZXRlcnMgKGlubmVyIGxheWVycykgY2FuIGJlIHNldCBhcyB3ZWxsIGJ5IHBhc3NpbmcgYWRkaXRpb25hbCBuYW1lcyBpbiB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdC4gVGhlIG9yZGVyIGlzIGJ5IHRoZSBvcmRlciBvZiB0aGUgcGFyYW1ldGVycyBpbiB0aGUgbW9kZWwuIElmIE5vbmUsIHRoZSBpbnB1dHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBpbnB1dHMuIElmIGl0cyBhbHNvIE5vbmUsIGl0IGlzIGRlZmF1bHRlZCB0bzoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImlucHV0XzAiLCAiaW5wdXRfMSIsIC4uLgogICAgICAgIDpwYXJhbSBvdXRwdXRfbGF5ZXJzX25hbWVzOiBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgb3V0cHV0IG5vZGVzIG9mIHRoZSBncmFwaCBpbiBvcmRlci4gSWYgTm9uZSwgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG91dHB1dHMgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBvdXRwdXRzLiBJZiBpdHMgYWxzbyBOb25lLCBpdCBpcyBkZWZhdWx0ZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG86ICJvdXRwdXRfMCIgKGZvciBtdWx0aXBsZSBvdXRwdXRzLCB0aGlzIHBhcmFtZXRlciBtdXN0IGJlIHByb3ZpZGVkKS4KICAgICAgICA6cGFyYW0gZHluYW1pY19heGVzOiAgICAgICAgSWYgcGFydCBvZiB0aGUgaW5wdXQgLyBvdXRwdXQgc2hhcGUgaXMgZHluYW1pYywgbGlrZSAoYmF0Y2hfc2l6ZSwgMywgMzIsIDMyKSB5b3UgY2FuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNwZWNpZnkgaXQgYnkgZ2l2aW5nIGEgZHluYW1pYyBheGlzIHRvIHRoZSBpbnB1dCAvIG91dHB1dCBsYXllciBieSBpdHMgbmFtZSBhcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb2xsb3dzOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiaW5wdXQgbGF5ZXIgbmFtZSI6IHswOiAiYmF0Y2hfc2l6ZSJ9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm91dHB1dCBsYXllciBuYW1lIjogezA6ICJiYXRjaF9zaXplIn0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgcHJvdmlkZWQsIHRoZSAnaXNfYmF0Y2hlZCcgZmxhZyB3aWxsIGJlIGlnbm9yZWQuIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpc19iYXRjaGVkOiAgICAgICAgICBXaGV0aGVyIHRvIGluY2x1ZGUgYSBiYXRjaCBzaXplIGFzIHRoZSBmaXJzdCBheGlzIGluIGV2ZXJ5IGlucHV0IGFuZCBvdXRwdXQgbGF5ZXIuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBUcnVlLiBXaWxsIGJlIGlnbm9yZWQgaWYgJ2R5bmFtaWNfYXhlcycgaXMgcHJvdmlkZWQuCiAgICAgICAgIiIiCiAgICAgICAgIyBJbXBvcnQgdGhlIGZyYW1ld29yayBhbmQgaGFuZGxlcjoKICAgICAgICBpbXBvcnQgdG9yY2gKICAgICAgICBmcm9tIG1scnVuLmZyYW1ld29ya3MucHl0b3JjaCBpbXBvcnQgUHlUb3JjaFV0aWxzCgogICAgICAgICMgUGFyc2UgdGhlICdpbnB1dF9zaWduYXR1cmUnIHBhcmFtZXRlcjoKICAgICAgICBpZiBpbnB1dF9zaWduYXR1cmUgaXMgbm90IE5vbmU6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IHR1cGxlKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIHRvcmNoLnplcm9zKAogICAgICAgICAgICAgICAgICAgICAgICBzaXplPXNoYXBlLAogICAgICAgICAgICAgICAgICAgICAgICBkdHlwZT1QeVRvcmNoVXRpbHMuY29udmVydF92YWx1ZV90eXBlX3RvX3RvcmNoX2R0eXBlKAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgICAgICksCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICAgICAgXQogICAgICAgICAgICApCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NhbXBsZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICAgICBpbnB1dF9sYXllcnNfbmFtZXM9aW5wdXRfbGF5ZXJzX25hbWVzLAogICAgICAgICAgICBvdXRwdXRfbGF5ZXJzX25hbWVzPW91dHB1dF9sYXllcnNfbmFtZXMsCiAgICAgICAgICAgIGR5bmFtaWNfYXhlcz1keW5hbWljX2F4ZXMsCiAgICAgICAgICAgIGlzX2JhdGNoZWQ9aXNfYmF0Y2hlZCwKICAgICAgICApCgoKIyBNYXAgZm9yIGdldHRpbmcgdGhlIGNvbnZlcnNpb24gZnVuY3Rpb24gYWNjb3JkaW5nIHRvIHRoZSBwcm92aWRlZCBmcmFtZXdvcms6Cl9DT05WRVJTSU9OX01BUCA9IHsKICAgICJ0ZW5zb3JmbG93LmtlcmFzIjogX1RvT05OWENvbnZlcnNpb25zLnRmX2tlcmFzX3RvX29ubngsCiAgICAidG9yY2giOiBfVG9PTk5YQ29udmVyc2lvbnMucHl0b3JjaF90b19vbm54LAp9ICAjIHR5cGU6IERpY3Rbc3RyLCBDYWxsYWJsZV0KCgpkZWYgdG9fb25ueCgKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgbW9kZWxfcGF0aDogc3RyLAogICAgbG9hZF9tb2RlbF9rd2FyZ3M6IGRpY3QgPSBOb25lLAogICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgb3B0aW1pemVfbW9kZWw6IGJvb2wgPSBUcnVlLAogICAgZnJhbWV3b3JrX2t3YXJnczogRGljdFtzdHIsIEFueV0gPSBOb25lLAopOgogICAgIiIiCiAgICBDb252ZXJ0IHRoZSBnaXZlbiBtb2RlbCB0byBhbiBPTk5YIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgVGhlIE1MUnVuIGZ1bmN0aW9uIGV4ZWN1dGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxfcGF0aDogICAgICAgIFRoZSBtb2RlbCBwYXRoIHN0b3JlIG9iamVjdC4KICAgIDpwYXJhbSBsb2FkX21vZGVsX2t3YXJnczogS2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUgYEF1dG9NTFJ1bi5sb2FkX21vZGVsYCBtZXRob2QuCiAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiAgIFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbiBgbW9kZWxfbmFtZWAgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSB1c2VkIHdpdGggYW4gYWRkaXRpb25hbCBzdWZmaXggYF9vbm54YC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgIFdoZXRoZXIgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlIG1vZGVsLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgIDpwYXJhbSBmcmFtZXdvcmtfa3dhcmdzOiAgQWRkaXRpb25hbCBhcmd1bWVudHMgZWFjaCBmcmFtZXdvcmsgbWF5IHJlcXVpcmUgdG8gY29udmVydCB0byBPTk5YLiBUbyBnZXQgdGhlIGRvYyBzdHJpbmcKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb2YgdGhlIGRlc2lyZWQgZnJhbWV3b3JrIG9ubnggY29udmVyc2lvbiBmdW5jdGlvbiwgcGFzcyAiaGVscCIuCiAgICAiIiIKICAgIGZyb20gbWxydW4uZnJhbWV3b3Jrcy5hdXRvX21scnVuLmF1dG9fbWxydW4gaW1wb3J0IEF1dG9NTFJ1bgoKICAgICMgR2V0IGEgbW9kZWwgaGFuZGxlciBvZiB0aGUgcmVxdWlyZWQgZnJhbWV3b3JrOgogICAgbG9hZF9tb2RlbF9rd2FyZ3MgPSBsb2FkX21vZGVsX2t3YXJncyBvciB7fQogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWxfcGF0aCwgY29udGV4dD1jb250ZXh0LCAqKmxvYWRfbW9kZWxfa3dhcmdzCiAgICApCgogICAgIyBHZXQgdGhlIG1vZGVsJ3MgZnJhbWV3b3JrOgogICAgZnJhbWV3b3JrID0gbW9kZWxfaGFuZGxlci5GUkFNRVdPUktfTkFNRQoKICAgICMgVXNlIHRoZSBjb252ZXJzaW9uIG1hcCB0byBnZXQgdGhlIHNwZWNpZmljIGZyYW1ld29yayB0byBvbm54IGNvbnZlcnNpb246CiAgICBpZiBmcmFtZXdvcmsgbm90IGluIF9DT05WRVJTSU9OX01BUDoKICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgZiJUaGUgZm9sbG93aW5nIGZyYW1ld29yazogJ3tmcmFtZXdvcmt9JywgaGFzIG5vIE9OTlggY29udmVyc2lvbi4iCiAgICAgICAgKQogICAgY29udmVyc2lvbl9mdW5jdGlvbiA9IF9DT05WRVJTSU9OX01BUFtmcmFtZXdvcmtdCgogICAgIyBDaGVjayBpZiBuZWVkZWQgdG8gcHJpbnQgdGhlIGZ1bmN0aW9uJ3MgZG9jIHN0cmluZyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzID09ICJoZWxwIjoKICAgICAgICBwcmludChjb252ZXJzaW9uX2Z1bmN0aW9uLl9fZG9jX18pCiAgICAgICAgcmV0dXJuCgogICAgIyBTZXQgdGhlIGRlZmF1bHQgZW1wdHkgZnJhbWV3b3JrIGt3YXJncyBpZiBuZWVkZWQ6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzIGlzIE5vbmU6CiAgICAgICAgZnJhbWV3b3JrX2t3YXJncyA9IHt9CgogICAgIyBSdW4gdGhlIGNvbnZlcnNpb246CiAgICB0cnk6CiAgICAgICAgY29udmVyc2lvbl9mdW5jdGlvbigKICAgICAgICAgICAgbW9kZWxfaGFuZGxlcj1tb2RlbF9oYW5kbGVyLAogICAgICAgICAgICBvbm54X21vZGVsX25hbWU9b25ueF9tb2RlbF9uYW1lLAogICAgICAgICAgICBvcHRpbWl6ZV9tb2RlbD1vcHRpbWl6ZV9tb2RlbCwKICAgICAgICAgICAgKipmcmFtZXdvcmtfa3dhcmdzLAogICAgICAgICkKICAgIGV4Y2VwdCBUeXBlRXJyb3IgYXMgZXhjZXB0aW9uOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIkVSUk9SOiBBIFR5cGVFcnJvciBleGNlcHRpb24gd2FzIHJhaXNlZCBkdXJpbmcgdGhlIGNvbnZlcnNpb246XG57ZXhjZXB0aW9ufS4gIgogICAgICAgICAgICBmIlBsZWFzZSByZWFkIHRoZSB7ZnJhbWV3b3JrfSBmcmFtZXdvcmsgY29udmVyc2lvbiBmdW5jdGlvbiBkb2Mgc3RyaW5nIGJ5IHBhc3NpbmcgJ2hlbHAnIGluIHRoZSAiCiAgICAgICAgICAgIGYiJ2ZyYW1ld29ya19rd2FyZ3MnIGRpY3Rpb25hcnkgcGFyYW1ldGVyLiIKICAgICAgICApCgoKZGVmIG9wdGltaXplKAogICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICBtb2RlbF9wYXRoOiBzdHIsCiAgICBoYW5kbGVyX2luaXRfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIG9wdGltaXphdGlvbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBmaXhlZF9wb2ludDogYm9vbCA9IEZhbHNlLAogICAgb3B0aW1pemVkX21vZGVsX25hbWU6IHN0ciA9IE5vbmUsCik6CiAgICAiIiIKICAgIE9wdGltaXplIHRoZSBnaXZlbiBPTk5YIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgVGhlIE1MUnVuIGZ1bmN0aW9uIGV4ZWN1dGlvbiBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsX3BhdGg6ICAgICAgICAgICBQYXRoIHRvIHRoZSBPTk5YIG1vZGVsIG9iamVjdC4KICAgIDpwYXJhbSBoYW5kbGVyX2luaXRfa3dhcmdzOiAgS2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUgYE9OTlhNb2RlbEhhbmRsZXJgIGluaXQgbWV0aG9kIHByZWxvYWRpbmcuCiAgICA6cGFyYW0gb3B0aW1pemF0aW9uczogICAgICAgIExpc3Qgb2YgcG9zc2libGUgb3B0aW1pemF0aW9ucy4gVG8gc2VlIHdoYXQgb3B0aW1pemF0aW9ucyBhcmUgYXZhaWxhYmxlLCBwYXNzICJoZWxwIi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgYWxsIHRoZSBvcHRpbWl6YXRpb25zIHdpbGwgYmUgdXNlZC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gZml4ZWRfcG9pbnQ6ICAgICAgICAgIE9wdGltaXplIHRoZSB3ZWlnaHRzIHVzaW5nIGZpeGVkIHBvaW50LiBEZWZhdWx0ZWQgdG8gRmFsc2UuCiAgICA6cGFyYW0gb3B0aW1pemVkX21vZGVsX25hbWU6IFRoZSBuYW1lIG9mIHRoZSBvcHRpbWl6ZWQgbW9kZWwuIElmIE5vbmUsIHRoZSBvcmlnaW5hbCBtb2RlbCB3aWxsIGJlIG92ZXJyaWRkZW4uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgIiIiCiAgICAjIEltcG9ydCB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIGltcG9ydCBvbm54b3B0aW1pemVyCiAgICBmcm9tIG1scnVuLmZyYW1ld29ya3Mub25ueCBpbXBvcnQgT05OWE1vZGVsSGFuZGxlcgoKICAgICMgQ2hlY2sgaWYgbmVlZGVkIHRvIHByaW50IHRoZSBhdmFpbGFibGUgb3B0aW1pemF0aW9ucyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBvcHRpbWl6YXRpb25zID09ICJoZWxwIjoKICAgICAgICBhdmFpbGFibGVfcGFzc2VzID0gIlxuKiAiLmpvaW4ob25ueG9wdGltaXplci5nZXRfYXZhaWxhYmxlX3Bhc3NlcygpKQogICAgICAgIHByaW50KGYiVGhlIGF2YWlsYWJsZSBvcHRpbWl6YXRpb25zIGFyZTpcbioge2F2YWlsYWJsZV9wYXNzZXN9IikKICAgICAgICByZXR1cm4KCiAgICAjIENyZWF0ZSB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIGhhbmRsZXJfaW5pdF9rd2FyZ3MgPSBoYW5kbGVyX2luaXRfa3dhcmdzIG9yIHt9CiAgICBtb2RlbF9oYW5kbGVyID0gT05OWE1vZGVsSGFuZGxlcigKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsX3BhdGgsIGNvbnRleHQ9Y29udGV4dCwgKipoYW5kbGVyX2luaXRfa3dhcmdzCiAgICApCgogICAgIyBMb2FkIHRoZSBPTk5YIG1vZGVsOgogICAgbW9kZWxfaGFuZGxlci5sb2FkKCkKCiAgICAjIE9wdGltaXplIHRoZSBtb2RlbCB1c2luZyB0aGUgZ2l2ZW4gY29uZmlndXJhdGlvbnM6CiAgICBtb2RlbF9oYW5kbGVyLm9wdGltaXplKG9wdGltaXphdGlvbnM9b3B0aW1pemF0aW9ucywgZml4ZWRfcG9pbnQ9Zml4ZWRfcG9pbnQpCgogICAgIyBSZW5hbWUgaWYgbmVlZGVkOgogICAgaWYgb3B0aW1pemVkX21vZGVsX25hbWUgaXMgbm90IE5vbmU6CiAgICAgICAgbW9kZWxfaGFuZGxlci5zZXRfbW9kZWxfbmFtZShtb2RlbF9uYW1lPW9wdGltaXplZF9tb2RlbF9uYW1lKQoKICAgICMgTG9nIHRoZSBvcHRpbWl6ZWQgbW9kZWw6CiAgICBtb2RlbF9oYW5kbGVyLmxvZygpCg== + base_image: mlrun/mlrun + with_mlrun: false + auto_build: true + requirements: + - tqdm~=4.67.1 + - tensorflow~=2.19.0 + - tf_keras~=2.19.0 + - torch~=2.8.0 + - torchvision~=0.23.0 + - onnx~=1.17.0 + - onnxruntime~=1.19.2 + - onnxoptimizer~=0.3.13 + - onnxmltools~=1.13.0 + - tf2onnx~=1.16.1 + - plotly~=5.23 + origin_filename: '' + code_origin: '' +verbose: false diff --git a/functions/src/onnx_utils/item.yaml b/functions/src/onnx_utils/item.yaml index 803bd2599..5f129389f 100644 --- a/functions/src/onnx_utils/item.yaml +++ b/functions/src/onnx_utils/item.yaml @@ -13,7 +13,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.2 +mlrunVersion: 1.10.0 name: onnx_utils platformVersion: 3.5.0 spec: @@ -30,8 +30,8 @@ spec: - tqdm~=4.67.1 - tensorflow~=2.19.0 - tf_keras~=2.19.0 - - torch~=2.6.0 - - torchvision~=0.21.0 + - torch~=2.8.0 + - torchvision~=0.23.0 - onnx~=1.17.0 - onnxruntime~=1.19.2 - onnxoptimizer~=0.3.13 @@ -39,4 +39,4 @@ spec: - tf2onnx~=1.16.1 - plotly~=5.23 url: '' -version: 1.3.0 +version: 1.4.0 diff --git a/functions/src/onnx_utils/requirements.txt b/functions/src/onnx_utils/requirements.txt index d3d7dfd68..912b3d7e5 100644 --- a/functions/src/onnx_utils/requirements.txt +++ b/functions/src/onnx_utils/requirements.txt @@ -1,11 +1,10 @@ tqdm~=4.67.1 tensorflow~=2.19.0 tf_keras~=2.19.0 -torch~=2.6.0 -torchvision~=0.21.0 +torch~=2.8 +torchvision~=0.23.0 onnx~=1.17.0 onnxruntime~=1.19.2 onnxoptimizer~=0.3.13 onnxmltools~=1.13.0 -tf2onnx~=1.16.1 -plotly~=5.23 +plotly~=5.23 \ No newline at end of file diff --git a/functions/src/onnx_utils/test_onnx_utils.py b/functions/src/onnx_utils/test_onnx_utils.py index 2e01782f5..36113ce1d 100644 --- a/functions/src/onnx_utils/test_onnx_utils.py +++ b/functions/src/onnx_utils/test_onnx_utils.py @@ -17,6 +17,10 @@ import tempfile import mlrun +import pytest + +# Project name for tests (must match conftest.py) +PROJECT_NAME = "onnx-utils" # Choose our model's name: MODEL_NAME = "model" @@ -27,6 +31,30 @@ # Choose our optimized ONNX version model's name: OPTIMIZED_ONNX_MODEL_NAME = f"optimized_{ONNX_MODEL_NAME}" +REQUIRED_ENV_VARS = [ + "MLRUN_DBPATH", + "MLRUN_ARTIFACT_PATH", +] + + +def _validate_environment_variables() -> bool: + """ + Checks that all required Environment variables are set. + """ + environment_keys = os.environ.keys() + return all(key in environment_keys for key in REQUIRED_ENV_VARS) + + +def _is_tf2onnx_available() -> bool: + """ + Check if tf2onnx is installed (required for TensorFlow/Keras ONNX conversion). + """ + try: + import tf2onnx + return True + except ImportError: + return False + def _setup_environment() -> str: """ @@ -52,6 +80,11 @@ def _cleanup_environment(artifact_path: str): "runs", "artifacts", "functions", + "model.pt", + "model.zip", + "model_modules_map.json", + "onnx_model.onnx", + "optimized_onnx_model.onnx", ]: test_output_path = os.path.abspath(f"./{test_output}") if os.path.exists(test_output_path): @@ -114,6 +147,14 @@ def _log_pytorch_model(context: mlrun.MLClientCtx, model_name: str): model_handler.log() +@pytest.mark.skipif( + condition=not _validate_environment_variables(), + reason="Project's environment variables are not set", +) +@pytest.mark.skipif( + condition=not _is_tf2onnx_available(), + reason="tf2onnx is not installed", +) def test_to_onnx_help(): """ Test the 'to_onnx' handler, passing "help" in the 'framework_kwargs'. @@ -125,6 +166,7 @@ def test_to_onnx_help(): log_model_function = mlrun.code_to_function( filename="test_onnx_utils.py", name="log_model", + project=PROJECT_NAME, kind="job", image="mlrun/ml-models", ) @@ -132,20 +174,20 @@ def test_to_onnx_help(): # Run the function to log the model: log_model_run = log_model_function.run( handler="_log_tf_keras_model", - artifact_path=artifact_path, + output_path=artifact_path, params={"model_name": MODEL_NAME}, local=True, ) # Import the ONNX Utils function: - onnx_function = mlrun.import_function("function.yaml") + onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) # Run the function, passing "help" in 'framework_kwargs' and see that no exception was raised: is_test_passed = True try: onnx_function.run( handler="to_onnx", - artifact_path=artifact_path, + output_path=artifact_path, params={ # Take the logged model from the previous function. "model_path": log_model_run.status.artifacts[0]["spec"]["target_path"], @@ -166,6 +208,14 @@ def test_to_onnx_help(): assert is_test_passed +@pytest.mark.skipif( + condition=not _validate_environment_variables(), + reason="Project's environment variables are not set", +) +@pytest.mark.skipif( + condition=not _is_tf2onnx_available(), + reason="tf2onnx is not installed", +) def test_tf_keras_to_onnx(): """ Test the 'to_onnx' handler, giving it a tf.keras model. @@ -177,6 +227,7 @@ def test_tf_keras_to_onnx(): log_model_function = mlrun.code_to_function( filename="test_onnx_utils.py", name="log_model", + project=PROJECT_NAME, kind="job", image="mlrun/ml-models", ) @@ -184,18 +235,18 @@ def test_tf_keras_to_onnx(): # Run the function to log the model: log_model_run = log_model_function.run( handler="_log_tf_keras_model", - artifact_path=artifact_path, + output_path=artifact_path, params={"model_name": MODEL_NAME}, local=True, ) # Import the ONNX Utils function: - onnx_function = mlrun.import_function("function.yaml") + onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) # Run the function to convert our model to ONNX: onnx_function_run = onnx_function.run( handler="to_onnx", - artifact_path=artifact_path, + output_path=artifact_path, params={ # Take the logged model from the previous function. "model_path": log_model_run.status.artifacts[0]["spec"]["target_path"], @@ -215,6 +266,10 @@ def test_tf_keras_to_onnx(): assert "model" in onnx_function_run.outputs +@pytest.mark.skipif( + condition=not _validate_environment_variables(), + reason="Project's environment variables are not set", +) def test_pytorch_to_onnx(): """ Test the 'to_onnx' handler, giving it a pytorch model. @@ -226,6 +281,7 @@ def test_pytorch_to_onnx(): log_model_function = mlrun.code_to_function( filename="test_onnx_utils.py", name="log_model", + project=PROJECT_NAME, kind="job", image="mlrun/ml-models", ) @@ -233,25 +289,30 @@ def test_pytorch_to_onnx(): # Run the function to log the model: log_model_run = log_model_function.run( handler="_log_pytorch_model", - artifact_path=artifact_path, + output_path=artifact_path, params={"model_name": MODEL_NAME}, local=True, ) # Import the ONNX Utils function: - onnx_function = mlrun.import_function("function.yaml") + onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) + + # Get artifact paths - construct from artifact_path and run structure + run_artifact_dir = os.path.join(artifact_path, "log-model--log-pytorch-model", "0") + model_path = os.path.join(run_artifact_dir, "model") + modules_map_path = os.path.join(run_artifact_dir, "model_modules_map.json.json") # Run the function to convert our model to ONNX: onnx_function_run = onnx_function.run( handler="to_onnx", - artifact_path=artifact_path, + output_path=artifact_path, params={ # Take the logged model from the previous function. - "model_path": log_model_run.status.artifacts[1]["spec"]["target_path"], + "model_path": model_path, "load_model_kwargs": { "model_name": MODEL_NAME, "model_class": "mobilenet_v2", - "modules_map": log_model_run.status.artifacts[0]["spec"]["target_path"], + "modules_map": modules_map_path, }, "onnx_model_name": ONNX_MODEL_NAME, "framework_kwargs": {"input_signature": [((32, 3, 224, 224), "float32")]}, @@ -269,6 +330,10 @@ def test_pytorch_to_onnx(): assert "model" in onnx_function_run.outputs +@pytest.mark.skipif( + condition=not _validate_environment_variables(), + reason="Project's environment variables are not set", +) def test_optimize_help(): """ Test the 'optimize' handler, passing "help" in the 'optimizations'. @@ -277,14 +342,14 @@ def test_optimize_help(): artifact_path = _setup_environment() # Import the ONNX Utils function: - onnx_function = mlrun.import_function("function.yaml") + onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) # Run the function, passing "help" in 'optimizations' and see that no exception was raised: is_test_passed = True try: onnx_function.run( handler="optimize", - artifact_path=artifact_path, + output_path=artifact_path, params={ "model_path": "", "optimizations": "help", @@ -303,6 +368,14 @@ def test_optimize_help(): assert is_test_passed +@pytest.mark.skipif( + condition=not _validate_environment_variables(), + reason="Project's environment variables are not set", +) +@pytest.mark.skipif( + condition=not _is_tf2onnx_available(), + reason="tf2onnx is not installed", +) def test_optimize(): """ Test the 'optimize' handler, giving it a model from the ONNX zoo git repository. @@ -314,6 +387,7 @@ def test_optimize(): log_model_function = mlrun.code_to_function( filename="test_onnx_utils.py", name="log_model", + project=PROJECT_NAME, kind="job", image="mlrun/ml-models", ) @@ -321,18 +395,18 @@ def test_optimize(): # Run the function to log the model: log_model_run = log_model_function.run( handler="_log_tf_keras_model", - artifact_path=artifact_path, + output_path=artifact_path, params={"model_name": MODEL_NAME}, local=True, ) # Import the ONNX Utils function: - onnx_function = mlrun.import_function("function.yaml") + onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) # Run the function to convert our model to ONNX: to_onnx_function_run = onnx_function.run( handler="to_onnx", - artifact_path=artifact_path, + output_path=artifact_path, params={ # Take the logged model from the previous function. "model_path": log_model_run.status.artifacts[0]["spec"]["target_path"], @@ -345,7 +419,7 @@ def test_optimize(): # Run the function to optimize our model: optimize_function_run = onnx_function.run( handler="optimize", - artifact_path=artifact_path, + output_path=artifact_path, params={ # Take the logged model from the previous function. "model_path": to_onnx_function_run.status.artifacts[0]["spec"][ From 977d12a01812362b419aaa63fb1d9a4552734da9 Mon Sep 17 00:00:00 2001 From: Omer_Mimon Date: Tue, 10 Feb 2026 16:20:57 +0200 Subject: [PATCH 2/4] Add conftest fixture for test environment and update notebook to PyTorch demo - Centralize test setup/cleanup in conftest autouse fixture - Rewrite notebook demo from Keras to a working PyTorch MobileNetV2 example --- functions/src/onnx_utils/conftest.py | 58 ++ functions/src/onnx_utils/onnx_utils.ipynb | 971 +++++++++++++++++--- functions/src/onnx_utils/test_onnx_utils.py | 142 +-- 3 files changed, 970 insertions(+), 201 deletions(-) create mode 100644 functions/src/onnx_utils/conftest.py diff --git a/functions/src/onnx_utils/conftest.py b/functions/src/onnx_utils/conftest.py new file mode 100644 index 000000000..00740e02e --- /dev/null +++ b/functions/src/onnx_utils/conftest.py @@ -0,0 +1,58 @@ +# Copyright 2019 Iguazio +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import shutil +import tempfile + +import mlrun +import pytest + +PROJECT_NAME = "onnx-utils" + + +@pytest.fixture(scope="session") +def onnx_project(): + """Create/get the MLRun project once per test session.""" + return mlrun.get_or_create_project(PROJECT_NAME, context="./") + + +@pytest.fixture(autouse=True) +def test_environment(onnx_project): + """Setup and cleanup test artifacts for each test.""" + artifact_path = tempfile.mkdtemp() + yield artifact_path + # Cleanup - only remove files/dirs from the directory containing this test file, + # never from an arbitrary CWD (which could be the project root). + test_dir = os.path.dirname(os.path.abspath(__file__)) + for test_output in [ + "schedules", + "runs", + "artifacts", + "functions", + "model.pt", + "model.zip", + "model_modules_map.json", + "model_modules_map.json.json", + "onnx_model.onnx", + "optimized_onnx_model.onnx", + ]: + test_output_path = os.path.join(test_dir, test_output) + if os.path.exists(test_output_path): + if os.path.isdir(test_output_path): + shutil.rmtree(test_output_path) + else: + os.remove(test_output_path) + if os.path.exists(artifact_path): + shutil.rmtree(artifact_path) diff --git a/functions/src/onnx_utils/onnx_utils.ipynb b/functions/src/onnx_utils/onnx_utils.ipynb index 78203a45d..c0aaa3547 100644 --- a/functions/src/onnx_utils/onnx_utils.ipynb +++ b/functions/src/onnx_utils/onnx_utils.ipynb @@ -77,9 +77,9 @@ "source": [ "### 1.2. Demo\n", "\n", - "We will use the `TF.Keras` framework, a `MobileNetV2` as our model and we will convert it to ONNX using the `to_onnx` handler.\n", + "We will use the `PyTorch` framework, a `MobileNetV2` as our model and we will convert it to ONNX using the `to_onnx` handler.\n", "\n", - "1.2.1. First we will set a temporary artifact path for our model to be saved in and choose the models names:" + "1.2.1. First we will set the artifact path for our model to be saved in and choose the models names:" ] }, { @@ -87,28 +87,15 @@ "metadata": { "pycharm": { "name": "#%%\n" + }, + "ExecuteTime": { + "end_time": "2026-02-10T14:13:28.256582Z", + "start_time": "2026-02-10T14:13:28.250886Z" } }, - "source": [ - "import os\n", - "os.environ[\"TF_USE_LEGACY_KERAS\"] = \"true\"\n", - "from tempfile import TemporaryDirectory\n", - "\n", - "# Create a temporary directory for the model artifact:\n", - "ARTIFACT_PATH = TemporaryDirectory().name\n", - "os.makedirs(ARTIFACT_PATH)\n", - "\n", - "# Choose our model's name:\n", - "MODEL_NAME = \"mobilenetv2\"\n", - "\n", - "# Choose our ONNX version model's name:\n", - "ONNX_MODEL_NAME = \"onnx_mobilenetv2\"\n", - "\n", - "# Choose our optimized ONNX version model's name:\n", - "OPTIMIZED_ONNX_MODEL_NAME = \"optimized_onnx_mobilenetv2\"" - ], + "source": "import os\nimport tempfile\n\n# Set environment variables BEFORE importing mlrun:\nos.environ[\"MLRUN_DBPATH\"] = \"https://mlrun-api.default-tenant.app.innovation-dev.iguazio-cd2.com\"\nos.environ[\"V3IO_USERNAME\"] = \"omerm\"\nos.environ[\"V3IO_ACCESS_KEY\"] = \"618ed0ad-6b68-4be8-8785-b1124437eae2\"\n\n# Use a temporary directory for model artifacts (safe cleanup):\nARTIFACT_PATH = tempfile.mkdtemp()\nos.environ[\"MLRUN_ARTIFACT_PATH\"] = ARTIFACT_PATH\n\n# Project name:\nPROJECT_NAME = \"onnx-utils\"\n\n# Choose our model's name:\nMODEL_NAME = \"mobilenetv2\"\n\n# Choose our ONNX version model's name:\nONNX_MODEL_NAME = \"onnx_mobilenetv2\"\n\n# Choose our optimized ONNX version model's name:\nOPTIMIZED_ONNX_MODEL_NAME = \"optimized_onnx_mobilenetv2\"", "outputs": [], - "execution_count": null + "execution_count": 1 }, { "cell_type": "markdown", @@ -118,87 +105,88 @@ } }, "source": [ - "1.2.2. Download the model from `keras.applications` and log it with MLRun's `TFKerasModelHandler`:" + "1.2.2. Download the model from `torchvision.models` and log it with MLRun's `PyTorchModelHandler`:" ] }, { - "cell_type": "code", "metadata": { - "pycharm": { - "name": "#%%\n" + "ExecuteTime": { + "end_time": "2026-02-10T14:00:15.032590Z", + "start_time": "2026-02-10T14:00:15.031196Z" } }, - "source": [ - "# mlrun: start-code" - ], + "cell_type": "code", + "source": "# mlrun: start-code", "outputs": [], - "execution_count": null + "execution_count": 8 }, { + "metadata": { + "ExecuteTime": { + "end_time": "2026-02-10T14:14:00.992001Z", + "start_time": "2026-02-10T14:13:33.115438Z" + } + }, "cell_type": "code", - "metadata": {}, "source": [ - "from tensorflow import keras\n", + "import torchvision\n", "\n", "import mlrun\n", - "import mlrun.frameworks.tf_keras as mlrun_tf_keras\n", + "from mlrun.frameworks.pytorch import PyTorchModelHandler\n", "\n", "\n", "def get_model(context: mlrun.MLClientCtx, model_name: str):\n", " # Download the MobileNetV2 model:\n", - " model = keras.applications.mobilenet_v2.MobileNetV2()\n", + " model = torchvision.models.mobilenet_v2()\n", "\n", " # Initialize a model handler for logging the model:\n", - " model_handler = mlrun_tf_keras.TFKerasModelHandler(\n", + " model_handler = PyTorchModelHandler(\n", " model_name=model_name,\n", " model=model,\n", - " context=context\n", + " model_class=\"mobilenet_v2\",\n", + " modules_map={\"torchvision.models\": \"mobilenet_v2\"},\n", + " context=context,\n", " )\n", "\n", " # Log the model:\n", " model_handler.log()" ], "outputs": [], - "execution_count": null + "execution_count": 2 }, { - "cell_type": "code", "metadata": { - "pycharm": { - "name": "#%%\n" + "ExecuteTime": { + "end_time": "2026-02-10T14:00:15.040221Z", + "start_time": "2026-02-10T14:00:15.038886Z" } }, - "source": [ - "# mlrun: end-code" - ], + "cell_type": "code", + "source": "# mlrun: end-code", "outputs": [], - "execution_count": null - }, - { - "cell_type": "markdown", - "metadata": { - "pycharm": { - "name": "#%% md\n" - } - }, - "source": [ - "1.2.3. Create the function using MLRun's `code_to_function` and run it:" - ] + "execution_count": 10 }, { "cell_type": "code", "metadata": { "pycharm": { "name": "#%%\n" + }, + "ExecuteTime": { + "end_time": "2026-02-10T14:14:34.429194Z", + "start_time": "2026-02-10T14:14:07.906087Z" } }, "source": [ "import mlrun\n", "\n", + "# Create or get the MLRun project:\n", + "project = mlrun.get_or_create_project(PROJECT_NAME, context=\"./\")\n", "\n", "# Create the function parsing this notebook's code using 'code_to_function':\n", "get_model_function = mlrun.code_to_function(\n", " name=\"get_mobilenetv2\",\n", + " project=PROJECT_NAME,\n", " kind=\"job\",\n", " image=\"mlrun/ml-models\"\n", ")\n", @@ -206,15 +194,267 @@ "# Run the function to log the model:\n", "get_model_run = get_model_function.run(\n", " handler=\"get_model\",\n", - " artifact_path=ARTIFACT_PATH,\n", + " output_path=ARTIFACT_PATH,\n", " params={\n", " \"model_name\": MODEL_NAME\n", " },\n", " local=True\n", ")" ], - "outputs": [], - "execution_count": null + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> 2026-02-10 16:14:24,932 [info] Created and saved project: {\"context\":\"./\",\"from_template\":null,\"name\":\"onnx-utils\",\"overwrite\":false,\"save\":true}\n", + "> 2026-02-10 16:14:24,933 [info] Project created successfully: {\"project_name\":\"onnx-utils\",\"stored_in_db\":true}\n", + "> 2026-02-10 16:14:31,659 [info] Storing function: {\"db\":null,\"name\":\"get-mobilenetv2-get-model\",\"uid\":\"7b9d1b54375b44e191d73685a382c910\"}\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
projectuiditerstartendstatekindnamelabelsinputsparametersresultsartifact_uris
onnx-utils0Feb 10 14:14:32NaTcompletedrunget-mobilenetv2-get-model
v3io_user=omerm
kind=local
owner=omerm
host=M-KCX16N69X3
model_name=mobilenetv2
mobilenetv2_modules_map.json=store://artifacts/onnx-utils/#0@7b9d1b54375b44e191d73685a382c910
model=store://models/onnx-utils/mobilenetv2#0@7b9d1b54375b44e191d73685a382c910^e0393bc5b070fd55cc57cecb94160ce412498e0f
\n", + "
\n", + "
\n", + "
\n", + " Title\n", + " ×\n", + "
\n", + " \n", + "
\n", + "
\n" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " > to track results use the .show() or .logs() methods or click here to open in UI" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> 2026-02-10 16:14:34,427 [info] Run execution finished: {\"name\":\"get-mobilenetv2-get-model\",\"status\":\"completed\"}\n" + ] + } + ], + "execution_count": 3 }, { "cell_type": "markdown", @@ -228,33 +468,271 @@ "metadata": { "pycharm": { "name": "#%%\n" + }, + "ExecuteTime": { + "end_time": "2026-02-10T14:14:53.863947Z", + "start_time": "2026-02-10T14:14:48.088349Z" } }, - "source": [ - "# Import the ONNX function from the marketplace:\n", - "onnx_utils_function = mlrun.import_function(\"hub://onnx_utils\")\n", - "\n", - "# Run the function to convert our model to ONNX:\n", - "to_onnx_run = onnx_utils_function.run(\n", - " handler=\"to_onnx\",\n", - " artifact_path=ARTIFACT_PATH,\n", - " params={\n", - " \"model_name\": MODEL_NAME,\n", - " \"model_path\": get_model_run.outputs[MODEL_NAME], # <- Take the logged model from the previous function.\n", - " \"onnx_model_name\": ONNX_MODEL_NAME,\n", - " \"optimize_model\": False # <- For optimizing it later in the demo, we mark the flag as False\n", - " },\n", - " local=True\n", - ")" + "source": "# Import the ONNX function from the marketplace:\nonnx_utils_function = mlrun.import_function(\"hub://onnx_utils\", project=PROJECT_NAME)\n\n# Construct the model path from the run directory structure:\nmodel_path = os.path.join(ARTIFACT_PATH, \"get-mobilenetv2-get-model\", \"0\", \"model\")\nmodules_map_path = os.path.join(ARTIFACT_PATH, \"get-mobilenetv2-get-model\", \"0\", \"mobilenetv2_modules_map.json.json\")\n\n# Run the function to convert our model to ONNX:\nto_onnx_run = onnx_utils_function.run(\n handler=\"to_onnx\",\n output_path=ARTIFACT_PATH,\n params={\n \"model_name\": MODEL_NAME,\n \"model_path\": model_path,\n \"load_model_kwargs\": {\n \"model_name\": MODEL_NAME,\n \"model_class\": \"mobilenet_v2\",\n \"modules_map\": modules_map_path,\n },\n \"onnx_model_name\": ONNX_MODEL_NAME,\n \"optimize_model\": False, # <- For optimizing it later in the demo, we mark the flag as False\n \"framework_kwargs\": {\"input_signature\": [((32, 3, 224, 224), \"float32\")]},\n },\n local=True\n)", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> 2026-02-10 16:14:48,519 [info] Storing function: {\"db\":null,\"name\":\"onnx-utils-to-onnx\",\"uid\":\"95deb2c7dbf0460291efb25c48eeebd7\"}\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
projectuiditerstartendstatekindnamelabelsinputsparametersresultsartifact_uris
onnx-utils0Feb 10 14:14:49NaTcompletedrunonnx-utils-to-onnx
v3io_user=omerm
kind=local
owner=omerm
host=M-KCX16N69X3
model_name=mobilenetv2
model_path=/var/folders/rn/q8gs952n26982d36y50w_2rw0000gp/T/tmpvs5qvbxr/get-mobilenetv2-get-model/0/model
load_model_kwargs={'model_name': 'mobilenetv2', 'model_class': 'mobilenet_v2', 'modules_map': '/var/folders/rn/q8gs952n26982d36y50w_2rw0000gp/T/tmpvs5qvbxr/get-mobilenetv2-get-model/0/mobilenetv2_modules_map.json.json'}
onnx_model_name=onnx_mobilenetv2
optimize_model=False
framework_kwargs={'input_signature': [((32, 3, 224, 224), 'float32')]}
model=store://models/onnx-utils/onnx_mobilenetv2#0@95deb2c7dbf0460291efb25c48eeebd7^03e4286da44d015cf5465d43e809a504d15f7f63
\n", + "
\n", + "
\n", + "
\n", + " Title\n", + " ×\n", + "
\n", + " \n", + "
\n", + "
\n" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " > to track results use the .show() or .logs() methods or click here to open in UI" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> 2026-02-10 16:14:53,862 [info] Run execution finished: {\"name\":\"onnx-utils-to-onnx\",\"status\":\"completed\"}\n" + ] + } ], - "outputs": [], - "execution_count": null + "execution_count": 4 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "1.2.5. Now, listing the artifact directory we will see both our `tf.keras` model and the `onnx` model:" + "1.2.5. Now we verify the ONNX model was created:" ] }, { @@ -262,16 +740,29 @@ "metadata": { "pycharm": { "name": "#%%\n" + }, + "ExecuteTime": { + "end_time": "2026-02-10T14:14:56.820411Z", + "start_time": "2026-02-10T14:14:56.817892Z" } }, "source": [ "import os\n", "\n", - "\n", - "print(os.listdir(ARTIFACT_PATH))" + "onnx_model_file = os.path.join(ARTIFACT_PATH, \"onnx-utils-to-onnx\", \"0\", \"model\", \"onnx_mobilenetv2.onnx\")\n", + "assert os.path.isfile(onnx_model_file), f\"ONNX model not found at {onnx_model_file}\"\n", + "print(f\"ONNX model created at: {onnx_model_file}\")" ], - "outputs": [], - "execution_count": null + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ONNX model created at: /var/folders/rn/q8gs952n26982d36y50w_2rw0000gp/T/tmpvs5qvbxr/onnx-utils-to-onnx/0/model/onnx_mobilenetv2.onnx\n" + ] + } + ], + "execution_count": 5 }, { "cell_type": "markdown", @@ -308,28 +799,281 @@ "metadata": { "pycharm": { "name": "#%%\n" + }, + "ExecuteTime": { + "end_time": "2026-02-10T14:15:03.415997Z", + "start_time": "2026-02-10T14:15:00.637332Z" } }, - "source": [ - "onnx_utils_function.run(\n", - " handler=\"optimize\",\n", - " artifact_path=ARTIFACT_PATH,\n", - " params={\n", - " \"model_name\": ONNX_MODEL_NAME,\n", - " \"model_path\": to_onnx_run.output(ONNX_MODEL_NAME), # <- Take the logged model from the previous function.\n", - " \"optimized_model_name\": OPTIMIZED_ONNX_MODEL_NAME,\n", - " },\n", - " local=True\n", - ")" + "source": "# Construct the ONNX model path from the run directory structure:\nonnx_model_path = os.path.join(ARTIFACT_PATH, \"onnx-utils-to-onnx\", \"0\", \"model\")\n\nonnx_utils_function.run(\n handler=\"optimize\",\n output_path=ARTIFACT_PATH,\n params={\n \"model_path\": onnx_model_path,\n \"handler_init_kwargs\": {\"model_name\": ONNX_MODEL_NAME},\n \"optimized_model_name\": OPTIMIZED_ONNX_MODEL_NAME,\n },\n local=True\n)", + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> 2026-02-10 16:15:00,639 [info] Storing function: {\"db\":null,\"name\":\"onnx-utils-optimize\",\"uid\":\"0c30d7af94814dcabde8152a1951fb5d\"}\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
projectuiditerstartendstatekindnamelabelsinputsparametersresultsartifact_uris
onnx-utils0Feb 10 14:15:01NaTcompletedrunonnx-utils-optimize
v3io_user=omerm
kind=local
owner=omerm
host=M-KCX16N69X3
model_path=/var/folders/rn/q8gs952n26982d36y50w_2rw0000gp/T/tmpvs5qvbxr/onnx-utils-to-onnx/0/model
handler_init_kwargs={'model_name': 'onnx_mobilenetv2'}
optimized_model_name=optimized_onnx_mobilenetv2
model=store://models/onnx-utils/optimized_onnx_mobilenetv2#0@0c30d7af94814dcabde8152a1951fb5d^599547984e83a664dc1c2708607d06731edb5ac2
\n", + "
\n", + "
\n", + "
\n", + " Title\n", + " ×\n", + "
\n", + " \n", + "
\n", + "
\n" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ], + "text/html": [ + " > to track results use the .show() or .logs() methods or click here to open in UI" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "> 2026-02-10 16:15:03,414 [info] Run execution finished: {\"name\":\"onnx-utils-optimize\",\"status\":\"completed\"}\n" + ] + }, + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } ], - "outputs": [], - "execution_count": null + "execution_count": 6 }, { "cell_type": "markdown", "metadata": {}, "source": [ - "2.2.2. And now our model was optimized and can be seen under the `ARTIFACT_PATH`:" + "2.2.2. And now our model was optimized. Let us verify:" ] }, { @@ -337,13 +1081,27 @@ "metadata": { "pycharm": { "name": "#%%\n" + }, + "ExecuteTime": { + "end_time": "2026-02-10T14:15:05.748413Z", + "start_time": "2026-02-10T14:15:05.745309Z" } }, "source": [ - "print(os.listdir(ARTIFACT_PATH))" + "optimized_model_file = os.path.join(ARTIFACT_PATH, \"onnx-utils-optimize\", \"0\", \"model\", \"optimized_onnx_mobilenetv2.onnx\")\n", + "assert os.path.isfile(optimized_model_file), f\"Optimized ONNX model not found at {optimized_model_file}\"\n", + "print(f\"Optimized ONNX model created at: {optimized_model_file}\")" ], - "outputs": [], - "execution_count": null + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Optimized ONNX model created at: /var/folders/rn/q8gs952n26982d36y50w_2rw0000gp/T/tmpvs5qvbxr/onnx-utils-optimize/0/model/optimized_onnx_mobilenetv2.onnx\n" + ] + } + ], + "execution_count": 7 }, { "cell_type": "markdown", @@ -353,7 +1111,7 @@ } }, "source": [ - "Lastly, run this code to clean up the models:" + "Lastly, run this code to clean up all generated files and directories:" ] }, { @@ -361,23 +1119,22 @@ "metadata": { "pycharm": { "name": "#%%\n" + }, + "ExecuteTime": { + "end_time": "2026-02-10T14:00:28.409998Z", + "start_time": "2026-02-10T13:57:21.679146Z" } }, - "source": [ - "import shutil\n", - "\n", - "\n", - "shutil.rmtree(ARTIFACT_PATH)" - ], + "source": "import shutil\n\n# Clean up the temporary artifact directory:\nif os.path.exists(ARTIFACT_PATH):\n shutil.rmtree(ARTIFACT_PATH)", "outputs": [], "execution_count": null } ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "mlrun_functions", "language": "python", - "name": "python3" + "name": "mlrun_functions" }, "language_info": { "codemirror_mode": { @@ -394,4 +1151,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/functions/src/onnx_utils/test_onnx_utils.py b/functions/src/onnx_utils/test_onnx_utils.py index 36113ce1d..08a536128 100644 --- a/functions/src/onnx_utils/test_onnx_utils.py +++ b/functions/src/onnx_utils/test_onnx_utils.py @@ -13,8 +13,6 @@ # limitations under the License. # import os -import shutil -import tempfile import mlrun import pytest @@ -34,6 +32,8 @@ REQUIRED_ENV_VARS = [ "MLRUN_DBPATH", "MLRUN_ARTIFACT_PATH", + "V3IO_USERNAME", + "V3IO_ACCESS_KEY", ] @@ -56,47 +56,6 @@ def _is_tf2onnx_available() -> bool: return False -def _setup_environment() -> str: - """ - Setup the test environment, creating the artifacts path of the test. - - :returns: The temporary directory created for the test artifacts path. - """ - artifact_path = tempfile.TemporaryDirectory().name - os.makedirs(artifact_path) - return artifact_path - - -def _cleanup_environment(artifact_path: str): - """ - Cleanup the test environment, deleting files and artifacts created during the test. - - :param artifact_path: The artifact path to delete. - """ - # Clean the local directory: - for test_output in [ - *os.listdir(artifact_path), - "schedules", - "runs", - "artifacts", - "functions", - "model.pt", - "model.zip", - "model_modules_map.json", - "onnx_model.onnx", - "optimized_onnx_model.onnx", - ]: - test_output_path = os.path.abspath(f"./{test_output}") - if os.path.exists(test_output_path): - if os.path.isdir(test_output_path): - shutil.rmtree(test_output_path) - else: - os.remove(test_output_path) - - # Clean the artifacts directory: - shutil.rmtree(artifact_path) - - def _log_tf_keras_model(context: mlrun.MLClientCtx, model_name: str): """ Create and log a tf.keras model - MobileNetV2. @@ -151,16 +110,11 @@ def _log_pytorch_model(context: mlrun.MLClientCtx, model_name: str): condition=not _validate_environment_variables(), reason="Project's environment variables are not set", ) -@pytest.mark.skipif( - condition=not _is_tf2onnx_available(), - reason="tf2onnx is not installed", -) -def test_to_onnx_help(): +def test_to_onnx_help(test_environment): """ Test the 'to_onnx' handler, passing "help" in the 'framework_kwargs'. """ - # Setup the tests environment: - artifact_path = _setup_environment() + artifact_path = test_environment # Create the function: log_model_function = mlrun.code_to_function( @@ -172,13 +126,18 @@ def test_to_onnx_help(): ) # Run the function to log the model: - log_model_run = log_model_function.run( - handler="_log_tf_keras_model", + log_model_function.run( + handler="_log_pytorch_model", output_path=artifact_path, params={"model_name": MODEL_NAME}, local=True, ) + # Get artifact paths - construct from artifact_path and run structure + run_artifact_dir = os.path.join(artifact_path, "log-model--log-pytorch-model", "0") + model_path = os.path.join(run_artifact_dir, "model") + modules_map_path = os.path.join(run_artifact_dir, "model_modules_map.json.json") + # Import the ONNX Utils function: onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) @@ -190,8 +149,12 @@ def test_to_onnx_help(): output_path=artifact_path, params={ # Take the logged model from the previous function. - "model_path": log_model_run.status.artifacts[0]["spec"]["target_path"], - "load_model_kwargs": {"model_name": MODEL_NAME}, + "model_path": model_path, + "load_model_kwargs": { + "model_name": MODEL_NAME, + "model_class": "mobilenet_v2", + "modules_map": modules_map_path, + }, "framework_kwargs": "help", }, local=True, @@ -202,9 +165,6 @@ def test_to_onnx_help(): ) is_test_passed = False - # Cleanup the tests environment: - _cleanup_environment(artifact_path=artifact_path) - assert is_test_passed @@ -216,12 +176,11 @@ def test_to_onnx_help(): condition=not _is_tf2onnx_available(), reason="tf2onnx is not installed", ) -def test_tf_keras_to_onnx(): +def test_tf_keras_to_onnx(test_environment): """ Test the 'to_onnx' handler, giving it a tf.keras model. """ - # Setup the tests environment: - artifact_path = _setup_environment() + artifact_path = test_environment # Create the function: log_model_function = mlrun.code_to_function( @@ -256,9 +215,6 @@ def test_tf_keras_to_onnx(): local=True, ) - # Cleanup the tests environment: - _cleanup_environment(artifact_path=artifact_path) - # Print the outputs list: print(f"Produced outputs: {onnx_function_run.outputs}") @@ -270,12 +226,11 @@ def test_tf_keras_to_onnx(): condition=not _validate_environment_variables(), reason="Project's environment variables are not set", ) -def test_pytorch_to_onnx(): +def test_pytorch_to_onnx(test_environment): """ Test the 'to_onnx' handler, giving it a pytorch model. """ - # Setup the tests environment: - artifact_path = _setup_environment() + artifact_path = test_environment # Create the function: log_model_function = mlrun.code_to_function( @@ -320,9 +275,6 @@ def test_pytorch_to_onnx(): local=True, ) - # Cleanup the tests environment: - _cleanup_environment(artifact_path=artifact_path) - # Print the outputs list: print(f"Produced outputs: {onnx_function_run.outputs}") @@ -334,12 +286,11 @@ def test_pytorch_to_onnx(): condition=not _validate_environment_variables(), reason="Project's environment variables are not set", ) -def test_optimize_help(): +def test_optimize_help(test_environment): """ Test the 'optimize' handler, passing "help" in the 'optimizations'. """ - # Setup the tests environment: - artifact_path = _setup_environment() + artifact_path = test_environment # Import the ONNX Utils function: onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) @@ -362,9 +313,6 @@ def test_optimize_help(): ) is_test_passed = False - # Cleanup the tests environment: - _cleanup_environment(artifact_path=artifact_path) - assert is_test_passed @@ -372,16 +320,11 @@ def test_optimize_help(): condition=not _validate_environment_variables(), reason="Project's environment variables are not set", ) -@pytest.mark.skipif( - condition=not _is_tf2onnx_available(), - reason="tf2onnx is not installed", -) -def test_optimize(): +def test_optimize(test_environment): """ - Test the 'optimize' handler, giving it a model from the ONNX zoo git repository. + Test the 'optimize' handler, giving it a pytorch model converted to ONNX. """ - # Setup the tests environment: - artifact_path = _setup_environment() + artifact_path = test_environment # Create the function: log_model_function = mlrun.code_to_function( @@ -393,47 +336,58 @@ def test_optimize(): ) # Run the function to log the model: - log_model_run = log_model_function.run( - handler="_log_tf_keras_model", + log_model_function.run( + handler="_log_pytorch_model", output_path=artifact_path, params={"model_name": MODEL_NAME}, local=True, ) + # Get artifact paths - construct from artifact_path and run structure + run_artifact_dir = os.path.join(artifact_path, "log-model--log-pytorch-model", "0") + model_path = os.path.join(run_artifact_dir, "model") + modules_map_path = os.path.join(run_artifact_dir, "model_modules_map.json.json") + # Import the ONNX Utils function: onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) # Run the function to convert our model to ONNX: - to_onnx_function_run = onnx_function.run( + onnx_function.run( handler="to_onnx", output_path=artifact_path, params={ # Take the logged model from the previous function. - "model_path": log_model_run.status.artifacts[0]["spec"]["target_path"], - "load_model_kwargs": {"model_name": MODEL_NAME}, + "model_path": model_path, + "load_model_kwargs": { + "model_name": MODEL_NAME, + "model_class": "mobilenet_v2", + "modules_map": modules_map_path, + }, "onnx_model_name": ONNX_MODEL_NAME, + "framework_kwargs": {"input_signature": [((32, 3, 224, 224), "float32")]}, }, local=True, ) + # Get the ONNX model path from the to_onnx run output + onnx_run_artifact_dir = os.path.join( + artifact_path, "onnx-utils-to-onnx", "0" + ) + onnx_model_path = os.path.join(onnx_run_artifact_dir, "model") + # Run the function to optimize our model: optimize_function_run = onnx_function.run( handler="optimize", output_path=artifact_path, params={ # Take the logged model from the previous function. - "model_path": to_onnx_function_run.status.artifacts[0]["spec"][ - "target_path" - ], + "model_path": onnx_model_path, "handler_init_kwargs": {"model_name": ONNX_MODEL_NAME}, "optimized_model_name": OPTIMIZED_ONNX_MODEL_NAME, }, local=True, ) - # Cleanup the tests environment: - _cleanup_environment(artifact_path=artifact_path) - # Print the outputs list: print(f"Produced outputs: {optimize_function_run.outputs}") From 3de423d26fabeaac1ac5d4b6717010575732992f Mon Sep 17 00:00:00 2001 From: Omer_Mimon Date: Tue, 10 Feb 2026 16:21:57 +0200 Subject: [PATCH 3/4] deleted iguazio credentials --- functions/src/onnx_utils/onnx_utils.ipynb | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/functions/src/onnx_utils/onnx_utils.ipynb b/functions/src/onnx_utils/onnx_utils.ipynb index c0aaa3547..14c810fab 100644 --- a/functions/src/onnx_utils/onnx_utils.ipynb +++ b/functions/src/onnx_utils/onnx_utils.ipynb @@ -93,7 +93,25 @@ "start_time": "2026-02-10T14:13:28.250886Z" } }, - "source": "import os\nimport tempfile\n\n# Set environment variables BEFORE importing mlrun:\nos.environ[\"MLRUN_DBPATH\"] = \"https://mlrun-api.default-tenant.app.innovation-dev.iguazio-cd2.com\"\nos.environ[\"V3IO_USERNAME\"] = \"omerm\"\nos.environ[\"V3IO_ACCESS_KEY\"] = \"618ed0ad-6b68-4be8-8785-b1124437eae2\"\n\n# Use a temporary directory for model artifacts (safe cleanup):\nARTIFACT_PATH = tempfile.mkdtemp()\nos.environ[\"MLRUN_ARTIFACT_PATH\"] = ARTIFACT_PATH\n\n# Project name:\nPROJECT_NAME = \"onnx-utils\"\n\n# Choose our model's name:\nMODEL_NAME = \"mobilenetv2\"\n\n# Choose our ONNX version model's name:\nONNX_MODEL_NAME = \"onnx_mobilenetv2\"\n\n# Choose our optimized ONNX version model's name:\nOPTIMIZED_ONNX_MODEL_NAME = \"optimized_onnx_mobilenetv2\"", + "source": [ + "import os\n", + "import tempfile\n", + "# Use a temporary directory for model artifacts (safe cleanup):\n", + "ARTIFACT_PATH = tempfile.mkdtemp()\n", + "os.environ[\"MLRUN_ARTIFACT_PATH\"] = ARTIFACT_PATH\n", + "\n", + "# Project name:\n", + "PROJECT_NAME = \"onnx-utils\"\n", + "\n", + "# Choose our model's name:\n", + "MODEL_NAME = \"mobilenetv2\"\n", + "\n", + "# Choose our ONNX version model's name:\n", + "ONNX_MODEL_NAME = \"onnx_mobilenetv2\"\n", + "\n", + "# Choose our optimized ONNX version model's name:\n", + "OPTIMIZED_ONNX_MODEL_NAME = \"optimized_onnx_mobilenetv2\"" + ], "outputs": [], "execution_count": 1 }, @@ -1151,4 +1169,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} \ No newline at end of file +} From 29ff211962e8d811d4df377593a759c0f6fb9677 Mon Sep 17 00:00:00 2001 From: Omer_Mimon Date: Wed, 11 Feb 2026 11:21:08 +0200 Subject: [PATCH 4/4] Remove conftest.py and inline fixtures into test_onnx_utils.py Move onnx_project and test_environment fixtures directly into the test file to reduce unnecessary indirection for a single test module. --- functions/src/onnx_utils/conftest.py | 58 --------------------- functions/src/onnx_utils/test_onnx_utils.py | 39 +++++++++++++- 2 files changed, 38 insertions(+), 59 deletions(-) delete mode 100644 functions/src/onnx_utils/conftest.py diff --git a/functions/src/onnx_utils/conftest.py b/functions/src/onnx_utils/conftest.py deleted file mode 100644 index 00740e02e..000000000 --- a/functions/src/onnx_utils/conftest.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2019 Iguazio -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import os -import shutil -import tempfile - -import mlrun -import pytest - -PROJECT_NAME = "onnx-utils" - - -@pytest.fixture(scope="session") -def onnx_project(): - """Create/get the MLRun project once per test session.""" - return mlrun.get_or_create_project(PROJECT_NAME, context="./") - - -@pytest.fixture(autouse=True) -def test_environment(onnx_project): - """Setup and cleanup test artifacts for each test.""" - artifact_path = tempfile.mkdtemp() - yield artifact_path - # Cleanup - only remove files/dirs from the directory containing this test file, - # never from an arbitrary CWD (which could be the project root). - test_dir = os.path.dirname(os.path.abspath(__file__)) - for test_output in [ - "schedules", - "runs", - "artifacts", - "functions", - "model.pt", - "model.zip", - "model_modules_map.json", - "model_modules_map.json.json", - "onnx_model.onnx", - "optimized_onnx_model.onnx", - ]: - test_output_path = os.path.join(test_dir, test_output) - if os.path.exists(test_output_path): - if os.path.isdir(test_output_path): - shutil.rmtree(test_output_path) - else: - os.remove(test_output_path) - if os.path.exists(artifact_path): - shutil.rmtree(artifact_path) diff --git a/functions/src/onnx_utils/test_onnx_utils.py b/functions/src/onnx_utils/test_onnx_utils.py index 08a536128..59c6c2b38 100644 --- a/functions/src/onnx_utils/test_onnx_utils.py +++ b/functions/src/onnx_utils/test_onnx_utils.py @@ -13,11 +13,12 @@ # limitations under the License. # import os +import shutil +import tempfile import mlrun import pytest -# Project name for tests (must match conftest.py) PROJECT_NAME = "onnx-utils" # Choose our model's name: @@ -56,6 +57,42 @@ def _is_tf2onnx_available() -> bool: return False +@pytest.fixture(scope="session") +def onnx_project(): + """Create/get the MLRun project once per test session.""" + return mlrun.get_or_create_project(PROJECT_NAME, context="./") + + +@pytest.fixture(autouse=True) +def test_environment(onnx_project): + """Setup and cleanup test artifacts for each test.""" + artifact_path = tempfile.mkdtemp() + yield artifact_path + # Cleanup - only remove files/dirs from the directory containing this test file, + # never from an arbitrary CWD (which could be the project root). + test_dir = os.path.dirname(os.path.abspath(__file__)) + for test_output in [ + "schedules", + "runs", + "artifacts", + "functions", + "model.pt", + "model.zip", + "model_modules_map.json", + "model_modules_map.json.json", + "onnx_model.onnx", + "optimized_onnx_model.onnx", + ]: + test_output_path = os.path.join(test_dir, test_output) + if os.path.exists(test_output_path): + if os.path.isdir(test_output_path): + shutil.rmtree(test_output_path) + else: + os.remove(test_output_path) + if os.path.exists(artifact_path): + shutil.rmtree(artifact_path) + + def _log_tf_keras_model(context: mlrun.MLClientCtx, model_name: str): """ Create and log a tf.keras model - MobileNetV2.