From 4f07d57b94fde752bdd4da8c0614b251a57cfd3c Mon Sep 17 00:00:00 2001 From: Omer_Mimon Date: Tue, 20 Jan 2026 15:43:57 +0200 Subject: [PATCH] Update onnx_utils dependencies and improve test robustness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Upgrade PyTorch (2.6.0 → 2.8.0) and TorchVision (0.21.0 → 0.23.0) for better compatibility and performance - Update MLRun version requirement to 1.10.0 in item.yaml - Bump function version to 1.4.0 Test improvements: - Add environment variable validation (MLRUN_DBPATH, MLRUN_ARTIFACT_PATH) - Add conditional test skipping based on tf2onnx availability - Fix cleanup function to properly remove test artifacts (model.pt, model_modules_map.json, onnx_model.onnx, etc.) - Update deprecated artifact_path parameter to output_path - Add explicit project context to all MLRun function calls - Fix PyTorch test artifact path construction --- functions/src/onnx_utils/function.yaml | 74 +++++++------- functions/src/onnx_utils/item.yaml | 8 +- functions/src/onnx_utils/requirements.txt | 7 +- functions/src/onnx_utils/test_onnx_utils.py | 108 +++++++++++++++++--- 4 files changed, 135 insertions(+), 62 deletions(-) diff --git a/functions/src/onnx_utils/function.yaml b/functions/src/onnx_utils/function.yaml index 05a0f0bc2..091002cdc 100644 --- a/functions/src/onnx_utils/function.yaml +++ b/functions/src/onnx_utils/function.yaml @@ -1,39 +1,13 @@ -kind: job metadata: + name: onnx-utils + tag: '' categories: - utilities - deep-learning - name: onnx-utils - tag: '' -verbose: false +kind: job spec: - build: - code_origin: '' - base_image: mlrun/mlrun - origin_filename: '' - functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgQ2FsbGFibGUsIERpY3QsIExpc3QsIFR1cGxlCgppbXBvcnQgbWxydW4KCgpjbGFzcyBfVG9PTk5YQ29udmVyc2lvbnM6CiAgICAiIiIKICAgIEFuIE9OTlggY29udmVyc2lvbiBmdW5jdGlvbnMgbGlicmFyeSBjbGFzcy4KICAgICIiIgoKICAgIEBzdGF0aWNtZXRob2QKICAgIGRlZiB0Zl9rZXJhc190b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50XSwgc3RyXV0gPSBOb25lLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBDb252ZXJ0IGEgVEYuS2VyYXMgbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICBBbiBpbml0aWFsaXplZCBURktlcmFzTW9kZWxIYW5kbGVyIHdpdGggYSBsb2FkZWQgbW9kZWwgdG8gY29udmVydCB0byBPTk5YLgogICAgICAgIDpwYXJhbSBvbm54X21vZGVsX25hbWU6IFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbiBgbW9kZWxfbmFtZWAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICBXaGV0aGVyIG9yIG5vdCB0byBvcHRpbWl6ZSB0aGUgT05OWCBtb2RlbCB1c2luZyAnb25ueG9wdGltaXplcicgYmVmb3JlIHNhdmluZyB0aGUgbW9kZWwuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgRGVmYXVsdGVkIHRvIFRydWUuCiAgICAgICAgOnBhcmFtIGlucHV0X3NpZ25hdHVyZTogQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEgbGlzdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHdoZXJlIGVhY2ggZWxlbWVudCBpcyBhbiBpbnB1dCBsYXllciB0dXBsZS4gQW4gaW5wdXQgbGF5ZXIgdHVwbGUgaXMgYSB0dXBsZSBvZjoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBbMF0gPSBMYXllcidzIHNoYXBlLCBhIHR1cGxlIG9mIGludGVnZXJzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBOb25lLCB0aGUgaW5wdXQgc2lnbmF0dXJlIHdpbGwgYmUgdHJpZWQgdG8gYmUgcmVhZCBmcm9tIHRoZSBtb2RlbCBhcnRpZmFjdC4gRGVmYXVsdGVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG8gTm9uZS4KICAgICAgICAiIiIKICAgICAgICAjIEltcG9ydCB0aGUgZnJhbWV3b3JrIGFuZCBoYW5kbGVyOgogICAgICAgIGltcG9ydCB0ZW5zb3JmbG93IGFzIHRmCiAgICAgICAgZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLnRmX2tlcmFzIGltcG9ydCBURktlcmFzVXRpbHMKCiAgICAgICAgIyBDaGVjayB0aGUgZ2l2ZW4gJ2lucHV0X3NpZ25hdHVyZScgcGFyYW1ldGVyOgogICAgICAgIGlmIGlucHV0X3NpZ25hdHVyZSBpcyBOb25lOgogICAgICAgICAgICAjIFJlYWQgdGhlIGlucHV0cyBmcm9tIHRoZSBtb2RlbDoKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgbW9kZWxfaGFuZGxlci5yZWFkX2lucHV0c19mcm9tX21vZGVsKCkKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlcnJvcjoKICAgICAgICAgICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1blJ1bnRpbWVFcnJvcigKICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSBwcm92aWRlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXIuIFRoZSBmdW5jdGlvbiB0cmllZCByZWFkaW5nIHRoZSBpbnB1dCBsYXllcnMgIgogICAgICAgICAgICAgICAgICAgIGYiaW5mb3JtYXRpb24gYXV0b21hdGljYWxseSBidXQgZmFpbGVkIHdpdGggdGhlIGZvbGxvd2luZyBlcnJvcjoge2Vycm9yfSIKICAgICAgICAgICAgICAgICkKICAgICAgICBlbHNlOgogICAgICAgICAgICAjIFBhcnNlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXI6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IFsKICAgICAgICAgICAgICAgIHRmLlRlbnNvclNwZWMoCiAgICAgICAgICAgICAgICAgICAgc2hhcGU9c2hhcGUsCiAgICAgICAgICAgICAgICAgICAgZHR5cGU9VEZLZXJhc1V0aWxzLmNvbnZlcnRfdmFsdWVfdHlwZV90b190Zl9kdHlwZSgKICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICBdCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICkKCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgcHl0b3JjaF90b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50LCAuLi5dLCBzdHJdXSA9IE5vbmUsCiAgICAgICAgaW5wdXRfbGF5ZXJzX25hbWVzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgICAgIG91dHB1dF9sYXllcnNfbmFtZXM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICAgICAgZHluYW1pY19heGVzOiBEaWN0W3N0ciwgRGljdFtpbnQsIHN0cl1dID0gTm9uZSwKICAgICAgICBpc19iYXRjaGVkOiBib29sID0gVHJ1ZSwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCBhIFB5VG9yY2ggbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICAgICAgQW4gaW5pdGlhbGl6ZWQgUHlUb3JjaE1vZGVsSGFuZGxlciB3aXRoIGEgbG9hZGVkIG1vZGVsIHRvIGNvbnZlcnQgdG8gT05OWC4KICAgICAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiAgICAgVGhlIG5hbWUgdG8gdXNlIHRvIGxvZyB0aGUgY29udmVydGVkIE9OTlggbW9kZWwuIElmIG5vdCBnaXZlbiwgdGhlIGdpdmVuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtb2RlbF9uYW1lYCB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgICAgV2hldGhlciBvciBub3QgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsLiBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgICAgICA6cGFyYW0gaW5wdXRfc2lnbmF0dXJlOiAgICAgQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdCB3aGVyZSBlYWNoIGVsZW1lbnQgaXMgYW4gaW5wdXQgbGF5ZXIgdHVwbGUuIEFuIGlucHV0IGxheWVyIHR1cGxlIGlzIGEgdHVwbGUgb2Y6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFswXSA9IExheWVyJ3Mgc2hhcGUsIGEgdHVwbGUgb2YgaW50ZWdlcnMuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgdGhlIGlucHV0IHNpZ25hdHVyZSB3aWxsIGJlIHRyaWVkIHRvIGJlIHJlYWQgZnJvbSB0aGUgbW9kZWwgYXJ0aWZhY3QuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpbnB1dF9sYXllcnNfbmFtZXM6ICBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgaW5wdXQgbm9kZXMgb2YgdGhlIGdyYXBoIGluIG9yZGVyLiBBbGwgb2YgdGhlIG90aGVyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHBhcmFtZXRlcnMgKGlubmVyIGxheWVycykgY2FuIGJlIHNldCBhcyB3ZWxsIGJ5IHBhc3NpbmcgYWRkaXRpb25hbCBuYW1lcyBpbiB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdC4gVGhlIG9yZGVyIGlzIGJ5IHRoZSBvcmRlciBvZiB0aGUgcGFyYW1ldGVycyBpbiB0aGUgbW9kZWwuIElmIE5vbmUsIHRoZSBpbnB1dHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBpbnB1dHMuIElmIGl0cyBhbHNvIE5vbmUsIGl0IGlzIGRlZmF1bHRlZCB0bzoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImlucHV0XzAiLCAiaW5wdXRfMSIsIC4uLgogICAgICAgIDpwYXJhbSBvdXRwdXRfbGF5ZXJzX25hbWVzOiBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgb3V0cHV0IG5vZGVzIG9mIHRoZSBncmFwaCBpbiBvcmRlci4gSWYgTm9uZSwgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG91dHB1dHMgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBvdXRwdXRzLiBJZiBpdHMgYWxzbyBOb25lLCBpdCBpcyBkZWZhdWx0ZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG86ICJvdXRwdXRfMCIgKGZvciBtdWx0aXBsZSBvdXRwdXRzLCB0aGlzIHBhcmFtZXRlciBtdXN0IGJlIHByb3ZpZGVkKS4KICAgICAgICA6cGFyYW0gZHluYW1pY19heGVzOiAgICAgICAgSWYgcGFydCBvZiB0aGUgaW5wdXQgLyBvdXRwdXQgc2hhcGUgaXMgZHluYW1pYywgbGlrZSAoYmF0Y2hfc2l6ZSwgMywgMzIsIDMyKSB5b3UgY2FuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNwZWNpZnkgaXQgYnkgZ2l2aW5nIGEgZHluYW1pYyBheGlzIHRvIHRoZSBpbnB1dCAvIG91dHB1dCBsYXllciBieSBpdHMgbmFtZSBhcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb2xsb3dzOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiaW5wdXQgbGF5ZXIgbmFtZSI6IHswOiAiYmF0Y2hfc2l6ZSJ9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm91dHB1dCBsYXllciBuYW1lIjogezA6ICJiYXRjaF9zaXplIn0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgcHJvdmlkZWQsIHRoZSAnaXNfYmF0Y2hlZCcgZmxhZyB3aWxsIGJlIGlnbm9yZWQuIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpc19iYXRjaGVkOiAgICAgICAgICBXaGV0aGVyIHRvIGluY2x1ZGUgYSBiYXRjaCBzaXplIGFzIHRoZSBmaXJzdCBheGlzIGluIGV2ZXJ5IGlucHV0IGFuZCBvdXRwdXQgbGF5ZXIuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBUcnVlLiBXaWxsIGJlIGlnbm9yZWQgaWYgJ2R5bmFtaWNfYXhlcycgaXMgcHJvdmlkZWQuCiAgICAgICAgIiIiCiAgICAgICAgIyBJbXBvcnQgdGhlIGZyYW1ld29yayBhbmQgaGFuZGxlcjoKICAgICAgICBpbXBvcnQgdG9yY2gKICAgICAgICBmcm9tIG1scnVuLmZyYW1ld29ya3MucHl0b3JjaCBpbXBvcnQgUHlUb3JjaFV0aWxzCgogICAgICAgICMgUGFyc2UgdGhlICdpbnB1dF9zaWduYXR1cmUnIHBhcmFtZXRlcjoKICAgICAgICBpZiBpbnB1dF9zaWduYXR1cmUgaXMgbm90IE5vbmU6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IHR1cGxlKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIHRvcmNoLnplcm9zKAogICAgICAgICAgICAgICAgICAgICAgICBzaXplPXNoYXBlLAogICAgICAgICAgICAgICAgICAgICAgICBkdHlwZT1QeVRvcmNoVXRpbHMuY29udmVydF92YWx1ZV90eXBlX3RvX3RvcmNoX2R0eXBlKAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgICAgICksCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICAgICAgXQogICAgICAgICAgICApCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NhbXBsZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICAgICBpbnB1dF9sYXllcnNfbmFtZXM9aW5wdXRfbGF5ZXJzX25hbWVzLAogICAgICAgICAgICBvdXRwdXRfbGF5ZXJzX25hbWVzPW91dHB1dF9sYXllcnNfbmFtZXMsCiAgICAgICAgICAgIGR5bmFtaWNfYXhlcz1keW5hbWljX2F4ZXMsCiAgICAgICAgICAgIGlzX2JhdGNoZWQ9aXNfYmF0Y2hlZCwKICAgICAgICApCgoKIyBNYXAgZm9yIGdldHRpbmcgdGhlIGNvbnZlcnNpb24gZnVuY3Rpb24gYWNjb3JkaW5nIHRvIHRoZSBwcm92aWRlZCBmcmFtZXdvcms6Cl9DT05WRVJTSU9OX01BUCA9IHsKICAgICJ0ZW5zb3JmbG93LmtlcmFzIjogX1RvT05OWENvbnZlcnNpb25zLnRmX2tlcmFzX3RvX29ubngsCiAgICAidG9yY2giOiBfVG9PTk5YQ29udmVyc2lvbnMucHl0b3JjaF90b19vbm54LAp9ICAjIHR5cGU6IERpY3Rbc3RyLCBDYWxsYWJsZV0KCgpkZWYgdG9fb25ueCgKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgbW9kZWxfcGF0aDogc3RyLAogICAgbG9hZF9tb2RlbF9rd2FyZ3M6IGRpY3QgPSBOb25lLAogICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgb3B0aW1pemVfbW9kZWw6IGJvb2wgPSBUcnVlLAogICAgZnJhbWV3b3JrX2t3YXJnczogRGljdFtzdHIsIEFueV0gPSBOb25lLAopOgogICAgIiIiCiAgICBDb252ZXJ0IHRoZSBnaXZlbiBtb2RlbCB0byBhbiBPTk5YIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgVGhlIE1MUnVuIGZ1bmN0aW9uIGV4ZWN1dGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxfcGF0aDogICAgICAgIFRoZSBtb2RlbCBwYXRoIHN0b3JlIG9iamVjdC4KICAgIDpwYXJhbSBsb2FkX21vZGVsX2t3YXJnczogS2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUgYEF1dG9NTFJ1bi5sb2FkX21vZGVsYCBtZXRob2QuCiAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiAgIFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbiBgbW9kZWxfbmFtZWAgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSB1c2VkIHdpdGggYW4gYWRkaXRpb25hbCBzdWZmaXggYF9vbm54YC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgIFdoZXRoZXIgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlIG1vZGVsLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgIDpwYXJhbSBmcmFtZXdvcmtfa3dhcmdzOiAgQWRkaXRpb25hbCBhcmd1bWVudHMgZWFjaCBmcmFtZXdvcmsgbWF5IHJlcXVpcmUgdG8gY29udmVydCB0byBPTk5YLiBUbyBnZXQgdGhlIGRvYyBzdHJpbmcKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb2YgdGhlIGRlc2lyZWQgZnJhbWV3b3JrIG9ubnggY29udmVyc2lvbiBmdW5jdGlvbiwgcGFzcyAiaGVscCIuCiAgICAiIiIKICAgIGZyb20gbWxydW4uZnJhbWV3b3Jrcy5hdXRvX21scnVuLmF1dG9fbWxydW4gaW1wb3J0IEF1dG9NTFJ1bgoKICAgICMgR2V0IGEgbW9kZWwgaGFuZGxlciBvZiB0aGUgcmVxdWlyZWQgZnJhbWV3b3JrOgogICAgbG9hZF9tb2RlbF9rd2FyZ3MgPSBsb2FkX21vZGVsX2t3YXJncyBvciB7fQogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWxfcGF0aCwgY29udGV4dD1jb250ZXh0LCAqKmxvYWRfbW9kZWxfa3dhcmdzCiAgICApCgogICAgIyBHZXQgdGhlIG1vZGVsJ3MgZnJhbWV3b3JrOgogICAgZnJhbWV3b3JrID0gbW9kZWxfaGFuZGxlci5GUkFNRVdPUktfTkFNRQoKICAgICMgVXNlIHRoZSBjb252ZXJzaW9uIG1hcCB0byBnZXQgdGhlIHNwZWNpZmljIGZyYW1ld29yayB0byBvbm54IGNvbnZlcnNpb246CiAgICBpZiBmcmFtZXdvcmsgbm90IGluIF9DT05WRVJTSU9OX01BUDoKICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgZiJUaGUgZm9sbG93aW5nIGZyYW1ld29yazogJ3tmcmFtZXdvcmt9JywgaGFzIG5vIE9OTlggY29udmVyc2lvbi4iCiAgICAgICAgKQogICAgY29udmVyc2lvbl9mdW5jdGlvbiA9IF9DT05WRVJTSU9OX01BUFtmcmFtZXdvcmtdCgogICAgIyBDaGVjayBpZiBuZWVkZWQgdG8gcHJpbnQgdGhlIGZ1bmN0aW9uJ3MgZG9jIHN0cmluZyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzID09ICJoZWxwIjoKICAgICAgICBwcmludChjb252ZXJzaW9uX2Z1bmN0aW9uLl9fZG9jX18pCiAgICAgICAgcmV0dXJuCgogICAgIyBTZXQgdGhlIGRlZmF1bHQgZW1wdHkgZnJhbWV3b3JrIGt3YXJncyBpZiBuZWVkZWQ6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzIGlzIE5vbmU6CiAgICAgICAgZnJhbWV3b3JrX2t3YXJncyA9IHt9CgogICAgIyBSdW4gdGhlIGNvbnZlcnNpb246CiAgICB0cnk6CiAgICAgICAgY29udmVyc2lvbl9mdW5jdGlvbigKICAgICAgICAgICAgbW9kZWxfaGFuZGxlcj1tb2RlbF9oYW5kbGVyLAogICAgICAgICAgICBvbm54X21vZGVsX25hbWU9b25ueF9tb2RlbF9uYW1lLAogICAgICAgICAgICBvcHRpbWl6ZV9tb2RlbD1vcHRpbWl6ZV9tb2RlbCwKICAgICAgICAgICAgKipmcmFtZXdvcmtfa3dhcmdzLAogICAgICAgICkKICAgIGV4Y2VwdCBUeXBlRXJyb3IgYXMgZXhjZXB0aW9uOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIkVSUk9SOiBBIFR5cGVFcnJvciBleGNlcHRpb24gd2FzIHJhaXNlZCBkdXJpbmcgdGhlIGNvbnZlcnNpb246XG57ZXhjZXB0aW9ufS4gIgogICAgICAgICAgICBmIlBsZWFzZSByZWFkIHRoZSB7ZnJhbWV3b3JrfSBmcmFtZXdvcmsgY29udmVyc2lvbiBmdW5jdGlvbiBkb2Mgc3RyaW5nIGJ5IHBhc3NpbmcgJ2hlbHAnIGluIHRoZSAiCiAgICAgICAgICAgIGYiJ2ZyYW1ld29ya19rd2FyZ3MnIGRpY3Rpb25hcnkgcGFyYW1ldGVyLiIKICAgICAgICApCgoKZGVmIG9wdGltaXplKAogICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICBtb2RlbF9wYXRoOiBzdHIsCiAgICBoYW5kbGVyX2luaXRfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIG9wdGltaXphdGlvbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBmaXhlZF9wb2ludDogYm9vbCA9IEZhbHNlLAogICAgb3B0aW1pemVkX21vZGVsX25hbWU6IHN0ciA9IE5vbmUsCik6CiAgICAiIiIKICAgIE9wdGltaXplIHRoZSBnaXZlbiBPTk5YIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgVGhlIE1MUnVuIGZ1bmN0aW9uIGV4ZWN1dGlvbiBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsX3BhdGg6ICAgICAgICAgICBQYXRoIHRvIHRoZSBPTk5YIG1vZGVsIG9iamVjdC4KICAgIDpwYXJhbSBoYW5kbGVyX2luaXRfa3dhcmdzOiAgS2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUgYE9OTlhNb2RlbEhhbmRsZXJgIGluaXQgbWV0aG9kIHByZWxvYWRpbmcuCiAgICA6cGFyYW0gb3B0aW1pemF0aW9uczogICAgICAgIExpc3Qgb2YgcG9zc2libGUgb3B0aW1pemF0aW9ucy4gVG8gc2VlIHdoYXQgb3B0aW1pemF0aW9ucyBhcmUgYXZhaWxhYmxlLCBwYXNzICJoZWxwIi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgYWxsIHRoZSBvcHRpbWl6YXRpb25zIHdpbGwgYmUgdXNlZC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gZml4ZWRfcG9pbnQ6ICAgICAgICAgIE9wdGltaXplIHRoZSB3ZWlnaHRzIHVzaW5nIGZpeGVkIHBvaW50LiBEZWZhdWx0ZWQgdG8gRmFsc2UuCiAgICA6cGFyYW0gb3B0aW1pemVkX21vZGVsX25hbWU6IFRoZSBuYW1lIG9mIHRoZSBvcHRpbWl6ZWQgbW9kZWwuIElmIE5vbmUsIHRoZSBvcmlnaW5hbCBtb2RlbCB3aWxsIGJlIG92ZXJyaWRkZW4uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgIiIiCiAgICAjIEltcG9ydCB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIGltcG9ydCBvbm54b3B0aW1pemVyCiAgICBmcm9tIG1scnVuLmZyYW1ld29ya3Mub25ueCBpbXBvcnQgT05OWE1vZGVsSGFuZGxlcgoKICAgICMgQ2hlY2sgaWYgbmVlZGVkIHRvIHByaW50IHRoZSBhdmFpbGFibGUgb3B0aW1pemF0aW9ucyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBvcHRpbWl6YXRpb25zID09ICJoZWxwIjoKICAgICAgICBhdmFpbGFibGVfcGFzc2VzID0gIlxuKiAiLmpvaW4ob25ueG9wdGltaXplci5nZXRfYXZhaWxhYmxlX3Bhc3NlcygpKQogICAgICAgIHByaW50KGYiVGhlIGF2YWlsYWJsZSBvcHRpbWl6YXRpb25zIGFyZTpcbioge2F2YWlsYWJsZV9wYXNzZXN9IikKICAgICAgICByZXR1cm4KCiAgICAjIENyZWF0ZSB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIGhhbmRsZXJfaW5pdF9rd2FyZ3MgPSBoYW5kbGVyX2luaXRfa3dhcmdzIG9yIHt9CiAgICBtb2RlbF9oYW5kbGVyID0gT05OWE1vZGVsSGFuZGxlcigKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsX3BhdGgsIGNvbnRleHQ9Y29udGV4dCwgKipoYW5kbGVyX2luaXRfa3dhcmdzCiAgICApCgogICAgIyBMb2FkIHRoZSBPTk5YIG1vZGVsOgogICAgbW9kZWxfaGFuZGxlci5sb2FkKCkKCiAgICAjIE9wdGltaXplIHRoZSBtb2RlbCB1c2luZyB0aGUgZ2l2ZW4gY29uZmlndXJhdGlvbnM6CiAgICBtb2RlbF9oYW5kbGVyLm9wdGltaXplKG9wdGltaXphdGlvbnM9b3B0aW1pemF0aW9ucywgZml4ZWRfcG9pbnQ9Zml4ZWRfcG9pbnQpCgogICAgIyBSZW5hbWUgaWYgbmVlZGVkOgogICAgaWYgb3B0aW1pemVkX21vZGVsX25hbWUgaXMgbm90IE5vbmU6CiAgICAgICAgbW9kZWxfaGFuZGxlci5zZXRfbW9kZWxfbmFtZShtb2RlbF9uYW1lPW9wdGltaXplZF9tb2RlbF9uYW1lKQoKICAgICMgTG9nIHRoZSBvcHRpbWl6ZWQgbW9kZWw6CiAgICBtb2RlbF9oYW5kbGVyLmxvZygpCg== - requirements: - - tqdm~=4.67.1 - - tensorflow~=2.19.0 - - tf_keras~=2.19.0 - - torch~=2.6.0 - - torchvision~=0.21.0 - - onnx~=1.17.0 - - onnxruntime~=1.19.2 - - onnxoptimizer~=0.3.13 - - onnxmltools~=1.13.0 - - tf2onnx~=1.16.1 - - plotly~=5.23 - with_mlrun: false - auto_build: true - disable_auto_mount: false - description: ONNX intigration in MLRun, some utils functions for the ONNX framework, - optimizing and converting models from different framework to ONNX using MLRun. - image: '' entry_points: tf_keras_to_onnx: - doc: Convert a TF.Keras model to an ONNX model and log it back to MLRun as a - new model object. name: tf_keras_to_onnx parameters: - name: model_handler @@ -58,12 +32,12 @@ spec: data type, a mlrun.data_types.ValueType string. If None, the input signature will be tried to be read from the model artifact. Defaulted to None.' default: null + doc: Convert a TF.Keras model to an ONNX model and log it back to MLRun as a + new model object. + lineno: 26 has_varargs: false has_kwargs: false - lineno: 26 pytorch_to_onnx: - doc: Convert a PyTorch model to an ONNX model and log it back to MLRun as a - new model object. name: pytorch_to_onnx parameters: - name: model_handler @@ -116,11 +90,12 @@ spec: doc: Whether to include a batch size as the first axis in every input and output layer. Defaulted to True. Will be ignored if 'dynamic_axes' is provided. default: true + doc: Convert a PyTorch model to an ONNX model and log it back to MLRun as a + new model object. + lineno: 81 has_varargs: false has_kwargs: false - lineno: 81 to_onnx: - doc: Convert the given model to an ONNX model. name: to_onnx parameters: - name: context @@ -150,11 +125,11 @@ spec: get the doc string of the desired framework onnx conversion function, pass "help". default: null + doc: Convert the given model to an ONNX model. + lineno: 160 has_varargs: false has_kwargs: false - lineno: 160 optimize: - doc: Optimize the given ONNX model. name: optimize parameters: - name: context @@ -181,9 +156,34 @@ spec: doc: The name of the optimized model. If None, the original model will be overridden. Defaulted to None. default: null + doc: Optimize the given ONNX model. + lineno: 224 has_varargs: false has_kwargs: false - lineno: 224 + image: '' default_handler: to_onnx allow_empty_resources: true command: '' + disable_auto_mount: false + description: ONNX intigration in MLRun, some utils functions for the ONNX framework, + optimizing and converting models from different framework to ONNX using MLRun. + build: + functionSourceCode: IyBDb3B5cmlnaHQgMjAxOSBJZ3VhemlvCiMKIyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKIyB5b3UgbWF5IG5vdCB1c2UgdGhpcyBmaWxlIGV4Y2VwdCBpbiBjb21wbGlhbmNlIHdpdGggdGhlIExpY2Vuc2UuCiMgWW91IG1heSBvYnRhaW4gYSBjb3B5IG9mIHRoZSBMaWNlbnNlIGF0CiMKIyAgICAgaHR0cDovL3d3dy5hcGFjaGUub3JnL2xpY2Vuc2VzL0xJQ0VOU0UtMi4wCiMKIyBVbmxlc3MgcmVxdWlyZWQgYnkgYXBwbGljYWJsZSBsYXcgb3IgYWdyZWVkIHRvIGluIHdyaXRpbmcsIHNvZnR3YXJlCiMgZGlzdHJpYnV0ZWQgdW5kZXIgdGhlIExpY2Vuc2UgaXMgZGlzdHJpYnV0ZWQgb24gYW4gIkFTIElTIiBCQVNJUywKIyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KIyBTZWUgdGhlIExpY2Vuc2UgZm9yIHRoZSBzcGVjaWZpYyBsYW5ndWFnZSBnb3Zlcm5pbmcgcGVybWlzc2lvbnMgYW5kCiMgbGltaXRhdGlvbnMgdW5kZXIgdGhlIExpY2Vuc2UuCiMKZnJvbSB0eXBpbmcgaW1wb3J0IEFueSwgQ2FsbGFibGUsIERpY3QsIExpc3QsIFR1cGxlCgppbXBvcnQgbWxydW4KCgpjbGFzcyBfVG9PTk5YQ29udmVyc2lvbnM6CiAgICAiIiIKICAgIEFuIE9OTlggY29udmVyc2lvbiBmdW5jdGlvbnMgbGlicmFyeSBjbGFzcy4KICAgICIiIgoKICAgIEBzdGF0aWNtZXRob2QKICAgIGRlZiB0Zl9rZXJhc190b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50XSwgc3RyXV0gPSBOb25lLAogICAgKToKICAgICAgICAiIiIKICAgICAgICBDb252ZXJ0IGEgVEYuS2VyYXMgbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICBBbiBpbml0aWFsaXplZCBURktlcmFzTW9kZWxIYW5kbGVyIHdpdGggYSBsb2FkZWQgbW9kZWwgdG8gY29udmVydCB0byBPTk5YLgogICAgICAgIDpwYXJhbSBvbm54X21vZGVsX25hbWU6IFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbiBgbW9kZWxfbmFtZWAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICBXaGV0aGVyIG9yIG5vdCB0byBvcHRpbWl6ZSB0aGUgT05OWCBtb2RlbCB1c2luZyAnb25ueG9wdGltaXplcicgYmVmb3JlIHNhdmluZyB0aGUgbW9kZWwuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgRGVmYXVsdGVkIHRvIFRydWUuCiAgICAgICAgOnBhcmFtIGlucHV0X3NpZ25hdHVyZTogQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEgbGlzdAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHdoZXJlIGVhY2ggZWxlbWVudCBpcyBhbiBpbnB1dCBsYXllciB0dXBsZS4gQW4gaW5wdXQgbGF5ZXIgdHVwbGUgaXMgYSB0dXBsZSBvZjoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBbMF0gPSBMYXllcidzIHNoYXBlLCBhIHR1cGxlIG9mIGludGVnZXJzLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBJZiBOb25lLCB0aGUgaW5wdXQgc2lnbmF0dXJlIHdpbGwgYmUgdHJpZWQgdG8gYmUgcmVhZCBmcm9tIHRoZSBtb2RlbCBhcnRpZmFjdC4gRGVmYXVsdGVkCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG8gTm9uZS4KICAgICAgICAiIiIKICAgICAgICAjIEltcG9ydCB0aGUgZnJhbWV3b3JrIGFuZCBoYW5kbGVyOgogICAgICAgIGltcG9ydCB0ZW5zb3JmbG93IGFzIHRmCiAgICAgICAgZnJvbSBtbHJ1bi5mcmFtZXdvcmtzLnRmX2tlcmFzIGltcG9ydCBURktlcmFzVXRpbHMKCiAgICAgICAgIyBDaGVjayB0aGUgZ2l2ZW4gJ2lucHV0X3NpZ25hdHVyZScgcGFyYW1ldGVyOgogICAgICAgIGlmIGlucHV0X3NpZ25hdHVyZSBpcyBOb25lOgogICAgICAgICAgICAjIFJlYWQgdGhlIGlucHV0cyBmcm9tIHRoZSBtb2RlbDoKICAgICAgICAgICAgdHJ5OgogICAgICAgICAgICAgICAgbW9kZWxfaGFuZGxlci5yZWFkX2lucHV0c19mcm9tX21vZGVsKCkKICAgICAgICAgICAgZXhjZXB0IEV4Y2VwdGlvbiBhcyBlcnJvcjoKICAgICAgICAgICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1blJ1bnRpbWVFcnJvcigKICAgICAgICAgICAgICAgICAgICBmIlBsZWFzZSBwcm92aWRlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXIuIFRoZSBmdW5jdGlvbiB0cmllZCByZWFkaW5nIHRoZSBpbnB1dCBsYXllcnMgIgogICAgICAgICAgICAgICAgICAgIGYiaW5mb3JtYXRpb24gYXV0b21hdGljYWxseSBidXQgZmFpbGVkIHdpdGggdGhlIGZvbGxvd2luZyBlcnJvcjoge2Vycm9yfSIKICAgICAgICAgICAgICAgICkKICAgICAgICBlbHNlOgogICAgICAgICAgICAjIFBhcnNlIHRoZSAnaW5wdXRfc2lnbmF0dXJlJyBwYXJhbWV0ZXI6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IFsKICAgICAgICAgICAgICAgIHRmLlRlbnNvclNwZWMoCiAgICAgICAgICAgICAgICAgICAgc2hhcGU9c2hhcGUsCiAgICAgICAgICAgICAgICAgICAgZHR5cGU9VEZLZXJhc1V0aWxzLmNvbnZlcnRfdmFsdWVfdHlwZV90b190Zl9kdHlwZSgKICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgKSwKICAgICAgICAgICAgICAgICkKICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICBdCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICkKCiAgICBAc3RhdGljbWV0aG9kCiAgICBkZWYgcHl0b3JjaF90b19vbm54KAogICAgICAgIG1vZGVsX2hhbmRsZXIsCiAgICAgICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgICAgIG9wdGltaXplX21vZGVsOiBib29sID0gVHJ1ZSwKICAgICAgICBpbnB1dF9zaWduYXR1cmU6IExpc3RbVHVwbGVbVHVwbGVbaW50LCAuLi5dLCBzdHJdXSA9IE5vbmUsCiAgICAgICAgaW5wdXRfbGF5ZXJzX25hbWVzOiBMaXN0W3N0cl0gPSBOb25lLAogICAgICAgIG91dHB1dF9sYXllcnNfbmFtZXM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICAgICAgZHluYW1pY19heGVzOiBEaWN0W3N0ciwgRGljdFtpbnQsIHN0cl1dID0gTm9uZSwKICAgICAgICBpc19iYXRjaGVkOiBib29sID0gVHJ1ZSwKICAgICk6CiAgICAgICAgIiIiCiAgICAgICAgQ29udmVydCBhIFB5VG9yY2ggbW9kZWwgdG8gYW4gT05OWCBtb2RlbCBhbmQgbG9nIGl0IGJhY2sgdG8gTUxSdW4gYXMgYSBuZXcgbW9kZWwgb2JqZWN0LgoKICAgICAgICA6cGFyYW0gbW9kZWxfaGFuZGxlcjogICAgICAgQW4gaW5pdGlhbGl6ZWQgUHlUb3JjaE1vZGVsSGFuZGxlciB3aXRoIGEgbG9hZGVkIG1vZGVsIHRvIGNvbnZlcnQgdG8gT05OWC4KICAgICAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiAgICAgVGhlIG5hbWUgdG8gdXNlIHRvIGxvZyB0aGUgY29udmVydGVkIE9OTlggbW9kZWwuIElmIG5vdCBnaXZlbiwgdGhlIGdpdmVuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIGBtb2RlbF9uYW1lYCB3aWxsIGJlIHVzZWQgd2l0aCBhbiBhZGRpdGlvbmFsIHN1ZmZpeCBgX29ubnhgLiBEZWZhdWx0ZWQgdG8gTm9uZS4KICAgICAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgICAgV2hldGhlciBvciBub3QgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG1vZGVsLiBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgICAgICA6cGFyYW0gaW5wdXRfc2lnbmF0dXJlOiAgICAgQSBsaXN0IG9mIHRoZSBpbnB1dCBsYXllcnMgc2hhcGUgYW5kIGRhdGEgdHlwZSBwcm9wZXJ0aWVzLiBFeHBlY3RlZCB0byByZWNlaXZlIGEKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdCB3aGVyZSBlYWNoIGVsZW1lbnQgaXMgYW4gaW5wdXQgbGF5ZXIgdHVwbGUuIEFuIGlucHV0IGxheWVyIHR1cGxlIGlzIGEgdHVwbGUgb2Y6CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFswXSA9IExheWVyJ3Mgc2hhcGUsIGEgdHVwbGUgb2YgaW50ZWdlcnMuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIFsxXSA9IExheWVyJ3MgZGF0YSB0eXBlLCBhIG1scnVuLmRhdGFfdHlwZXMuVmFsdWVUeXBlIHN0cmluZy4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgdGhlIGlucHV0IHNpZ25hdHVyZSB3aWxsIGJlIHRyaWVkIHRvIGJlIHJlYWQgZnJvbSB0aGUgbW9kZWwgYXJ0aWZhY3QuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpbnB1dF9sYXllcnNfbmFtZXM6ICBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgaW5wdXQgbm9kZXMgb2YgdGhlIGdyYXBoIGluIG9yZGVyLiBBbGwgb2YgdGhlIG90aGVyCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHBhcmFtZXRlcnMgKGlubmVyIGxheWVycykgY2FuIGJlIHNldCBhcyB3ZWxsIGJ5IHBhc3NpbmcgYWRkaXRpb25hbCBuYW1lcyBpbiB0aGUKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgbGlzdC4gVGhlIG9yZGVyIGlzIGJ5IHRoZSBvcmRlciBvZiB0aGUgcGFyYW1ldGVycyBpbiB0aGUgbW9kZWwuIElmIE5vbmUsIHRoZSBpbnB1dHMKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBpbnB1dHMuIElmIGl0cyBhbHNvIE5vbmUsIGl0IGlzIGRlZmF1bHRlZCB0bzoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgImlucHV0XzAiLCAiaW5wdXRfMSIsIC4uLgogICAgICAgIDpwYXJhbSBvdXRwdXRfbGF5ZXJzX25hbWVzOiBMaXN0IG9mIG5hbWVzIHRvIGFzc2lnbiB0byB0aGUgb3V0cHV0IG5vZGVzIG9mIHRoZSBncmFwaCBpbiBvcmRlci4gSWYgTm9uZSwgdGhlCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIG91dHB1dHMgd2lsbCBiZSByZWFkIGZyb20gdGhlIGhhbmRsZXIncyBvdXRwdXRzLiBJZiBpdHMgYWxzbyBOb25lLCBpdCBpcyBkZWZhdWx0ZWQKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgdG86ICJvdXRwdXRfMCIgKGZvciBtdWx0aXBsZSBvdXRwdXRzLCB0aGlzIHBhcmFtZXRlciBtdXN0IGJlIHByb3ZpZGVkKS4KICAgICAgICA6cGFyYW0gZHluYW1pY19heGVzOiAgICAgICAgSWYgcGFydCBvZiB0aGUgaW5wdXQgLyBvdXRwdXQgc2hhcGUgaXMgZHluYW1pYywgbGlrZSAoYmF0Y2hfc2l6ZSwgMywgMzIsIDMyKSB5b3UgY2FuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIHNwZWNpZnkgaXQgYnkgZ2l2aW5nIGEgZHluYW1pYyBheGlzIHRvIHRoZSBpbnB1dCAvIG91dHB1dCBsYXllciBieSBpdHMgbmFtZSBhcwogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICBmb2xsb3dzOiB7CiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAiaW5wdXQgbGF5ZXIgbmFtZSI6IHswOiAiYmF0Y2hfc2l6ZSJ9LAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIm91dHB1dCBsYXllciBuYW1lIjogezA6ICJiYXRjaF9zaXplIn0sCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIH0KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgcHJvdmlkZWQsIHRoZSAnaXNfYmF0Y2hlZCcgZmxhZyB3aWxsIGJlIGlnbm9yZWQuIERlZmF1bHRlZCB0byBOb25lLgogICAgICAgIDpwYXJhbSBpc19iYXRjaGVkOiAgICAgICAgICBXaGV0aGVyIHRvIGluY2x1ZGUgYSBiYXRjaCBzaXplIGFzIHRoZSBmaXJzdCBheGlzIGluIGV2ZXJ5IGlucHV0IGFuZCBvdXRwdXQgbGF5ZXIuCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBUcnVlLiBXaWxsIGJlIGlnbm9yZWQgaWYgJ2R5bmFtaWNfYXhlcycgaXMgcHJvdmlkZWQuCiAgICAgICAgIiIiCiAgICAgICAgIyBJbXBvcnQgdGhlIGZyYW1ld29yayBhbmQgaGFuZGxlcjoKICAgICAgICBpbXBvcnQgdG9yY2gKICAgICAgICBmcm9tIG1scnVuLmZyYW1ld29ya3MucHl0b3JjaCBpbXBvcnQgUHlUb3JjaFV0aWxzCgogICAgICAgICMgUGFyc2UgdGhlICdpbnB1dF9zaWduYXR1cmUnIHBhcmFtZXRlcjoKICAgICAgICBpZiBpbnB1dF9zaWduYXR1cmUgaXMgbm90IE5vbmU6CiAgICAgICAgICAgIGlucHV0X3NpZ25hdHVyZSA9IHR1cGxlKAogICAgICAgICAgICAgICAgWwogICAgICAgICAgICAgICAgICAgIHRvcmNoLnplcm9zKAogICAgICAgICAgICAgICAgICAgICAgICBzaXplPXNoYXBlLAogICAgICAgICAgICAgICAgICAgICAgICBkdHlwZT1QeVRvcmNoVXRpbHMuY29udmVydF92YWx1ZV90eXBlX3RvX3RvcmNoX2R0eXBlKAogICAgICAgICAgICAgICAgICAgICAgICAgICAgdmFsdWVfdHlwZT12YWx1ZV90eXBlCiAgICAgICAgICAgICAgICAgICAgICAgICksCiAgICAgICAgICAgICAgICAgICAgKQogICAgICAgICAgICAgICAgICAgIGZvciAoc2hhcGUsIHZhbHVlX3R5cGUpIGluIGlucHV0X3NpZ25hdHVyZQogICAgICAgICAgICAgICAgXQogICAgICAgICAgICApCgogICAgICAgICMgQ29udmVydCB0byBPTk5YOgogICAgICAgIG1vZGVsX2hhbmRsZXIudG9fb25ueCgKICAgICAgICAgICAgbW9kZWxfbmFtZT1vbm54X21vZGVsX25hbWUsCiAgICAgICAgICAgIGlucHV0X3NhbXBsZT1pbnB1dF9zaWduYXR1cmUsCiAgICAgICAgICAgIG9wdGltaXplPW9wdGltaXplX21vZGVsLAogICAgICAgICAgICBpbnB1dF9sYXllcnNfbmFtZXM9aW5wdXRfbGF5ZXJzX25hbWVzLAogICAgICAgICAgICBvdXRwdXRfbGF5ZXJzX25hbWVzPW91dHB1dF9sYXllcnNfbmFtZXMsCiAgICAgICAgICAgIGR5bmFtaWNfYXhlcz1keW5hbWljX2F4ZXMsCiAgICAgICAgICAgIGlzX2JhdGNoZWQ9aXNfYmF0Y2hlZCwKICAgICAgICApCgoKIyBNYXAgZm9yIGdldHRpbmcgdGhlIGNvbnZlcnNpb24gZnVuY3Rpb24gYWNjb3JkaW5nIHRvIHRoZSBwcm92aWRlZCBmcmFtZXdvcms6Cl9DT05WRVJTSU9OX01BUCA9IHsKICAgICJ0ZW5zb3JmbG93LmtlcmFzIjogX1RvT05OWENvbnZlcnNpb25zLnRmX2tlcmFzX3RvX29ubngsCiAgICAidG9yY2giOiBfVG9PTk5YQ29udmVyc2lvbnMucHl0b3JjaF90b19vbm54LAp9ICAjIHR5cGU6IERpY3Rbc3RyLCBDYWxsYWJsZV0KCgpkZWYgdG9fb25ueCgKICAgIGNvbnRleHQ6IG1scnVuLk1MQ2xpZW50Q3R4LAogICAgbW9kZWxfcGF0aDogc3RyLAogICAgbG9hZF9tb2RlbF9rd2FyZ3M6IGRpY3QgPSBOb25lLAogICAgb25ueF9tb2RlbF9uYW1lOiBzdHIgPSBOb25lLAogICAgb3B0aW1pemVfbW9kZWw6IGJvb2wgPSBUcnVlLAogICAgZnJhbWV3b3JrX2t3YXJnczogRGljdFtzdHIsIEFueV0gPSBOb25lLAopOgogICAgIiIiCiAgICBDb252ZXJ0IHRoZSBnaXZlbiBtb2RlbCB0byBhbiBPTk5YIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgVGhlIE1MUnVuIGZ1bmN0aW9uIGV4ZWN1dGlvbiBjb250ZXh0CiAgICA6cGFyYW0gbW9kZWxfcGF0aDogICAgICAgIFRoZSBtb2RlbCBwYXRoIHN0b3JlIG9iamVjdC4KICAgIDpwYXJhbSBsb2FkX21vZGVsX2t3YXJnczogS2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUgYEF1dG9NTFJ1bi5sb2FkX21vZGVsYCBtZXRob2QuCiAgICA6cGFyYW0gb25ueF9tb2RlbF9uYW1lOiAgIFRoZSBuYW1lIHRvIHVzZSB0byBsb2cgdGhlIGNvbnZlcnRlZCBPTk5YIG1vZGVsLiBJZiBub3QgZ2l2ZW4sIHRoZSBnaXZlbiBgbW9kZWxfbmFtZWAgd2lsbAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBiZSB1c2VkIHdpdGggYW4gYWRkaXRpb25hbCBzdWZmaXggYF9vbm54YC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gb3B0aW1pemVfbW9kZWw6ICAgIFdoZXRoZXIgdG8gb3B0aW1pemUgdGhlIE9OTlggbW9kZWwgdXNpbmcgJ29ubnhvcHRpbWl6ZXInIGJlZm9yZSBzYXZpbmcgdGhlIG1vZGVsLgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICBEZWZhdWx0ZWQgdG8gVHJ1ZS4KICAgIDpwYXJhbSBmcmFtZXdvcmtfa3dhcmdzOiAgQWRkaXRpb25hbCBhcmd1bWVudHMgZWFjaCBmcmFtZXdvcmsgbWF5IHJlcXVpcmUgdG8gY29udmVydCB0byBPTk5YLiBUbyBnZXQgdGhlIGRvYyBzdHJpbmcKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgb2YgdGhlIGRlc2lyZWQgZnJhbWV3b3JrIG9ubnggY29udmVyc2lvbiBmdW5jdGlvbiwgcGFzcyAiaGVscCIuCiAgICAiIiIKICAgIGZyb20gbWxydW4uZnJhbWV3b3Jrcy5hdXRvX21scnVuLmF1dG9fbWxydW4gaW1wb3J0IEF1dG9NTFJ1bgoKICAgICMgR2V0IGEgbW9kZWwgaGFuZGxlciBvZiB0aGUgcmVxdWlyZWQgZnJhbWV3b3JrOgogICAgbG9hZF9tb2RlbF9rd2FyZ3MgPSBsb2FkX21vZGVsX2t3YXJncyBvciB7fQogICAgbW9kZWxfaGFuZGxlciA9IEF1dG9NTFJ1bi5sb2FkX21vZGVsKAogICAgICAgIG1vZGVsX3BhdGg9bW9kZWxfcGF0aCwgY29udGV4dD1jb250ZXh0LCAqKmxvYWRfbW9kZWxfa3dhcmdzCiAgICApCgogICAgIyBHZXQgdGhlIG1vZGVsJ3MgZnJhbWV3b3JrOgogICAgZnJhbWV3b3JrID0gbW9kZWxfaGFuZGxlci5GUkFNRVdPUktfTkFNRQoKICAgICMgVXNlIHRoZSBjb252ZXJzaW9uIG1hcCB0byBnZXQgdGhlIHNwZWNpZmljIGZyYW1ld29yayB0byBvbm54IGNvbnZlcnNpb246CiAgICBpZiBmcmFtZXdvcmsgbm90IGluIF9DT05WRVJTSU9OX01BUDoKICAgICAgICByYWlzZSBtbHJ1bi5lcnJvcnMuTUxSdW5JbnZhbGlkQXJndW1lbnRFcnJvcigKICAgICAgICAgICAgZiJUaGUgZm9sbG93aW5nIGZyYW1ld29yazogJ3tmcmFtZXdvcmt9JywgaGFzIG5vIE9OTlggY29udmVyc2lvbi4iCiAgICAgICAgKQogICAgY29udmVyc2lvbl9mdW5jdGlvbiA9IF9DT05WRVJTSU9OX01BUFtmcmFtZXdvcmtdCgogICAgIyBDaGVjayBpZiBuZWVkZWQgdG8gcHJpbnQgdGhlIGZ1bmN0aW9uJ3MgZG9jIHN0cmluZyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzID09ICJoZWxwIjoKICAgICAgICBwcmludChjb252ZXJzaW9uX2Z1bmN0aW9uLl9fZG9jX18pCiAgICAgICAgcmV0dXJuCgogICAgIyBTZXQgdGhlIGRlZmF1bHQgZW1wdHkgZnJhbWV3b3JrIGt3YXJncyBpZiBuZWVkZWQ6CiAgICBpZiBmcmFtZXdvcmtfa3dhcmdzIGlzIE5vbmU6CiAgICAgICAgZnJhbWV3b3JrX2t3YXJncyA9IHt9CgogICAgIyBSdW4gdGhlIGNvbnZlcnNpb246CiAgICB0cnk6CiAgICAgICAgY29udmVyc2lvbl9mdW5jdGlvbigKICAgICAgICAgICAgbW9kZWxfaGFuZGxlcj1tb2RlbF9oYW5kbGVyLAogICAgICAgICAgICBvbm54X21vZGVsX25hbWU9b25ueF9tb2RlbF9uYW1lLAogICAgICAgICAgICBvcHRpbWl6ZV9tb2RlbD1vcHRpbWl6ZV9tb2RlbCwKICAgICAgICAgICAgKipmcmFtZXdvcmtfa3dhcmdzLAogICAgICAgICkKICAgIGV4Y2VwdCBUeXBlRXJyb3IgYXMgZXhjZXB0aW9uOgogICAgICAgIHJhaXNlIG1scnVuLmVycm9ycy5NTFJ1bkludmFsaWRBcmd1bWVudEVycm9yKAogICAgICAgICAgICBmIkVSUk9SOiBBIFR5cGVFcnJvciBleGNlcHRpb24gd2FzIHJhaXNlZCBkdXJpbmcgdGhlIGNvbnZlcnNpb246XG57ZXhjZXB0aW9ufS4gIgogICAgICAgICAgICBmIlBsZWFzZSByZWFkIHRoZSB7ZnJhbWV3b3JrfSBmcmFtZXdvcmsgY29udmVyc2lvbiBmdW5jdGlvbiBkb2Mgc3RyaW5nIGJ5IHBhc3NpbmcgJ2hlbHAnIGluIHRoZSAiCiAgICAgICAgICAgIGYiJ2ZyYW1ld29ya19rd2FyZ3MnIGRpY3Rpb25hcnkgcGFyYW1ldGVyLiIKICAgICAgICApCgoKZGVmIG9wdGltaXplKAogICAgY29udGV4dDogbWxydW4uTUxDbGllbnRDdHgsCiAgICBtb2RlbF9wYXRoOiBzdHIsCiAgICBoYW5kbGVyX2luaXRfa3dhcmdzOiBkaWN0ID0gTm9uZSwKICAgIG9wdGltaXphdGlvbnM6IExpc3Rbc3RyXSA9IE5vbmUsCiAgICBmaXhlZF9wb2ludDogYm9vbCA9IEZhbHNlLAogICAgb3B0aW1pemVkX21vZGVsX25hbWU6IHN0ciA9IE5vbmUsCik6CiAgICAiIiIKICAgIE9wdGltaXplIHRoZSBnaXZlbiBPTk5YIG1vZGVsLgoKICAgIDpwYXJhbSBjb250ZXh0OiAgICAgICAgICAgICAgVGhlIE1MUnVuIGZ1bmN0aW9uIGV4ZWN1dGlvbiBjb250ZXh0LgogICAgOnBhcmFtIG1vZGVsX3BhdGg6ICAgICAgICAgICBQYXRoIHRvIHRoZSBPTk5YIG1vZGVsIG9iamVjdC4KICAgIDpwYXJhbSBoYW5kbGVyX2luaXRfa3dhcmdzOiAgS2V5d29yZCBhcmd1bWVudHMgdG8gcGFzcyB0byB0aGUgYE9OTlhNb2RlbEhhbmRsZXJgIGluaXQgbWV0aG9kIHByZWxvYWRpbmcuCiAgICA6cGFyYW0gb3B0aW1pemF0aW9uczogICAgICAgIExpc3Qgb2YgcG9zc2libGUgb3B0aW1pemF0aW9ucy4gVG8gc2VlIHdoYXQgb3B0aW1pemF0aW9ucyBhcmUgYXZhaWxhYmxlLCBwYXNzICJoZWxwIi4KICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgSWYgTm9uZSwgYWxsIHRoZSBvcHRpbWl6YXRpb25zIHdpbGwgYmUgdXNlZC4gRGVmYXVsdGVkIHRvIE5vbmUuCiAgICA6cGFyYW0gZml4ZWRfcG9pbnQ6ICAgICAgICAgIE9wdGltaXplIHRoZSB3ZWlnaHRzIHVzaW5nIGZpeGVkIHBvaW50LiBEZWZhdWx0ZWQgdG8gRmFsc2UuCiAgICA6cGFyYW0gb3B0aW1pemVkX21vZGVsX25hbWU6IFRoZSBuYW1lIG9mIHRoZSBvcHRpbWl6ZWQgbW9kZWwuIElmIE5vbmUsIHRoZSBvcmlnaW5hbCBtb2RlbCB3aWxsIGJlIG92ZXJyaWRkZW4uCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIERlZmF1bHRlZCB0byBOb25lLgogICAgIiIiCiAgICAjIEltcG9ydCB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIGltcG9ydCBvbm54b3B0aW1pemVyCiAgICBmcm9tIG1scnVuLmZyYW1ld29ya3Mub25ueCBpbXBvcnQgT05OWE1vZGVsSGFuZGxlcgoKICAgICMgQ2hlY2sgaWYgbmVlZGVkIHRvIHByaW50IHRoZSBhdmFpbGFibGUgb3B0aW1pemF0aW9ucyAoImhlbHAiIGlzIHBhc3NlZCk6CiAgICBpZiBvcHRpbWl6YXRpb25zID09ICJoZWxwIjoKICAgICAgICBhdmFpbGFibGVfcGFzc2VzID0gIlxuKiAiLmpvaW4ob25ueG9wdGltaXplci5nZXRfYXZhaWxhYmxlX3Bhc3NlcygpKQogICAgICAgIHByaW50KGYiVGhlIGF2YWlsYWJsZSBvcHRpbWl6YXRpb25zIGFyZTpcbioge2F2YWlsYWJsZV9wYXNzZXN9IikKICAgICAgICByZXR1cm4KCiAgICAjIENyZWF0ZSB0aGUgbW9kZWwgaGFuZGxlcjoKICAgIGhhbmRsZXJfaW5pdF9rd2FyZ3MgPSBoYW5kbGVyX2luaXRfa3dhcmdzIG9yIHt9CiAgICBtb2RlbF9oYW5kbGVyID0gT05OWE1vZGVsSGFuZGxlcigKICAgICAgICBtb2RlbF9wYXRoPW1vZGVsX3BhdGgsIGNvbnRleHQ9Y29udGV4dCwgKipoYW5kbGVyX2luaXRfa3dhcmdzCiAgICApCgogICAgIyBMb2FkIHRoZSBPTk5YIG1vZGVsOgogICAgbW9kZWxfaGFuZGxlci5sb2FkKCkKCiAgICAjIE9wdGltaXplIHRoZSBtb2RlbCB1c2luZyB0aGUgZ2l2ZW4gY29uZmlndXJhdGlvbnM6CiAgICBtb2RlbF9oYW5kbGVyLm9wdGltaXplKG9wdGltaXphdGlvbnM9b3B0aW1pemF0aW9ucywgZml4ZWRfcG9pbnQ9Zml4ZWRfcG9pbnQpCgogICAgIyBSZW5hbWUgaWYgbmVlZGVkOgogICAgaWYgb3B0aW1pemVkX21vZGVsX25hbWUgaXMgbm90IE5vbmU6CiAgICAgICAgbW9kZWxfaGFuZGxlci5zZXRfbW9kZWxfbmFtZShtb2RlbF9uYW1lPW9wdGltaXplZF9tb2RlbF9uYW1lKQoKICAgICMgTG9nIHRoZSBvcHRpbWl6ZWQgbW9kZWw6CiAgICBtb2RlbF9oYW5kbGVyLmxvZygpCg== + base_image: mlrun/mlrun + with_mlrun: false + auto_build: true + requirements: + - tqdm~=4.67.1 + - tensorflow~=2.19.0 + - tf_keras~=2.19.0 + - torch~=2.8.0 + - torchvision~=0.23.0 + - onnx~=1.17.0 + - onnxruntime~=1.19.2 + - onnxoptimizer~=0.3.13 + - onnxmltools~=1.13.0 + - tf2onnx~=1.16.1 + - plotly~=5.23 + origin_filename: '' + code_origin: '' +verbose: false diff --git a/functions/src/onnx_utils/item.yaml b/functions/src/onnx_utils/item.yaml index 803bd2599..5f129389f 100644 --- a/functions/src/onnx_utils/item.yaml +++ b/functions/src/onnx_utils/item.yaml @@ -13,7 +13,7 @@ labels: author: Iguazio maintainers: [] marketplaceType: '' -mlrunVersion: 1.7.2 +mlrunVersion: 1.10.0 name: onnx_utils platformVersion: 3.5.0 spec: @@ -30,8 +30,8 @@ spec: - tqdm~=4.67.1 - tensorflow~=2.19.0 - tf_keras~=2.19.0 - - torch~=2.6.0 - - torchvision~=0.21.0 + - torch~=2.8.0 + - torchvision~=0.23.0 - onnx~=1.17.0 - onnxruntime~=1.19.2 - onnxoptimizer~=0.3.13 @@ -39,4 +39,4 @@ spec: - tf2onnx~=1.16.1 - plotly~=5.23 url: '' -version: 1.3.0 +version: 1.4.0 diff --git a/functions/src/onnx_utils/requirements.txt b/functions/src/onnx_utils/requirements.txt index d3d7dfd68..912b3d7e5 100644 --- a/functions/src/onnx_utils/requirements.txt +++ b/functions/src/onnx_utils/requirements.txt @@ -1,11 +1,10 @@ tqdm~=4.67.1 tensorflow~=2.19.0 tf_keras~=2.19.0 -torch~=2.6.0 -torchvision~=0.21.0 +torch~=2.8 +torchvision~=0.23.0 onnx~=1.17.0 onnxruntime~=1.19.2 onnxoptimizer~=0.3.13 onnxmltools~=1.13.0 -tf2onnx~=1.16.1 -plotly~=5.23 +plotly~=5.23 \ No newline at end of file diff --git a/functions/src/onnx_utils/test_onnx_utils.py b/functions/src/onnx_utils/test_onnx_utils.py index 2e01782f5..36113ce1d 100644 --- a/functions/src/onnx_utils/test_onnx_utils.py +++ b/functions/src/onnx_utils/test_onnx_utils.py @@ -17,6 +17,10 @@ import tempfile import mlrun +import pytest + +# Project name for tests (must match conftest.py) +PROJECT_NAME = "onnx-utils" # Choose our model's name: MODEL_NAME = "model" @@ -27,6 +31,30 @@ # Choose our optimized ONNX version model's name: OPTIMIZED_ONNX_MODEL_NAME = f"optimized_{ONNX_MODEL_NAME}" +REQUIRED_ENV_VARS = [ + "MLRUN_DBPATH", + "MLRUN_ARTIFACT_PATH", +] + + +def _validate_environment_variables() -> bool: + """ + Checks that all required Environment variables are set. + """ + environment_keys = os.environ.keys() + return all(key in environment_keys for key in REQUIRED_ENV_VARS) + + +def _is_tf2onnx_available() -> bool: + """ + Check if tf2onnx is installed (required for TensorFlow/Keras ONNX conversion). + """ + try: + import tf2onnx + return True + except ImportError: + return False + def _setup_environment() -> str: """ @@ -52,6 +80,11 @@ def _cleanup_environment(artifact_path: str): "runs", "artifacts", "functions", + "model.pt", + "model.zip", + "model_modules_map.json", + "onnx_model.onnx", + "optimized_onnx_model.onnx", ]: test_output_path = os.path.abspath(f"./{test_output}") if os.path.exists(test_output_path): @@ -114,6 +147,14 @@ def _log_pytorch_model(context: mlrun.MLClientCtx, model_name: str): model_handler.log() +@pytest.mark.skipif( + condition=not _validate_environment_variables(), + reason="Project's environment variables are not set", +) +@pytest.mark.skipif( + condition=not _is_tf2onnx_available(), + reason="tf2onnx is not installed", +) def test_to_onnx_help(): """ Test the 'to_onnx' handler, passing "help" in the 'framework_kwargs'. @@ -125,6 +166,7 @@ def test_to_onnx_help(): log_model_function = mlrun.code_to_function( filename="test_onnx_utils.py", name="log_model", + project=PROJECT_NAME, kind="job", image="mlrun/ml-models", ) @@ -132,20 +174,20 @@ def test_to_onnx_help(): # Run the function to log the model: log_model_run = log_model_function.run( handler="_log_tf_keras_model", - artifact_path=artifact_path, + output_path=artifact_path, params={"model_name": MODEL_NAME}, local=True, ) # Import the ONNX Utils function: - onnx_function = mlrun.import_function("function.yaml") + onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) # Run the function, passing "help" in 'framework_kwargs' and see that no exception was raised: is_test_passed = True try: onnx_function.run( handler="to_onnx", - artifact_path=artifact_path, + output_path=artifact_path, params={ # Take the logged model from the previous function. "model_path": log_model_run.status.artifacts[0]["spec"]["target_path"], @@ -166,6 +208,14 @@ def test_to_onnx_help(): assert is_test_passed +@pytest.mark.skipif( + condition=not _validate_environment_variables(), + reason="Project's environment variables are not set", +) +@pytest.mark.skipif( + condition=not _is_tf2onnx_available(), + reason="tf2onnx is not installed", +) def test_tf_keras_to_onnx(): """ Test the 'to_onnx' handler, giving it a tf.keras model. @@ -177,6 +227,7 @@ def test_tf_keras_to_onnx(): log_model_function = mlrun.code_to_function( filename="test_onnx_utils.py", name="log_model", + project=PROJECT_NAME, kind="job", image="mlrun/ml-models", ) @@ -184,18 +235,18 @@ def test_tf_keras_to_onnx(): # Run the function to log the model: log_model_run = log_model_function.run( handler="_log_tf_keras_model", - artifact_path=artifact_path, + output_path=artifact_path, params={"model_name": MODEL_NAME}, local=True, ) # Import the ONNX Utils function: - onnx_function = mlrun.import_function("function.yaml") + onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) # Run the function to convert our model to ONNX: onnx_function_run = onnx_function.run( handler="to_onnx", - artifact_path=artifact_path, + output_path=artifact_path, params={ # Take the logged model from the previous function. "model_path": log_model_run.status.artifacts[0]["spec"]["target_path"], @@ -215,6 +266,10 @@ def test_tf_keras_to_onnx(): assert "model" in onnx_function_run.outputs +@pytest.mark.skipif( + condition=not _validate_environment_variables(), + reason="Project's environment variables are not set", +) def test_pytorch_to_onnx(): """ Test the 'to_onnx' handler, giving it a pytorch model. @@ -226,6 +281,7 @@ def test_pytorch_to_onnx(): log_model_function = mlrun.code_to_function( filename="test_onnx_utils.py", name="log_model", + project=PROJECT_NAME, kind="job", image="mlrun/ml-models", ) @@ -233,25 +289,30 @@ def test_pytorch_to_onnx(): # Run the function to log the model: log_model_run = log_model_function.run( handler="_log_pytorch_model", - artifact_path=artifact_path, + output_path=artifact_path, params={"model_name": MODEL_NAME}, local=True, ) # Import the ONNX Utils function: - onnx_function = mlrun.import_function("function.yaml") + onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) + + # Get artifact paths - construct from artifact_path and run structure + run_artifact_dir = os.path.join(artifact_path, "log-model--log-pytorch-model", "0") + model_path = os.path.join(run_artifact_dir, "model") + modules_map_path = os.path.join(run_artifact_dir, "model_modules_map.json.json") # Run the function to convert our model to ONNX: onnx_function_run = onnx_function.run( handler="to_onnx", - artifact_path=artifact_path, + output_path=artifact_path, params={ # Take the logged model from the previous function. - "model_path": log_model_run.status.artifacts[1]["spec"]["target_path"], + "model_path": model_path, "load_model_kwargs": { "model_name": MODEL_NAME, "model_class": "mobilenet_v2", - "modules_map": log_model_run.status.artifacts[0]["spec"]["target_path"], + "modules_map": modules_map_path, }, "onnx_model_name": ONNX_MODEL_NAME, "framework_kwargs": {"input_signature": [((32, 3, 224, 224), "float32")]}, @@ -269,6 +330,10 @@ def test_pytorch_to_onnx(): assert "model" in onnx_function_run.outputs +@pytest.mark.skipif( + condition=not _validate_environment_variables(), + reason="Project's environment variables are not set", +) def test_optimize_help(): """ Test the 'optimize' handler, passing "help" in the 'optimizations'. @@ -277,14 +342,14 @@ def test_optimize_help(): artifact_path = _setup_environment() # Import the ONNX Utils function: - onnx_function = mlrun.import_function("function.yaml") + onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) # Run the function, passing "help" in 'optimizations' and see that no exception was raised: is_test_passed = True try: onnx_function.run( handler="optimize", - artifact_path=artifact_path, + output_path=artifact_path, params={ "model_path": "", "optimizations": "help", @@ -303,6 +368,14 @@ def test_optimize_help(): assert is_test_passed +@pytest.mark.skipif( + condition=not _validate_environment_variables(), + reason="Project's environment variables are not set", +) +@pytest.mark.skipif( + condition=not _is_tf2onnx_available(), + reason="tf2onnx is not installed", +) def test_optimize(): """ Test the 'optimize' handler, giving it a model from the ONNX zoo git repository. @@ -314,6 +387,7 @@ def test_optimize(): log_model_function = mlrun.code_to_function( filename="test_onnx_utils.py", name="log_model", + project=PROJECT_NAME, kind="job", image="mlrun/ml-models", ) @@ -321,18 +395,18 @@ def test_optimize(): # Run the function to log the model: log_model_run = log_model_function.run( handler="_log_tf_keras_model", - artifact_path=artifact_path, + output_path=artifact_path, params={"model_name": MODEL_NAME}, local=True, ) # Import the ONNX Utils function: - onnx_function = mlrun.import_function("function.yaml") + onnx_function = mlrun.import_function("function.yaml", project=PROJECT_NAME) # Run the function to convert our model to ONNX: to_onnx_function_run = onnx_function.run( handler="to_onnx", - artifact_path=artifact_path, + output_path=artifact_path, params={ # Take the logged model from the previous function. "model_path": log_model_run.status.artifacts[0]["spec"]["target_path"], @@ -345,7 +419,7 @@ def test_optimize(): # Run the function to optimize our model: optimize_function_run = onnx_function.run( handler="optimize", - artifact_path=artifact_path, + output_path=artifact_path, params={ # Take the logged model from the previous function. "model_path": to_onnx_function_run.status.artifacts[0]["spec"][