diff --git a/gooddata-pipelines/.gitignore b/gooddata-pipelines/.gitignore new file mode 100644 index 0000000..bcfc50e --- /dev/null +++ b/gooddata-pipelines/.gitignore @@ -0,0 +1,5 @@ +**venv/* +**/__pycache__/* +**.env +**.log +dist/** diff --git a/gooddata-pipelines/LICENSE.txt b/gooddata-pipelines/LICENSE.txt new file mode 100644 index 0000000..fef9097 --- /dev/null +++ b/gooddata-pipelines/LICENSE.txt @@ -0,0 +1,714 @@ +# (C) 2025 GoodData Corporation +BSD License + +Copyright (c) 2023-2025, GoodData Corporation. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted, provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================================ + + Dependencies + +================================================================================ + +- pydantic (2.11.3) [MIT] +- requests (2.32.3) [Apache-2.0] +- types-requests (2.32.0.20250602) +- gooddata-sdk (1.43.0) [MIT] +- boto3 (1.39.3) [Apache-2.0] +- boto3-stubs (1.39.3) [MIT] + + +-------------------------------------------------------------------------------- +Package Title: pydantic (2.11.3) +-------------------------------------------------------------------------------- + +* Declared Licenses * +MIT + +The MIT License (MIT) + +Copyright (c) 2017 to present Pydantic Services Inc. and individual contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +-------------------------------------------------------------------------------- +Package Title: requests (2.32.3) +-------------------------------------------------------------------------------- + +* Declared Licenses * +Apache-2.0 + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + + +-------------------------------------------------------------------------------- +Package Title: types-requests (2.32.3) +-------------------------------------------------------------------------------- + +* Declared Licenses * +Apache-2.0 + +The "typeshed" project is licensed under the terms of the Apache license, as +reproduced below. + += = = = = + +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + += = = = = + +Parts of typeshed are licensed under different licenses (like the MIT +license), reproduced below. + += = = = = + +The MIT License + +Copyright (c) 2015 Jukka Lehtosalo and contributors + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + += = = = = + +-------------------------------------------------------------------------------- +Package Title: gooddata-sdk (1.43.0) +-------------------------------------------------------------------------------- + +* Declared Licenses * +MIT + +Copyright (c) 2022-2024 GoodData Corporation + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +-------------------------------------------------------------------------------- +Package Title: boto3 (1.39.3) +-------------------------------------------------------------------------------- + +* Declared Licenses * +Apache-2.0 + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +-------------------------------------------------------------------------------- +Package Title: boto3-stubs (1.39.3) +-------------------------------------------------------------------------------- + +* Declared Licenses * +MIT + +MIT License + +Copyright (c) 2022 Vlad Emelianov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/gooddata-pipelines/README.md b/gooddata-pipelines/README.md new file mode 100644 index 0000000..ce85364 --- /dev/null +++ b/gooddata-pipelines/README.md @@ -0,0 +1,63 @@ +# GoodData Pipelines + +A high level library for automating the lifecycle of GoodData Cloud (GDC). + +You can use the package to manage following resoursec in GDC: + +1. Provisioning (create, update, delete) + - User profiles + - User Groups + - User/Group permissions + - User Data Filters + - Child workspaces (incl. Workspace Data Filter settings) +1. _[PLANNED]:_ Backup and restore of workspaces +1. _[PLANNED]:_ Custom fields management + - extend the Logical Data Model of a child workspace + +In case you are not interested in incorporating a library in your own program, but would like to use a ready-made script, consider having a look at [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools). + +## Provisioning + +The entities can be managed either in _full load_ or _incremental_ way. + +Full load means that the input data should represent the full and complete desired state of GDC after the script has finished. For example, you would include specification of all child workspaces you want to exist in GDC in the input data for workspace provisioning. Any workspaces present in GDC and not defined in the source data (i.e., your input) will be deleted. + +On the other hand, the incremental load treats the source data as instructions for a specific change, e.g., a creation or a deletion of a specific workspace. You can specify which workspaces you would want to delete or create, while the rest of the workspaces already present in GDC will remain as they are, ignored by the provisioning script. + +The provisioning module exposes _Provisioner_ classes reflecting the different entities. The typical usage would involve importing the Provisioner class and the data input data model for the class and planned provisioning method: + +```python +import os +from csv import DictReader +from pathlib import Path + +# Import the Entity Provisioner class and corresponing model from gooddata_pipelines library +from gooddata_pipelines import UserFullLoad, UserProvisioner + +# Optional: you can set up logging and subscribe it to the Provisioner +from utils.logger import setup_logging + +setup_logging() +logger = logging.getLogger(__name__) + +# Create the Provisioner instance - you can also create the instance from a GDC yaml profile +provisioner = UserProvisioner( + host=os.environ["GDC_HOSTNAME"], token=os.environ["GDC_AUTH_TOKEN"] +) + +# Optional: subscribe to logs +provisioner.logger.subscribe(logger) + +# Load your data from your data source +source_data_path: Path = Path("path/to/some.csv") +source_data_reader = DictReader(source_data_path.read_text().splitlines()) +source_data = [row for row in source_data_reader] + +# Validate your input data with +full_load_data: list[UserFullLoad] = UserFullLoad.from_list_of_dicts( + source_data +) +provisioner.full_load(full_load_data) +``` + +Ready made scripts covering the basic use cases can be found here in the [GoodData Productivity Tools](https://github.com/gooddata/gooddata-productivity-tools) repository diff --git a/gooddata-pipelines/TODO.md b/gooddata-pipelines/TODO.md new file mode 100644 index 0000000..fa9053b --- /dev/null +++ b/gooddata-pipelines/TODO.md @@ -0,0 +1,25 @@ +# TODO + +A list of outstanding tasks, features, or technical debt to be addressed in this project. + +## Pre-release + +- [ ] License file (FOSSA) + +## Features + +- [ ] Workspace restore + +## Refactoring / Debt + +- [ ] Cleanup custom exception +- [ ] Improve test coverage. Write missing unit tests for legacy code (e.g., user data filters) + +## Documentation + +- [ ] Improve package README +- [ ] Workspace provisioning +- [ ] User provisioning +- [ ] User group provisioning +- [ ] Permission provisioning +- [ ] User data filter provisioning diff --git a/gooddata-pipelines/gooddata_pipelines/__init__.py b/gooddata-pipelines/gooddata_pipelines/__init__.py new file mode 100644 index 0000000..bbaa5e4 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/__init__.py @@ -0,0 +1,59 @@ +# (C) 2025 GoodData Corporation + +from ._version import __version__ + +# -------- Backup and Restore -------- +from .backup_and_restore.backup_manager import BackupManager +from .backup_and_restore.models.storage import ( + BackupRestoreConfig, + StorageType, +) +from .backup_and_restore.storage.local_storage import LocalStorage +from .backup_and_restore.storage.s3_storage import S3Storage + +# -------- Provisioning -------- +from .provisioning.entities.user_data_filters.models.udf_models import ( + UserDataFilterFullLoad, +) +from .provisioning.entities.user_data_filters.user_data_filters import ( + UserDataFilterProvisioner, +) +from .provisioning.entities.users.models.permissions import ( + PermissionFullLoad, + PermissionIncrementalLoad, +) +from .provisioning.entities.users.models.user_groups import ( + UserGroupFullLoad, + UserGroupIncrementalLoad, +) +from .provisioning.entities.users.models.users import ( + UserFullLoad, + UserIncrementalLoad, +) +from .provisioning.entities.users.permissions import PermissionProvisioner +from .provisioning.entities.users.user_groups import UserGroupProvisioner +from .provisioning.entities.users.users import UserProvisioner +from .provisioning.entities.workspaces.models import WorkspaceFullLoad +from .provisioning.entities.workspaces.workspace import WorkspaceProvisioner + +__all__ = [ + "BackupManager", + "BackupRestoreConfig", + "StorageType", + "LocalStorage", + "S3Storage", + "WorkspaceFullLoad", + "WorkspaceProvisioner", + "UserIncrementalLoad", + "UserGroupIncrementalLoad", + "PermissionFullLoad", + "PermissionIncrementalLoad", + "UserFullLoad", + "UserGroupFullLoad", + "UserProvisioner", + "UserGroupProvisioner", + "PermissionProvisioner", + "UserDataFilterProvisioner", + "UserDataFilterFullLoad", + "__version__", +] diff --git a/gooddata-pipelines/gooddata_pipelines/_version.py b/gooddata-pipelines/gooddata_pipelines/_version.py new file mode 100644 index 0000000..6c30963 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/_version.py @@ -0,0 +1,7 @@ +# (C) 2025 GoodData Corporation +from importlib import metadata + +try: + __version__ = metadata.version("gooddata-pipelines") +except metadata.PackageNotFoundError: + __version__ = "unknown-version" diff --git a/gooddata-pipelines/gooddata_pipelines/api/__init__.py b/gooddata-pipelines/gooddata_pipelines/api/__init__.py new file mode 100644 index 0000000..c4b88ed --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/api/__init__.py @@ -0,0 +1,5 @@ +# (C) 2025 GoodData Corporation + +from .gooddata_api_wrapper import GoodDataAPI + +__all__ = ["GoodDataAPI"] diff --git a/gooddata-pipelines/gooddata_pipelines/api/exceptions.py b/gooddata-pipelines/gooddata_pipelines/api/exceptions.py new file mode 100644 index 0000000..4328b6f --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/api/exceptions.py @@ -0,0 +1,41 @@ +# (C) 2025 GoodData Corporation + +"""Exception class for Panther operations. + +This module defines the internally used `PantherException` class, which is used +to handle exceptions that occur during operations related to the Panther SDK or +GoodData Cloud API. +""" + + +class GoodDataApiException(Exception): + """Exception raised during Panther operations. + + This exception is used to indicate errors that occur during operations + related to interactions with the GoodData Python SDK or GoodData Cloud API. + It can include additional context provided through keyword arguments. + """ + + def __init__(self, message: str, **kwargs: str) -> None: + """Raise a PantherException with a message and optional context. + + Args: + message (str): The error message describing the exception. + **kwargs: Additional context for the exception, such as HTTP status, + API endpoint, and HTTP method or any other relevant information. + """ + + super().__init__(message) + self.error_message: str = message + + # Set default values for attributes. + # TODO: Consider if the defaults for these are still needed + # - the values were necessary for log schema implementations, which + # are not used anymore. + self.http_status: str = "500 Internal Server Error" + self.api_endpoint: str = "NA" + self.http_method: str = "NA" + + # Set attributes from kwargs. + for key, value in kwargs.items(): + setattr(self, key, value) diff --git a/gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py b/gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py new file mode 100644 index 0000000..1512cb6 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/api/gooddata_api.py @@ -0,0 +1,309 @@ +# (C) 2025 GoodData Corporation + +"""Interaction with GoodData Cloud via the direct API calls.""" + +import json +from typing import Any + +import requests + +# TODO: Limit the use of "typing.Any". Improve readability by using either models +# or typed dicts. + +TIMEOUT = 60 +REQUEST_PAGE_SIZE = 250 +API_VERSION = "v1" + + +class APIMethods: + headers: dict[str, str] + base_url: str + + @staticmethod + def _get_base_url(domain: str) -> str: + """Returns the root endpoint for the GoodData Cloud API. + + Method ensures that the URL starts with "https://" and does not + end with a trailing slash. + + Args: + domain (str): The domain of the GoodData Cloud instance. + Returns: + str: The base URL for the GoodData Cloud API. + """ + # Remove trailing slash if present. + if domain[-1] == "/": + domain = domain[:-1] + + if not domain.startswith("https://") and not domain.startswith( + "http://" + ): + domain = f"https://{domain}" + + if domain.startswith("http://") and not domain.startswith("https://"): + domain = domain.replace("http://", "https://") + + return f"{domain}/api/{API_VERSION}" + + def _get_url(self, endpoint: str) -> str: + """Returns the full URL for a given API endpoint. + + Args: + endpoint (str): The API endpoint to be appended to the base URL. + Returns: + str: The full URL for the API endpoint. + """ + return f"{self.base_url}{endpoint}" + + def get_custom_application_setting( + self, workspace_id: str, setting_id: str + ) -> requests.Response: + """Gets a custom application setting. + + Args: + workspace_id (str): The ID of the workspace. + setting_id (str): The ID of the custom application setting. + Returns: + requests.Response: The response from the server containing the + custom application setting. + """ + url = f"/entities/workspaces/{workspace_id}/customApplicationSettings/{setting_id}" + return self._get(url) + + def put_custom_application_setting( + self, workspace_id: str, setting_id: str, data: dict[str, Any] + ) -> requests.Response: + url = f"/entities/workspaces/{workspace_id}/customApplicationSettings/{setting_id}" + return self._put(url, data, self.headers) + + def post_custom_application_setting( + self, workspace_id: str, data: dict[str, Any] + ) -> requests.Response: + """Creates a custom application setting for a given workspace. + + Args: + workspace_id (str): The ID of the workspace. + data (dict[str, Any]): The data for the custom application setting. + Returns: + requests.Response: The response from the server containing the + created custom application setting. + """ + url = f"/entities/workspaces/{workspace_id}/customApplicationSettings/" + return self._post(url, data, self.headers) + + def get_all_workspace_data_filters( + self, workspace_id: str + ) -> requests.Response: + """Gets all workspace data filters for a given workspace. + + Args: + workspace_id (str): The ID of the workspace. + Returns: + requests.Response: The response from the server containing all + workspace data filters. + """ + url = f"/entities/workspaces/{workspace_id}/workspaceDataFilters" + return self._get(url) + + def get_workspace_data_filter_settings( + self, workspace_id: str + ) -> requests.Response: + """Gets all workspace data filter settings for a given workspace. + + Args: + workspace_id (str): The ID of the workspace. + Returns: + requests.Response: The response from the server containing all + workspace data filter settings. + """ + url = f"/entities/workspaces/{workspace_id}/workspaceDataFilterSettings?include=workspaceDataFilters" + return self._get(url) + + def get_workspace_data_filter_setting( + self, workspace_id: str, wdf_id: str + ) -> requests.Response: + """Gets a specific workspace data filter setting. + + Args: + workspace_id (str): The ID of the workspace. + wdf_id (str): The ID of the workspace data filter setting. + Returns: + requests.Response: The response from the server containing the + workspace data filter setting. + """ + url = f"/entities/workspaces/{workspace_id}/workspaceDataFilterSettings/{wdf_id}" + return self._get(url) + + def put_workspace_data_filter_setting( + self, + workspace_id: str, + wdf_setting: dict[str, Any], + ) -> requests.Response: + """Updates a workspace data filter setting. + + Args: + workspace_id (str): The ID of the workspace. + wdf_setting (dict[str, Any]): The workspace data filter setting to + update. + Returns: + requests.Response: The response from the server containing the + updated workspace data filter setting. + """ + wdf_setting_id = wdf_setting["data"]["id"] + endpoint = f"/entities/workspaces/{workspace_id}/workspaceDataFilterSettings/{wdf_setting_id}" + return self._put( + endpoint, + wdf_setting, + self.headers, + ) + + def post_workspace_data_filter_setting( + self, + workspace_id: str, + wdf_setting: dict[str, Any], + ) -> requests.Response: + """Creates a workspace data filter setting for a given workspace. + + Args: + workspace_id (str): The ID of the workspace. + wdf_setting (dict[str, Any]): The workspace data filter setting to + create. + Returns: + requests.Response: The response from the server containing the + created workspace data filter setting. + """ + endpoint = ( + f"/entities/workspaces/{workspace_id}/workspaceDataFilterSettings/" + ) + return self._post( + endpoint, + wdf_setting, + self.headers, + ) + + def delete_workspace_data_filter_setting( + self, + workspace_id: str, + wdf_setting_id: str, + ) -> requests.Response: + """Deletes a workspace data filter setting. + + Args: + workspace_id (str): The ID of the workspace. + wdf_setting_id (str): The ID of the workspace data filter setting + to delete. + Returns: + requests.Response: The response from the server confirming the + deletion of the workspace data filter setting. + """ + endpoint = f"/entities/workspaces/{workspace_id}/workspaceDataFilterSettings/{wdf_setting_id}" + return self._delete( + endpoint, + ) + + def post_workspace_data_filter( + self, workspace_id: str, data: dict[str, Any] + ) -> requests.Response: + """Creates a workspace data filter for a given workspace. + + Args: + workspace_id (str): The ID of the workspace. + data (dict[str, Any]): The data for the workspace data filter. + Returns: + requests.Response: The response from the server containing the + created workspace data filter. + """ + endpoint = f"/entities/workspaces/{workspace_id}/workspaceDataFilters" + return self._post(endpoint, data, self.headers) + + def get_user_data_filters(self, workspace_id: str) -> requests.Response: + """Gets the user data filters for a given workspace.""" + endpoint = f"/layout/workspaces/{workspace_id}/userDataFilters" + return self._get(endpoint) + + def get_automations(self, workspace_id: str) -> requests.Response: + """Gets the automations for a given workspace.""" + endpoint = ( + f"/entities/workspaces/{workspace_id}/automations?include=ALL" + ) + return self._get(endpoint) + + def _get( + self, endpoint: str, headers: dict[str, str] | None = None + ) -> requests.Response: + """Sends a GET request to the server. + + Args: + endpoint (str): The API endpoint to send the GET request to. + headers (dict[str, str] | None): Headers to include in the request. + If no headers are provided, the default headers will be used. + Returns: + requests.Response: The response from the server. + """ + url = self._get_url(endpoint) + request_headers = headers if headers else self.headers + + return requests.get(url, headers=request_headers, timeout=TIMEOUT) + + def _post( + self, + endpoint: str, + data: Any, + headers: dict | None = None, + ) -> requests.Response: + """Sends a POST request to the server with a given JSON object. + + Args: + endpoint (str): The API endpoint to send the POST request to. + data (Any): The JSON data to send in the request body. + headers (dict | None): Headers to include in the request. + If no headers are provided, the default headers will be used. + Returns: + requests.Response: The response from the server. + """ + url = self._get_url(endpoint) + request_headers = headers if headers else self.headers + data_json = json.dumps(data) + + return requests.post( + url, data=data_json, headers=request_headers, timeout=TIMEOUT + ) + + def _put( + self, + endpoint: str, + data: Any, + headers: dict | None = None, + ) -> requests.Response: + """Sends a PUT request to the server with a given JSON object. + + Args: + endpoint (str): The API endpoint to send the PUT request to. + data (Any): The JSON data to send in the request body. + headers (dict | None): Headers to include in the request. + If no headers are provided, the default headers will be used. + Returns: + requests.Response: The response from the server. + """ + url = self._get_url(endpoint) + request_headers = headers if headers else self.headers + data_json = json.dumps(data) + + return requests.put( + url, data=data_json, headers=request_headers, timeout=TIMEOUT + ) + + def _delete( + self, + endpoint: str, + ) -> requests.Response: + """Sends a DELETE request to the server. + + Args: + endpoint (str): The API endpoint to send the DELETE request to. + Returns: + requests.Response: The response from the server. + """ + url = self._get_url(endpoint) + + return requests.delete(url, headers=self.headers, timeout=TIMEOUT) diff --git a/gooddata-pipelines/gooddata_pipelines/api/gooddata_api_wrapper.py b/gooddata-pipelines/gooddata_pipelines/api/gooddata_api_wrapper.py new file mode 100644 index 0000000..eb6169c --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/api/gooddata_api_wrapper.py @@ -0,0 +1,36 @@ +# (C) 2025 GoodData Corporation + +"""Wrapper for interaction with GoodData Cloud.""" + +from gooddata_sdk.sdk import GoodDataSdk + +from gooddata_pipelines.api.gooddata_api import APIMethods +from gooddata_pipelines.api.gooddata_sdk import SDKMethods + + +class GoodDataAPI(SDKMethods, APIMethods): + """Wrapper class for the GoodData Cloud API. + + This class combines interactions with the GoodData Python SDK and direct API + calls. + """ + + def __init__(self, host: str, token: str) -> None: + """Initialize the GoodDataAPI with host and token. + + Args: + host (str): The GoodData Cloud host URL. + token (str): The authentication token for the GoodData Cloud API. + """ + self._domain: str = host + self._token: str = token + + # Initialize the GoodData SDK + self._sdk = GoodDataSdk.create(self._domain, self._token) + + # Set up utils for direct API interaction + self.base_url = self._get_base_url(self._domain) + self.headers: dict = { + "Authorization": f"Bearer {self._token}", + "Content-Type": "application/vnd.gooddata.api+json", + } diff --git a/gooddata-pipelines/gooddata_pipelines/api/gooddata_sdk.py b/gooddata-pipelines/gooddata_pipelines/api/gooddata_sdk.py new file mode 100644 index 0000000..a988913 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/api/gooddata_sdk.py @@ -0,0 +1,373 @@ +# (C) 2025 GoodData Corporation + +"""Interaction with GoodData Cloud via the Gooddata Python SDK.""" + +from pathlib import Path + +from gooddata_sdk.catalog.permission.declarative_model.permission import ( + CatalogDeclarativeWorkspacePermissions, +) +from gooddata_sdk.catalog.user.entity_model.user import CatalogUser +from gooddata_sdk.catalog.user.entity_model.user_group import CatalogUserGroup +from gooddata_sdk.catalog.workspace.declarative_model.workspace.workspace import ( + CatalogDeclarativeWorkspaceDataFilters, +) +from gooddata_sdk.catalog.workspace.entity_model.user_data_filter import ( + CatalogUserDataFilter, +) +from gooddata_sdk.catalog.workspace.entity_model.workspace import ( + CatalogWorkspace, +) +from gooddata_sdk.sdk import GoodDataSdk + +from gooddata_pipelines.api.utils import raise_with_context + + +def apply_to_all_methods(decorator): + def decorate(cls): + for attr in cls.__dict__: + if callable(getattr(cls, attr)) and not attr.startswith("__"): + setattr(cls, attr, decorator(getattr(cls, attr))) + return cls + + return decorate + + +@apply_to_all_methods(raise_with_context()) +class SDKMethods: + """ + Class to intaract with GoodData Cloud via the Gooddata Python SDK. + """ + + _sdk: GoodDataSdk + + def get_organization_id(self) -> str: + return self._sdk.catalog_organization.organization_id + + def check_workspace_exists(self, workspace_id: str) -> bool: + try: + self._sdk.catalog_workspace.get_workspace(workspace_id) + return True + except Exception: + return False + + def get_workspace(self, workspace_id: str, **_: str) -> CatalogWorkspace: + """ + Calls GoodData Python SDK to retrieve a workspace by its ID. + + Args: + workspace_id (str): The ID of the workspace to retrieve. + Returns: + CatalogWorkspace: The workspace object retrieved from the SDK. + Raises: + GoodDataApiException: If the workspace cannot be retrieved, an exception + is raised with additional context information. + """ + return self._sdk.catalog_workspace.get_workspace(workspace_id) + + def delete_panther_workspace(self, workspace_id: str) -> None: + """ + Calls GoodData Python SDK to delete a workspace by its ID. + + Args: + workspace_id (str): The ID of the workspace to delete. + Raises: + GoodDataApiException: If the workspace cannot be deleted, an exception + is raised with additional context information. + """ + self._sdk.catalog_workspace.delete_workspace(workspace_id) + + def create_or_update_panther_workspace( + self, + workspace_id: str, + workspace_name: str, + parent_id: str | None, + **_: str, + ) -> None: + """ + Calls GoodData Python SDK to create or update a workspace with the given ID, + name, and parent ID. + + Args: + workspace_id (str): The ID of the workspace to create or update. + workspace_name (str): The name of the workspace. + parent_id (str | None): The ID of the parent workspace, if any. + Returns: + None + Raises: + GoodDataApiException: If the workspace cannot be created or updated, + an exception is raised with additional context information. + """ + return self._sdk.catalog_workspace.create_or_update( + CatalogWorkspace( + workspace_id=workspace_id, + name=workspace_name, + parent_id=parent_id, + ) + ) + + def get_panther_children_workspaces( + self, parent_workspace_ids: set[str] + ) -> list[CatalogWorkspace]: + """ + Calls GoodData Python SDK to retrieve all workspaces in domain and filters the + result by the set of parent workspace IDs. + + Args: + parent_workspace_ids (set[str]): A set of parent workspace IDs to filter + child workspaces. + Returns: + list[CatalogWorkspace]: List of child workspaces in the parent workspace. + """ + all_workspaces: list[CatalogWorkspace] = self.list_workspaces() + + children: list[CatalogWorkspace] = [ + workspace + for workspace in all_workspaces + if workspace.parent_id in parent_workspace_ids + ] + + return children + + def list_workspaces(self) -> list[CatalogWorkspace]: + """Retrieves all workspaces in the GoodData Cloud domain. + + Returns: + list[CatalogWorkspace]: A list of all workspaces in the domain. + Raises: + GoodDataApiException: If the workspaces cannot be retrieved, an exception + is raised with additional context information. + """ + return self._sdk.catalog_workspace.list_workspaces() + + def get_declarative_permissions( + self, workspace_id: str + ) -> CatalogDeclarativeWorkspacePermissions: + """ + Retrieves the declarative permissions for a given workspace. + + Args: + workspace_id (str): The ID of the workspace for which to retrieve + permissions. + Returns: + CatalogDeclarativeWorkspacePermissions: The declarative permissions + for the workspace. + Raises: + GoodDataApiException: If the permissions cannot be retrieved, an exception + is raised with additional context information. + """ + return self._sdk.catalog_permission.get_declarative_permissions( + workspace_id + ) + + def put_declarative_permissions( + self, + workspace_id: str, + ws_permissions: CatalogDeclarativeWorkspacePermissions, + ) -> None: + """ + Updates the declarative permissions for a given workspace. + + Args: + workspace_id (str): The ID of the workspace for which to update + permissions. + ws_permissions (CatalogDeclarativeWorkspacePermissions): The new + declarative permissions to set for the workspace. + Returns: + None + Raises: + GoodDataApiException: If the permissions cannot be updated, an exception + is raised with additional context information. + """ + return self._sdk.catalog_permission.put_declarative_permissions( + workspace_id, ws_permissions + ) + + def get_user(self, user_id: str, **_: str) -> CatalogUser: + """ + Calls GoodData Python SDK to retrieve a user by its ID. + + Args: + user_id (str): The ID of the user to retrieve. + Returns: + CatalogUser: The user object retrieved from the SDK. + Raises: + GoodDataApiException: If the user cannot be retrieved, an exception + is raised with additional context information. + """ + return self._sdk.catalog_user.get_user(user_id) + + def create_or_update_user(self, user: CatalogUser, **_: str) -> None: + """ + Calls GoodData Python SDK to create or update a user. + + Args: + user (CatalogUser): The user object to create or update. + Returns: + None + Raises: + GoodDataApiException: If the user cannot be created or updated, + an exception is raised with additional context information. + """ + return self._sdk.catalog_user.create_or_update_user(user) + + def delete_user(self, user_id: str, **_: str) -> None: + """ + Calls GoodData Python SDK to delete a user by its ID. + + Args: + user_id (str): The ID of the user to delete. + Returns: + None + Raises: + GoodDataApiException: If the user cannot be deleted, an exception + is raised with additional context information. + """ + return self._sdk.catalog_user.delete_user(user_id) + + def get_user_group(self, user_group_id: str, **_: str) -> CatalogUserGroup: + """ + Calls GoodData Python SDK to retrieve a user group by its ID. + + Args: + user_group_id (str): The ID of the user group to retrieve. + Returns: + CatalogUserGroup: The user group object retrieved from the SDK. + Raises: + GoodDataApiException: If the user group cannot be retrieved, an exception + is raised with additional context information. + """ + return self._sdk.catalog_user.get_user_group(user_group_id) + + def list_user_groups(self) -> list[CatalogUserGroup]: + """ + Calls GoodData Python SDK to retrieve all user groups. + + Returns: + list[CatalogUserGroup]: A list of all user groups in the domain. + Raises: + GoodDataApiException: If the user groups cannot be retrieved, an + exception is raised with additional context information. + """ + return self._sdk.catalog_user.list_user_groups() + + def list_users(self) -> list[CatalogUser]: + """Calls GoodData Python SDK to retrieve all users. + + Returns: + list[CatalogUser]: A list of all users in the domain. + """ + return self._sdk.catalog_user.list_users() + + def create_or_update_user_group( + self, catalog_user_group: CatalogUserGroup, **_: str + ) -> None: + """Calls GoodData Python SDK to create or update a user group. + + Args: + catalog_user_group (CatalogUserGroup): The user group object to create or update. + Returns: + None + Raises: + GoodDataApiException: If the user group cannot be created or updated, + an exception is raised with additional context information. + """ + return self._sdk.catalog_user.create_or_update_user_group( + catalog_user_group + ) + + def delete_user_group(self, user_group_id: str) -> None: + """Calls GoodData Python SDK to delete a user group by its ID. + + Args: + user_group_id (str): The ID of the user group to delete. + Returns: + None + Raises: + GoodDataApiException: If the user group cannot be deleted, an exception + is raised with additional context information. + """ + return self._sdk.catalog_user.delete_user_group(user_group_id) + + def get_declarative_workspace_data_filters( + self, + ) -> CatalogDeclarativeWorkspaceDataFilters: + """Retrieves the declarative workspace data filters. + + Returns: + CatalogDeclarativeWorkspaceDataFilters: The declarative workspace data filters. + Raises: + GoodDataApiException: If the declarative workspace data filters cannot be retrieved, + an exception is raised with additional context information. + """ + return ( + self._sdk.catalog_workspace.get_declarative_workspace_data_filters() + ) + + def list_user_data_filters( + self, workspace_id: str + ) -> list[CatalogUserDataFilter]: + """Lists all user data filters for a given workspace. + + Args: + workspace_id (str): The ID of the workspace for which to list user data filters. + Returns: + list[CatalogUserDataFilter]: A list of user data filters for the specified workspace. + Raises: + GoodDataApiException: If the user data filters cannot be listed, an exception + is raised with additional context information. + """ + return self._sdk.catalog_workspace.list_user_data_filters(workspace_id) + + def delete_user_data_filter( + self, workspace_id: str, user_data_filter_id: str + ) -> None: + """Deletes a user data filter by its ID in the specified workspace. + + Args: + workspace_id (str): The ID of the workspace containing the user data filter. + user_data_filter_id (str): The ID of the user data filter to delete. + Returns: + None + Raises: + GoodDataApiException: If the user data filter cannot be deleted, an exception + is raised with additional context information. + """ + self._sdk.catalog_workspace.delete_user_data_filter( + workspace_id, user_data_filter_id + ) + + def create_or_update_user_data_filter( + self, workspace_id: str, user_data_filter: CatalogUserDataFilter + ) -> None: + """Creates or updates a user data filter in the specified workspace. + + Args: + workspace_id (str): The ID of the workspace where the user data filter + should be created or updated. + user_data_filter (CatalogUserDataFilter): The user data filter object to create or update. + Returns: + None + Raises: + GoodDataApiException: If the user data filter cannot be created or updated, + an exception is raised with additional context information. + """ + self._sdk.catalog_workspace.create_or_update_user_data_filter( + workspace_id, user_data_filter + ) + + def store_declarative_workspace( + self, workspace_id: str, export_path: Path + ) -> None: + """Stores the declarative workspace in the specified export path.""" + self._sdk.catalog_workspace.store_declarative_workspace( + workspace_id, export_path + ) + + def store_declarative_filter_views( + self, workspace_id: str, export_path: Path + ) -> None: + """Stores the declarative filter views in the specified export path.""" + self._sdk.catalog_workspace.store_declarative_filter_views( + workspace_id, export_path + ) diff --git a/gooddata-pipelines/gooddata_pipelines/api/utils.py b/gooddata-pipelines/gooddata_pipelines/api/utils.py new file mode 100644 index 0000000..10fc7a3 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/api/utils.py @@ -0,0 +1,43 @@ +# (C) 2025 GoodData Corporation + +"""Utility functions for GoodData Cloud API interactions.""" + +from typing import Any, Callable + +from gooddata_api_client import ApiException # type: ignore + +from gooddata_pipelines.api.exceptions import GoodDataApiException + + +def raise_with_context(**context_kwargs: str) -> Callable: + """ + Decorator to catch exceptions raised by SDK methods and raise a GoodDataApiException + with additional context information. + + Args: + context_kwargs (dict): Additional context information to include in the + GoodDataApiException. + """ + + def decorator(fn: Callable) -> Callable: + def wrapper(*method_args: Any, **method_kwargs: Any) -> Callable: + try: + return fn(*method_args, **method_kwargs) + except Exception as e: + # Process known exceptions + if isinstance(e, ApiException): + context_kwargs["http_status"] = f"{e.status} {e.reason}" + exception_content = e.body + else: + exception_content = str(e) + + # Format the exception message: "{exception_type}: {exception_content}" + message = f"{type(e).__name__}: {exception_content}" + + raise GoodDataApiException( + message, **context_kwargs, **method_kwargs + ) + + return wrapper + + return decorator diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/__init__.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/backup_input_processor.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/backup_input_processor.py new file mode 100644 index 0000000..ec75b21 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/backup_input_processor.py @@ -0,0 +1,195 @@ +# (C) 2025 GoodData Corporation + +from dataclasses import dataclass + +import requests + +from gooddata_pipelines.api import GoodDataAPI +from gooddata_pipelines.api.gooddata_api import API_VERSION +from gooddata_pipelines.backup_and_restore.csv_reader import CSVReader +from gooddata_pipelines.backup_and_restore.models.input_type import InputType +from gooddata_pipelines.backup_and_restore.models.workspace_response import ( + Workspace, + WorkspaceResponse, +) +from gooddata_pipelines.logger import LogObserver + + +class BackupInputProcessor: + """Class to handle the input CSV and prepare the actual input for the backup. + + Based on the InputType value, this class will determine which approach to take + in getting the IDs of workspaces to backup. It will then call appropriate + GoodData Cloud endpoints to get the IDs and return them as a list. + """ + + _api: GoodDataAPI + base_workspace_endpoint: str + hierarchy_endpoint: str + all_workspaces_endpoint: str + + def __init__(self, api: GoodDataAPI, page_size: int) -> None: + self._api = api + self.page_size = page_size + self.logger = LogObserver() + self.csv_reader = CSVReader() + + self.set_endpoints() + + def set_endpoints(self) -> None: + """Sets the hierarchy endpoint for the API client.""" + self.base_workspace_endpoint = "/api/v1/entities/workspaces" + self.hierarchy_endpoint = ( + f"{self.base_workspace_endpoint}?" + + "filter=parent.id=={parent_id}" + + f"&include=parent&page=0&size={self.page_size}&sort=name,asc&metaInclude=page,hierarchy" + ) + self.all_workspaces_endpoint = f"{self.base_workspace_endpoint}?page=0&size={self.page_size}&sort=name,asc&metaInclude=page" + + @dataclass + class _ProcessDataOutput: + workspace_ids: list[str] + sub_parents: list[str] | None = None + + def fetch_page(self, url: str) -> WorkspaceResponse: + """Fetch a page of workspaces.""" + + # Separate the API path from the URL so that it can be fed to the api class + endpoint: str = url.split(f"api/{API_VERSION}")[1] + response: requests.Response = self._api._get(endpoint) + if response.ok: + return WorkspaceResponse(**response.json()) + else: + raise RuntimeError( + f"Failed to fetch data from the API. URL: {endpoint}" + ) + + @staticmethod + def process_data(data: list[Workspace]) -> _ProcessDataOutput: + """Extract children and sub-parents from workspace data.""" + children: list[str] = [] + sub_parents: list[str] = [] + + for workspace in data: + # append child workspace IDs + children.append(workspace.id) + + # if hierarchy is present and has children, append child workspace ID to sub_parents + if workspace.meta and workspace.meta.hierarchy: + if workspace.meta.hierarchy.children_count > 0: + sub_parents.append(workspace.id) + return BackupInputProcessor._ProcessDataOutput(children, sub_parents) + + def log_paging_progress(self, response: WorkspaceResponse) -> None: + """Log the progress of paging through API responses if paginatino data is present""" + current_page: int | None + total_pages: int | None + + if response.meta.page: + current_page = response.meta.page.number + 1 + total_pages = response.meta.page.total_pages + else: + current_page = None + total_pages = None + + if current_page and total_pages: + self.logger.info(f"Fetched page: {current_page} of {total_pages}") + + def _paginate( + self, url: str | None + ) -> list["BackupInputProcessor._ProcessDataOutput"]: + """Paginates through the API responses and returns a list of workspace IDs.""" + result: list[BackupInputProcessor._ProcessDataOutput] = [] + while url: + response: WorkspaceResponse = self.fetch_page(url) + self.log_paging_progress(response) + result.append(self.process_data(response.data)) + url = response.links.next + + return result + + def get_hierarchy(self, parent_id: str) -> list[str]: + """Returns a list of workspace IDs in the hierarchy.""" + self.logger.info(f"Fetching children of {parent_id}") + url = self.hierarchy_endpoint.format(parent_id=parent_id) + + all_children, sub_parents = [], [] + + results: list[BackupInputProcessor._ProcessDataOutput] = self._paginate( + url + ) + + for result in results: + all_children.extend(result.workspace_ids) + if result.sub_parents: + sub_parents.extend(result.sub_parents) + + for subparent in sub_parents: + all_children += self.get_hierarchy(subparent) + + if not all_children: + self.logger.warning( + f"No child workspaces found for parent workspace ID: {parent_id}" + ) + + return all_children + + def get_all_workspaces(self) -> list[str]: + """Returns a list of all workspace IDs in the organization.""" + # TODO: can be optimized - requests can be sent asynchronously. + # Use the total number of pages to calculate the number of requests + # to be sent. Use semaphore or otherwise limit the number of concurrent + # requests to avoid putting too much load on the server. + self.logger.info("Fetching all workspaces") + url = self.all_workspaces_endpoint + + all_workspaces: list[str] = [] + + results: list[BackupInputProcessor._ProcessDataOutput] = self._paginate( + url + ) + + for result in results: + all_workspaces.extend(result.workspace_ids) + + if not all_workspaces: + self.logger.warning("No workspaces found in the organization.") + + return all_workspaces + + def get_ids_to_backup( + self, input_type: InputType, path_to_csv: str | None = None + ) -> list[str]: + """Returns the list of workspace IDs to back up based on the input type.""" + + if input_type in (InputType.LIST_OF_WORKSPACES, InputType.HIERARCHY): + if path_to_csv is None: + raise ValueError( + f"Path to CSV is required for this input type: {input_type.value}" + ) + + # If we're backing up based on the list, simply read it from the CSV + if input_type == InputType.LIST_OF_WORKSPACES: + return self.csv_reader.read_backup_csv(path_to_csv) + else: + # For hierarchy backup, we read the CSV and treat it as a list of + # parent workspace IDs. Then we retrieve the children of each parent, + # including their children, and so on. The parent workspaces are + # also included in the backup. + list_of_parents = self.csv_reader.read_backup_csv(path_to_csv) + list_of_children: list[str] = [] + + for parent in list_of_parents: + list_of_children.extend(self.get_hierarchy(parent)) + + return list_of_parents + list_of_children + + # If we're backing up the entire organization, we simply get all workspaces + elif input_type == InputType.ORGANIZATION: + list_of_workspaces = self.get_all_workspaces() + return list_of_workspaces + + else: + # This path should be unreachable as long as the conditions above + # exhaustively check all values of InputType Enum. + raise ValueError(f"Unsupported input type: {input_type.value}") diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/backup_manager.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/backup_manager.py new file mode 100644 index 0000000..0dd91d4 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/backup_manager.py @@ -0,0 +1,430 @@ +# (C) 2025 GoodData Corporation + +import json +import os +import shutil +import tempfile +import threading +import time +import traceback +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Type + +import requests +import yaml +from gooddata_sdk.utils import PROFILES_FILE_PATH, profile_content + +from gooddata_pipelines.api.gooddata_api_wrapper import GoodDataAPI +from gooddata_pipelines.backup_and_restore.backup_input_processor import ( + BackupInputProcessor, +) +from gooddata_pipelines.backup_and_restore.constants import ( + BackupSettings, + DirNames, +) +from gooddata_pipelines.backup_and_restore.models.input_type import InputType +from gooddata_pipelines.backup_and_restore.models.storage import ( + BackupRestoreConfig, + StorageType, +) +from gooddata_pipelines.backup_and_restore.storage.base_storage import ( + BackupStorage, +) +from gooddata_pipelines.backup_and_restore.storage.local_storage import ( + LocalStorage, +) +from gooddata_pipelines.backup_and_restore.storage.s3_storage import ( + S3Storage, +) +from gooddata_pipelines.logger import LogObserver + + +@dataclass +class BackupBatch: + list_of_ids: list[str] + + +class BackupManager: + storage: BackupStorage + + def __init__(self, host: str, token: str, config: BackupRestoreConfig): + self._api = GoodDataAPI(host, token) + self.logger = LogObserver() + + self.config = config + + self.storage = self.get_storage(self.config) + self.org_id = self._api.get_organization_id() + + self.loader = BackupInputProcessor(self._api, self.config.api_page_size) + + @classmethod + def create( + cls: Type["BackupManager"], + config: BackupRestoreConfig, + host: str, + token: str, + ) -> "BackupManager": + """Creates a backup worker instance using provided host and token.""" + return cls(host=host, token=token, config=config) + + @classmethod + def create_from_profile( + cls: Type["BackupManager"], + config: BackupRestoreConfig, + profile: str = "default", + profiles_path: Path = PROFILES_FILE_PATH, + ) -> "BackupManager": + """Creates a backup worker instance using a GoodData profile file.""" + content = profile_content(profile, profiles_path) + return cls(**content, config=config) + + def get_storage(self, conf: BackupRestoreConfig) -> BackupStorage: + """Returns the storage class based on the storage type.""" + if conf.storage_type == StorageType.S3: + return S3Storage(conf) + elif conf.storage_type == StorageType.LOCAL: + return LocalStorage(conf) + else: + raise RuntimeError( + f'Unsupported storage type "{conf.storage_type.value}".' + ) + + def get_user_data_filters(self, ws_id: str) -> dict: + """Returns the user data filters for the specified workspace.""" + response: requests.Response = self._api.get_user_data_filters(ws_id) + if response.ok: + return response.json() + else: + raise RuntimeError(f"{response.status_code}: {response.text}") + + def store_user_data_filters( + self, + user_data_filters: dict, + export_path: Path, + ws_id: str, + ): + """Stores the user data filters in the specified export path.""" + os.mkdir( + os.path.join( + export_path, + "gooddata_layouts", + self.org_id, + "workspaces", + ws_id, + "user_data_filters", + ) + ) + + for filter in user_data_filters["userDataFilters"]: + udf_file_path = os.path.join( + export_path, + "gooddata_layouts", + self.org_id, + "workspaces", + ws_id, + "user_data_filters", + filter["id"] + ".yaml", + ) + self.write_to_yaml(udf_file_path, filter) + + @staticmethod + def move_folder(source: Path, destination: Path) -> None: + """Moves the source folder to the destination.""" + shutil.move(source, destination) + + @staticmethod + def write_to_yaml(path: str, source): + """Writes the source to a YAML file.""" + with open(path, "w") as outfile: + yaml.dump(source, outfile) + + def get_automations_from_api(self, workspace_id: str) -> Any: + """Returns automations for the workspace as JSON.""" + response: requests.Response = self._api.get_automations(workspace_id) + if response.ok: + return response.json() + else: + raise RuntimeError( + f"Failed to get automations for {workspace_id}. " + + f"{response.status_code}: {response.text}" + ) + + def store_automations(self, export_path: Path, workspace_id: str) -> None: + """Stores the automations in the specified export path.""" + # Get the automations from the API + automations: Any = self.get_automations_from_api(workspace_id) + + automations_folder_path: Path = Path( + export_path, + "gooddata_layouts", + self.org_id, + "workspaces", + workspace_id, + "automations", + ) + + automations_file_path: Path = Path( + automations_folder_path, "automations.json" + ) + + os.mkdir(automations_folder_path) + + # Store the automations in a JSON file + if len(automations["data"]) > 0: + with open(automations_file_path, "w") as f: + json.dump(automations, f) + + def store_declarative_filter_views( + self, export_path: Path, workspace_id: str + ) -> None: + """Stores the filter views in the specified export path.""" + # Get the filter views YAML files from the API + self._api.store_declarative_filter_views(workspace_id, export_path) + + # Move filter views to the subfolder containing analytics model + self.move_folder( + Path(export_path, "gooddata_layouts", self.org_id, "filter_views"), + Path( + export_path, + "gooddata_layouts", + self.org_id, + "workspaces", + workspace_id, + "filter_views", + ), + ) + + def get_workspace_export( + self, + local_target_path: str, + workspaces_to_export: list[str], + ) -> None: + """ + Iterate over all workspaces in the workspaces_to_export list and store + their declarative_workspace and their respective user data filters. + """ + exported = False + for workspace_id in workspaces_to_export: + export_path = Path( + local_target_path, + self.org_id, + workspace_id, + BackupSettings.TIMESTAMP_SDK_FOLDER, + ) + + try: + user_data_filters = self.get_user_data_filters(workspace_id) + except Exception as e: + self.logger.error( + f"Skipping backup of {workspace_id} - check if workspace exists." + + f"{e.__class__.__name__}: {e}" + ) + continue + + try: + # TODO: consider using the API to get JSON declarations in memory + # or check if there is a way to get YAML structures directly from + # the SDK. That way we could save and package all the declarations + # directly instead of reorganizing the folder structures. That should + # be more transparent/readable and possibly safer for threading + self._api.store_declarative_workspace(workspace_id, export_path) + self.store_declarative_filter_views(export_path, workspace_id) + self.store_automations(export_path, workspace_id) + + self.store_user_data_filters( + user_data_filters, export_path, workspace_id + ) + self.logger.info(f"Stored export for {workspace_id}") + exported = True + except Exception as e: + self.logger.error( + f"Skipping {workspace_id}. {e.__class__.__name__} encountered: {e}" + ) + + if not exported: + raise RuntimeError( + "None of the workspaces were exported. Check that the source file " + + "is correct and that the workspaces exist." + ) + + def archive_gooddata_layouts_to_zip(self, folder: str) -> None: + """Archives the gooddata_layouts directory to a zip file.""" + try: + target_subdir = "" + for subdir, dirs, files in os.walk(folder): + if DirNames.LAYOUTS in dirs: + target_subdir = os.path.join(subdir, dirs[0]) + if DirNames.LDM in dirs: + inner_layouts_dir = subdir + "/gooddata_layouts" + os.mkdir(inner_layouts_dir) + for dir in dirs: + shutil.move( + os.path.join(subdir, dir), + os.path.join(inner_layouts_dir), + ) + shutil.make_archive(target_subdir, "zip", subdir) + shutil.rmtree(target_subdir) + except Exception as e: + self.logger.error(f"Error archiving {folder} to zip: {e}") + raise + + def split_to_batches( + self, workspaces_to_export: list[str], batch_size: int + ) -> list[BackupBatch]: + """Splits the list of workspaces to into batches of the specified size. + The batch is respresented as a list of workspace IDs. + Returns a list of batches (i.e. list of lists of IDs) + """ + list_of_batches = [] + while workspaces_to_export: + batch = BackupBatch(workspaces_to_export[:batch_size]) + workspaces_to_export = workspaces_to_export[batch_size:] + list_of_batches.append(batch) + + return list_of_batches + + def process_batch( + self, + batch: BackupBatch, + stop_event: threading.Event, + retry_count: int = 0, + ) -> None: + """Processes a single batch of workspaces for backup. + If the batch processing fails, the function will wait + and retry with exponential backoff up to BackupSettings.MAX_RETRIES. + The base wait time is defined by BackupSettings.RETRY_DELAY. + """ + if stop_event.is_set(): + # If the stop_event flag is set, return. This will terminate the thread. + return + + try: + with tempfile.TemporaryDirectory() as tmpdir: + self.get_workspace_export(tmpdir, batch.list_of_ids) + + self.archive_gooddata_layouts_to_zip( + str(Path(tmpdir, self.org_id)) + ) + + self.storage.export(tmpdir, self.org_id) + + except Exception as e: + if stop_event.is_set(): + return + + elif retry_count < BackupSettings.MAX_RETRIES: + # Retry with exponential backoff until MAX_RETRIES. + next_retry = retry_count + 1 + wait_time = BackupSettings.RETRY_DELAY**next_retry + self.logger.info( + f"{e.__class__.__name__} encountered while processing a batch. " + + f"Retrying {next_retry}/{BackupSettings.MAX_RETRIES} " + + f"in {wait_time} seconds..." + ) + + time.sleep(wait_time) + self.process_batch(batch, stop_event, next_retry) + else: + # If the batch fails after MAX_RETRIES, raise the error. + self.logger.error(f"Batch failed: {e.__class__.__name__}: {e}") + raise + + def process_batches_in_parallel( + self, + batches: list[BackupBatch], + ) -> None: + """ + Processes batches in parallel using concurrent.futures. Will stop the processing + if any one of the batches fails. + """ + + # Create a threading flag to control the threads that have already been started + stop_event = threading.Event() + + with ThreadPoolExecutor( + max_workers=BackupSettings.MAX_WORKERS + ) as executor: + # Set the futures tasks. + futures = [] + for batch in batches: + futures.append( + executor.submit( + self.process_batch, + batch, + stop_event, + ) + ) + + # Process futures as they complete + for future in as_completed(futures): + try: + future.result() + except Exception: + # On failure, set the flag to True - signal running processes to stop. + stop_event.set() + + # Cancel unstarted threads. + for f in futures: + if not f.done(): + f.cancel() + + raise + + def backup_workspaces(self, path_to_csv: str) -> None: + """Runs the backup process for a list of workspace IDs. + + Will read the list of workspace IDs from a CSV file and create backup for + each workspace in storage specified in the configuration. + + Args: + path_to_csv (str): Path to a CSV file containing a list of workspace IDs. + """ + self.backup(InputType.LIST_OF_WORKSPACES, path_to_csv) + + def backup_hierarchies(self, path_to_csv: str) -> None: + """Runs the backup process for a list of hierarchies. + + Will read the list of workspace IDs from a CSV file and create backup for + each those workspaces' hierarchies in storage specified in the configuration. + Workspace hierarchy means the workspace itself and all its direct and + indirect children. + + Args: + path_to_csv (str): Path to a CSV file containing a list of workspace IDs. + """ + self.backup(InputType.HIERARCHY, path_to_csv) + + def backup_entire_organization(self) -> None: + """Runs the backup process for the entire organization. + + Will create backup for all workspaces in the organization in storage + specified in the configuration. + """ + self.backup(InputType.ORGANIZATION) + + def backup( + self, input_type: InputType, path_to_csv: str | None = None + ) -> None: + """Runs the backup process with selected input type.""" + try: + workspaces_to_export: list[str] = self.loader.get_ids_to_backup( + input_type, path_to_csv + ) + batches = self.split_to_batches( + workspaces_to_export, self.config.batch_size + ) + + self.logger.info( + f"Exporting {len(workspaces_to_export)} workspaces in {len(batches)} batches." + ) + + self.process_batches_in_parallel(batches) + + self.logger.info("Backup completed") + except Exception as e: + self.logger.error(f"Backup failed: {e.__class__.__name__}: {e}") + self.logger.error(traceback.format_exc()) + raise diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/constants.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/constants.py new file mode 100644 index 0000000..27a9ce2 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/constants.py @@ -0,0 +1,42 @@ +import datetime +from dataclasses import dataclass + +from gooddata_sdk._version import __version__ as sdk_version + + +@dataclass(frozen=True) +class DirNames: + """ + Folder names used in the SDK backup process: + - LAYOUTS - GoodData Layouts + - LDM - Logical Data Model + - AM - Analytics Model + - UDF - User Data Filters + """ + + LAYOUTS = "gooddata_layouts" + LDM = "ldm" + AM = "analytics_model" + UDF = "user_data_filters" + + +@dataclass(frozen=True) +class ConcurrencyDefaults: + MAX_WORKERS = 2 + DEFAULT_BATCH_SIZE = 100 + + +@dataclass(frozen=True) +class ApiDefaults: + DEFAULT_PAGE_SIZE = 100 + + +@dataclass(frozen=True) +class BackupSettings(ConcurrencyDefaults, ApiDefaults): + MAX_RETRIES = 3 + RETRY_DELAY = 5 # seconds + TIMESTAMP_SDK_FOLDER = ( + str(datetime.datetime.now().strftime("%Y%m%d-%H%M%S")) + + "-" + + sdk_version.replace(".", "_") + ) diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/csv_reader.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/csv_reader.py new file mode 100644 index 0000000..0e37545 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/csv_reader.py @@ -0,0 +1,41 @@ +# (C) 2025 GoodData Corporation + +import csv +from typing import Iterator + + +class CSVReader: + """Class to read the input CSV file and return its content as a list of strings.""" + + @staticmethod + def read_backup_csv(file_path: str) -> list[str]: + """Reads the input CSV file, validates its structure, and returns its + content as a list of strings. + """ + + with open(file_path) as csv_file: + reader: Iterator[list[str]] = csv.reader( + csv_file, skipinitialspace=True + ) + + try: + # Skip the header + headers = next(reader) + + if len(headers) > 1: + raise ValueError( + "Input file contains more than one column. Please check the input and try again." + ) + + except StopIteration: + # Raise an error if the iterator is empty + raise ValueError("No content found in the CSV file.") + + # Read the content + content = [row[0] for row in reader] + + # If the content is empty (no rows), raise an error + if not content: + raise ValueError("No workspaces found in the CSV file.") + + return content diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/__init__.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/input_type.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/input_type.py new file mode 100644 index 0000000..23b2f1c --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/input_type.py @@ -0,0 +1,11 @@ +# (C) 2025 GoodData Corporation + +from enum import Enum + + +class InputType(Enum): + """Input type for the backup.""" + + LIST_OF_WORKSPACES = "list-of-workspaces" + HIERARCHY = "list-of-parents" + ORGANIZATION = "entire-organization" diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/storage.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/storage.py new file mode 100644 index 0000000..64a6337 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/storage.py @@ -0,0 +1,58 @@ +# (C) 2025 GoodData Corporation + +from enum import Enum +from typing import Annotated, TypeAlias + +import yaml +from pydantic import BaseModel, Field + +from gooddata_pipelines.backup_and_restore.constants import BackupSettings + + +class StorageType(Enum): + """Type of storage.""" + + S3 = "s3" + LOCAL = "local" + + +class S3StorageConfig(BaseModel): + """Configuration for S3 storage.""" + + backup_path: str + bucket: str + profile: str = "default" + + +class LocalStorageConfig(BaseModel): + """Placeholder for local storage config.""" + + +StorageConfig: TypeAlias = S3StorageConfig | LocalStorageConfig + + +class BackupRestoreConfig(BaseModel): + """Configuration for backup and restore.""" + + storage_type: StorageType + storage: StorageConfig | None = Field(default=None) + api_page_size: Annotated[ + int, + Field( + gt=0, + description="Page size must be greater than 0", + ), + ] = Field(default=BackupSettings.DEFAULT_PAGE_SIZE) + batch_size: Annotated[ + int, + Field( + gt=0, + description="Batch size must be greater than 0", + ), + ] = Field(default=BackupSettings.DEFAULT_BATCH_SIZE) + + @classmethod + def from_yaml(cls, conf_path: str) -> "BackupRestoreConfig": + with open(conf_path, "r") as stream: + conf: dict = yaml.safe_load(stream) + return cls(**conf) diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/workspace_response.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/workspace_response.py new file mode 100644 index 0000000..a1b8fef --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/models/workspace_response.py @@ -0,0 +1,51 @@ +# (C) 2025 GoodData Corporation + +from pydantic import ( + BaseModel, + ConfigDict, +) +from pydantic.alias_generators import ( + to_camel, +) + + +class Page(BaseModel): + size: int + total_elements: int + total_pages: int + number: int + + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + ) + + +class Hierarchy(BaseModel): + children_count: int + + model_config = ConfigDict( + alias_generator=to_camel, + populate_by_name=True, + ) + + +class Meta(BaseModel): + page: Page | None = None + hierarchy: Hierarchy | None = None + + +class Workspace(BaseModel): + id: str + meta: Meta | None = None + + +class Links(BaseModel): + self: str + next: str | None = None + + +class WorkspaceResponse(BaseModel): + data: list[Workspace] + links: Links + meta: Meta diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/storage/__init__.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/storage/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/storage/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/storage/base_storage.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/storage/base_storage.py new file mode 100644 index 0000000..fa28977 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/storage/base_storage.py @@ -0,0 +1,18 @@ +# (C) 2025 GoodData Corporation + +import abc + +from gooddata_pipelines.backup_and_restore.models.storage import ( + BackupRestoreConfig, +) +from gooddata_pipelines.logger import LogObserver + + +class BackupStorage(abc.ABC): + def __init__(self, conf: BackupRestoreConfig): + self.logger = LogObserver() + + @abc.abstractmethod + def export(self, folder, org_id): + """Exports the content of the folder to the storage.""" + raise NotImplementedError diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/storage/local_storage.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/storage/local_storage.py new file mode 100644 index 0000000..b2d28c3 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/storage/local_storage.py @@ -0,0 +1,33 @@ +# (C) 2025 GoodData Corporation + +import shutil +from pathlib import Path + +from gooddata_pipelines.backup_and_restore.models.storage import ( + BackupRestoreConfig, +) +from gooddata_pipelines.backup_and_restore.storage.base_storage import ( + BackupStorage, +) + + +class LocalStorage(BackupStorage): + def __init__(self, conf: BackupRestoreConfig): + super().__init__(conf) + + def _export(self, folder, org_id, export_folder="local_backups") -> None: + """Copies the content of the folder to local storage as backup.""" + self.logger.info(f"Saving {org_id} to local storage") + shutil.copytree( + Path(folder), Path(Path.cwd(), export_folder), dirs_exist_ok=True + ) + + def export(self, folder, org_id, export_folder="local_backups") -> None: + """Copies the content of the folder to local storage as backup.""" + try: + self._export(folder, org_id, export_folder) + except Exception as e: + self.logger.error( + f"Error exporting {folder} to {export_folder}: {e}" + ) + raise diff --git a/gooddata-pipelines/gooddata_pipelines/backup_and_restore/storage/s3_storage.py b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/storage/s3_storage.py new file mode 100644 index 0000000..c0c1f85 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/backup_and_restore/storage/s3_storage.py @@ -0,0 +1,80 @@ +# (C) 2025 GoodData Corporation + +import os + +import boto3 + +from gooddata_pipelines.backup_and_restore.models.storage import ( + BackupRestoreConfig, + S3StorageConfig, +) +from gooddata_pipelines.backup_and_restore.storage.base_storage import ( + BackupStorage, +) + + +class S3Storage(BackupStorage): + def __init__(self, conf: BackupRestoreConfig): + super().__init__(conf) + + if not isinstance(conf.storage, S3StorageConfig): + raise ValueError("S3 storage config is required") + + self._config = conf.storage + self._profile = self._config.profile + self._session = self._create_boto_session(self._profile) + self._resource = self._session.resource("s3") + self._bucket = self._resource.Bucket(self._config.bucket) # type: ignore [missing library stubs] + suffix = "/" if not self._config.backup_path.endswith("/") else "" + self._backup_path = self._config.backup_path + suffix + + self._verify_connection() + + def _create_boto_session(self, profile: str) -> boto3.Session: + try: + return boto3.Session(profile_name=profile) + except Exception: + self.logger.warning( + 'AWS profile "[default]" not found. Trying other fallback methods...' + ) + + return boto3.Session() + + def _verify_connection(self) -> None: + """ + Pings the S3 bucket to verify that the connection is working. + """ + try: + # TODO: install boto3 s3 stubs + self._resource.meta.client.head_bucket(Bucket=self._config.bucket) + except Exception as e: + raise RuntimeError( + f"Failed to connect to S3 bucket {self._config.bucket}: {e}" + ) + + def export(self, folder, org_id) -> None: + """Uploads the content of the folder to S3 as backup.""" + storage_path = self._config.bucket + "/" + self._backup_path + self.logger.info(f"Uploading {org_id} to {storage_path}") + folder = folder + "/" + org_id + for subdir, dirs, files in os.walk(folder): + full_path = os.path.join(subdir) + export_path = ( + self._backup_path + + org_id + + "/" + + full_path[len(folder) + 1 :] + + "/" + ) + self._bucket.put_object(Key=export_path) + + for file in files: + full_path = os.path.join(subdir, file) + with open(full_path, "rb") as data: + export_path = ( + self._backup_path + + org_id + + "/" + + full_path[len(folder) + 1 :] + ) + self._bucket.put_object(Key=export_path, Body=data) diff --git a/gooddata-pipelines/gooddata_pipelines/logger/__init__.py b/gooddata-pipelines/gooddata_pipelines/logger/__init__.py new file mode 100644 index 0000000..b19b689 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/logger/__init__.py @@ -0,0 +1,8 @@ +# (C) 2025 GoodData Corporation + +from .logger import LoggerLike, LogObserver + +__all__ = [ + "LoggerLike", + "LogObserver", +] diff --git a/gooddata-pipelines/gooddata_pipelines/logger/logger.py b/gooddata-pipelines/gooddata_pipelines/logger/logger.py new file mode 100644 index 0000000..5acc112 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/logger/logger.py @@ -0,0 +1,115 @@ +# (C) 2025 GoodData Corporation + +"""Logging observer for the GoodData Pipelines SDK. + +This module provides a singleton observer class `LogObserver` that allows +subscribing logger-like objects to receive log messages. The observer emits +unformatted log messages to all subscribed objects, which should implement +the `LoggerLike` protocol. +""" + +from enum import Enum +from typing import Any, Protocol + + +class SingletonMeta(type): + _instances: dict = {} + + def __call__(cls, *args: Any, **kwargs: Any) -> "SingletonMeta": + if cls not in cls._instances: + instance = super().__call__(*args, **kwargs) + cls._instances[cls] = instance + return cls._instances[cls] + + +class Severity(Enum): + """Severity levels for logging.""" + + INFO = "info" + WARNING = "warning" + ERROR = "error" + + +class LoggerLike(Protocol): + """A protocol for a logger-like object. + + This protocol defines the methods that a logger-like object should implement + to be compatible with the `LogObserver`. It includes methods for logging + messages at different severity levels: info, warning, and error. + """ + + def info(self, *args: Any, **kwargs: Any) -> None: ... + + def warning(self, *args: Any, **kwargs: Any) -> None: ... + + def error(self, *args: Any, **kwargs: Any) -> None: ... + + +class LogObserver(metaclass=SingletonMeta): + """Singleton observer class for logging messages. + + Emits unformatted log messages to all subscribed logger-like objects. + """ + + # TODO: in future we might want to add a timestamp or other metadata + # (severity...)? Currently that is left out to subscribers to handle. + + # TODO: with error we're dumping the context as string to the message + # that could be improved (either passing the context as a separate arg + # or handling the process here). + + def __init__(self) -> None: + self.subscribers: list[LoggerLike] = [] + + def subscribe(self, subscriber: LoggerLike) -> None: + """Subscribe a logger-like object to receive log messages. + + Args: + subscriber (LoggerLike): An object that implements the LoggerLike + protocol. + Returns: + None + """ + self.subscribers.append(subscriber) + + def unsubscribe(self, subscriber: LoggerLike) -> None: + """Unsubscribe a logger-like object from receiving log messages. + + Args: + subscriber (LoggerLike): An object that implements the LoggerLike + protocol. + + Returns: + None + """ + self.subscribers.remove(subscriber) + + def _notify(self, severity: Severity, msg: str) -> None: + """Notify all subscribers with a log message. + + Args: + severity (Severity): The severity level of the log message. + msg (str): The log message to be sent to subscribers. + + Returns: + None + """ + for subscriber in self.subscribers: + if severity == Severity.INFO: + subscriber.info(msg) + elif severity == Severity.WARNING: + subscriber.warning(msg) + elif severity == Severity.ERROR: + subscriber.error(msg) + + def info(self, msg: str) -> None: + """Log an info message.""" + self._notify(Severity.INFO, msg) + + def warning(self, msg: str) -> None: + """Log a warning message.""" + self._notify(Severity.WARNING, msg) + + def error(self, msg: str) -> None: + """Log an error message.""" + self._notify(Severity.ERROR, msg) diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/__init__.py b/gooddata-pipelines/gooddata_pipelines/provisioning/__init__.py new file mode 100644 index 0000000..03d7625 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/__init__.py @@ -0,0 +1,31 @@ +# (C) 2025 GoodData Corporation + +from .entities.users.models.permissions import ( + PermissionFullLoad, + PermissionIncrementalLoad, +) +from .entities.users.models.user_groups import ( + UserGroupFullLoad, + UserGroupIncrementalLoad, +) +from .entities.users.models.users import ( + UserFullLoad, + UserIncrementalLoad, +) +from .entities.users.permissions import PermissionProvisioner +from .entities.users.user_groups import UserGroupProvisioner +from .entities.users.users import UserProvisioner +from .entities.workspaces.workspace import WorkspaceProvisioner + +__all__ = [ + "PermissionFullLoad", + "PermissionIncrementalLoad", + "PermissionProvisioner", + "UserFullLoad", + "UserGroupFullLoad", + "UserIncrementalLoad", + "UserGroupIncrementalLoad", + "UserGroupProvisioner", + "UserProvisioner", + "WorkspaceProvisioner", +] diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/assets/wdf_setting.json b/gooddata-pipelines/gooddata_pipelines/provisioning/assets/wdf_setting.json new file mode 100644 index 0000000..cf9715a --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/assets/wdf_setting.json @@ -0,0 +1,14 @@ +{ + "data": { + "attributes": { + "filterValues": [] + }, + "id": "", + "relationships": { + "workspaceDataFilter": { + "data": { "id": "", "type": "workspaceDataFilter" } + } + }, + "type": "workspaceDataFilterSetting" + } +} diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/__init__.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/__init__.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/models/__init__.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/models/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/models/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/models/udf_models.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/models/udf_models.py new file mode 100644 index 0000000..b655e08 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/models/udf_models.py @@ -0,0 +1,29 @@ +# (C) 2025 GoodData Corporation + +"""This module defines data models for user data filters in a GoodData workspace.""" + +from dataclasses import dataclass, field + + +@dataclass +class UserDataFilterGroup: + udf_id: str + udf_values: list[str] + + +@dataclass +class WorkspaceUserDataFilters: + workspace_id: str + user_data_filters: list["UserDataFilterGroup"] = field(default_factory=list) + + +@dataclass +class UserDataFilterFullLoad: + workspace_id: str + udf_id: str + udf_value: str + + +@dataclass +class UserDataFilterIncrementalLoad(UserDataFilterFullLoad): + is_active: bool diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/user_data_filters.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/user_data_filters.py new file mode 100644 index 0000000..d7e356d --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/user_data_filters/user_data_filters.py @@ -0,0 +1,221 @@ +# (C) 2025 GoodData Corporation + +"""Module for provisioning user data filters in GoodData workspaces. + +This module provides the `UserDataFilterProvisioner` class, which is responsible +for creating, updating, and deleting user data filters in GoodData workspaces. +""" + +import re + +from gooddata_sdk.catalog.workspace.entity_model.user_data_filter import ( + CatalogEntityIdentifier, + CatalogUserDataFilter, + CatalogUserDataFilterAttributes, + CatalogUserDataFilterRelationships, +) + +from gooddata_pipelines.provisioning.entities.user_data_filters.models.udf_models import ( + UserDataFilterFullLoad, + UserDataFilterGroup, + UserDataFilterIncrementalLoad, + WorkspaceUserDataFilters, +) +from gooddata_pipelines.provisioning.provisioning import Provisioning +from gooddata_pipelines.provisioning.utils.exceptions import ContextException + + +class UserDataFilterProvisioner( + Provisioning[UserDataFilterFullLoad, UserDataFilterIncrementalLoad] +): + """Provisioning class for user data filters in GoodData workspaces. + + This class handles the creation, update, and deletion of user data filters + based on the provided source data. + + Requires setting the `ldm_column_name` and `maql_column_name` + attributes before calling the `provision` method. + + Usage: + ``` + provisioner = UserDataFilterProvisioner(api, source_group) + provisioner.set_ldm_column_name("ldm_column") + provisioner.set_maql_column_name("maql_column") + provisioner.provision() + ``` + """ + + source_group_full: list[UserDataFilterFullLoad] + source_group_incremental: list[UserDataFilterIncrementalLoad] + ldm_column_name: str = "" + maql_column_name: str = "" + + def set_ldm_column_name(self, ldm_column_name: str) -> None: + """Set the LDM column name for user data filters. + + Args: + ldm_column_name (str): The LDM column name to set. + """ + self.ldm_column_name = ldm_column_name + + def set_maql_column_name(self, maql_column_name: str) -> None: + """Set the MAQL column name for user data filters. + + Args: + maql_column_name (str): The MAQL column name to set. + """ + self.maql_column_name = maql_column_name + + @staticmethod + def _group_db_user_data_filters_by_ws_id( + user_data_filters: list[UserDataFilterFullLoad], + ) -> list[WorkspaceUserDataFilters]: + """Group user data filters by workspace ID and user ID.""" + ws_map: dict[str, dict[str, set[str]]] = {} + + for udf in user_data_filters: + ws_map.setdefault(udf.workspace_id, {}).setdefault( + udf.udf_id, set() + ).add(str(udf.udf_value)) + + result: list[WorkspaceUserDataFilters] = [] + + for ws_id, udf_dict in ws_map.items(): + udf_groups = [ + UserDataFilterGroup(udf_id=udf_id, udf_values=list(values)) + for udf_id, values in udf_dict.items() + ] + result.append( + WorkspaceUserDataFilters( + workspace_id=ws_id, user_data_filters=udf_groups + ) + ) + return result + + @staticmethod + def _extract_numbers_from_maql(maql: str) -> list[str]: + """Extract numbers from a MAQL string.""" + numbers = re.findall(r'"\d+"', maql) + return [number.strip('"') for number in numbers] + + def _skip_user_data_filter_update( + self, existing_udf: list[CatalogUserDataFilter], udf_value: list[str] + ) -> bool: + """Check if the user data filter update can be skipped.""" + if not existing_udf: + return False + existing_udfs = self._extract_numbers_from_maql( + existing_udf[0].attributes.maql + ) + return set(udf_value) == set(existing_udfs) + + def _create_user_data_filters( + self, user_data_filter_ids_to_create: list[WorkspaceUserDataFilters] + ) -> None: + """Create or update user data filters in GoodData workspaces.""" + for workspace_user_data_filter in user_data_filter_ids_to_create: + workspace_id = workspace_user_data_filter.workspace_id + user_data_filters = workspace_user_data_filter.user_data_filters + + gd_user_data_filters: list[CatalogUserDataFilter] = ( + self._api.list_user_data_filters(workspace_id) + ) + + gd_udf_ids = { + user.relationships.user["data"].id + for user in gd_user_data_filters + if user.relationships and user.relationships.user + } + + db_udf_ids = {udf.udf_id for udf in user_data_filters} + + udf_ids_to_delete: set[str] = gd_udf_ids.difference(db_udf_ids) + self._delete_user_data_filters(workspace_id, udf_ids_to_delete) + + udf_group: UserDataFilterGroup + for udf_group in user_data_filters: + udf_id: str = udf_group.udf_id + udf_values: list[str] = udf_group.udf_values + + existing_udf: list[CatalogUserDataFilter] = [ + udf for udf in gd_user_data_filters if udf.id == udf_id + ] + if self._skip_user_data_filter_update(existing_udf, udf_values): + continue + + formatted_udf_values = '", "'.join( + str(value) for value in udf_values + ) + maql = f'{self.maql_column_name} IN ("{formatted_udf_values}")' + + attributes = CatalogUserDataFilterAttributes(maql=maql) + relationships = CatalogUserDataFilterRelationships( + labels={ + "data": [ + CatalogEntityIdentifier( + id=self.ldm_column_name, type="label" + ) + ] + }, + user={ + "data": CatalogEntityIdentifier(id=udf_id, type="user") + }, + ) + user_data_filter = CatalogUserDataFilter( + id=udf_id, + attributes=attributes, + relationships=relationships, + ) + + try: + self._api.create_or_update_user_data_filter( + workspace_id, user_data_filter + ) + self.logger.info( + "Created or updated user data filters for user with id " + + f"{udf_id} for client with id {workspace_id}" + ) + except Exception as e: + raise ContextException( + f"Failed to create user data filters: {e}", + udf_group, + user_data_filter, + ) from e + + def _delete_user_data_filters( + self, workspace_id: str, udf_ids_to_delete: set[str] + ) -> None: + """Delete user data filters in GoodData workspaces.""" + for udf_id in udf_ids_to_delete: + try: + self._api.delete_user_data_filter(workspace_id, udf_id) + self.logger.info( + f"Deleted user data filters for user with id {udf_id}" + ) + except Exception as e: + raise ContextException( + f"Failed to delete user data filters: {e}" + ) from e + + def _provision_full_load(self) -> None: + """Provision user data filters in GoodData workspaces.""" + + if not self.maql_column_name: + raise ContextException( + "MAQL column name is not set. Please set it before provisioning." + ) + if not self.ldm_column_name: + raise ContextException( + "LDM column name is not set. Please set it before provisioning." + ) + + grouped_db_user_data_filters = ( + self._group_db_user_data_filters_by_ws_id(self.source_group_full) + ) + self._create_user_data_filters(grouped_db_user_data_filters) + + self.logger.info("User data filters provisioning completed") + + def _provision_incremental_load(self) -> None: + """Provision user data filters in GoodData workspaces.""" + raise NotImplementedError("Not implemented yet.") diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/__init__.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/__init__.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/permissions.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/permissions.py new file mode 100644 index 0000000..84325d7 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/permissions.py @@ -0,0 +1,241 @@ +# (C) 2025 GoodData Corporation +from dataclasses import dataclass +from enum import Enum +from typing import Any, Iterator, TypeAlias + +from gooddata_sdk.catalog.identifier import CatalogAssigneeIdentifier +from gooddata_sdk.catalog.permission.declarative_model.permission import ( + CatalogDeclarativeSingleWorkspacePermission, + CatalogDeclarativeWorkspacePermissions, +) + +from gooddata_pipelines.provisioning.utils.exceptions import BaseUserException + +# TODO: refactor the full load and incremental load models to reuse as much as possible +# TODO: use pydantic models instead of dataclasses? + +TargetsPermissionDict: TypeAlias = dict[str, dict[str, bool]] + + +class PermissionType(Enum): + user = "user" + user_group = "userGroup" + + +@dataclass(frozen=True) +class PermissionIncrementalLoad: + permission: str + workspace_id: str + id: str + type: PermissionType + is_active: bool + + @classmethod + def from_list_of_dicts( + cls, data: list[dict[str, Any]] + ) -> list["PermissionIncrementalLoad"]: + """Creates a list of User objects from list of dicts.""" + permissions = [] + for permission in data: + id = ( + permission["user_id"] + if permission["user_id"] + else permission["ug_id"] + ) + + if permission["user_id"]: + target_type = PermissionType.user + else: + target_type = PermissionType.user_group + + permissions.append( + PermissionIncrementalLoad( + permission=permission["ws_permissions"], + workspace_id=permission["ws_id"], + id=id, + type=target_type, + is_active=str(permission["is_active"]).lower() == "true", + ) + ) + return permissions + + +@dataclass(frozen=True) +class PermissionFullLoad: + permission: str + workspace_id: str + id: str + type: PermissionType + + @classmethod + def from_list_of_dicts( + cls, data: list[dict[str, Any]] + ) -> list["PermissionFullLoad"]: + """Creates a list of User objects from list of dicts.""" + permissions = [] + for permission in data: + id = ( + permission["user_id"] + if permission["user_id"] + else permission["ug_id"] + ) + + if permission["user_id"]: + target_type = PermissionType.user + else: + target_type = PermissionType.user_group + + permissions.append( + PermissionFullLoad( + permission=permission["ws_permissions"], + workspace_id=permission["ws_id"], + id=id, + type=target_type, + ) + ) + return permissions + + +@dataclass +class PermissionDeclaration: + users: TargetsPermissionDict + user_groups: TargetsPermissionDict + + @classmethod + def from_sdk_api( + cls, declaration: CatalogDeclarativeWorkspacePermissions + ) -> "PermissionDeclaration": + """ + Constructs an WSPermissionDeclaration instance + from GoodData SDK CatalogDeclarativeWorkspacePermissions. + """ + users: TargetsPermissionDict = {} + user_groups: TargetsPermissionDict = {} + + for permission in declaration.permissions: + permission_type, id = ( + permission.assignee.type, + permission.assignee.id, + ) + + if permission_type == PermissionType.user.value: + target_dict = users + else: + target_dict = user_groups + + id_permissions = target_dict.get(id) + if not id_permissions: + target_dict[id] = dict() + + target_dict[id][permission.name] = True + + return PermissionDeclaration(users, user_groups) + + @staticmethod + def _construct_upstream_permission( + permission: str, assignee: CatalogAssigneeIdentifier + ) -> CatalogDeclarativeSingleWorkspacePermission | None: + """Constructs single permission declaration for the SDK API.""" + try: + return CatalogDeclarativeSingleWorkspacePermission( + name=permission, assignee=assignee + ) + except Exception as e: + raise BaseUserException( + f"Failed to construct SDK declaration for type={assignee.type} ", + f"id={assignee.id}. Error: {e}", + ) + + def _permissions_for_target( + self, permissions: dict[str, bool], assignee: CatalogAssigneeIdentifier + ) -> Iterator[CatalogDeclarativeSingleWorkspacePermission]: + """Constructs permission declarations for a single target.""" + for permission, is_active in permissions.items(): + if not is_active: + continue + declaration = self._construct_upstream_permission( + permission, assignee + ) + if not declaration: + continue + yield declaration + + def to_sdk_api(self) -> CatalogDeclarativeWorkspacePermissions: + """ + Constructs the GoodData SDK CatalogDeclarativeWorkspacePermissions + object from the WSPermissionDeclaration instance. + """ + permission_declarations: list[ + CatalogDeclarativeSingleWorkspacePermission + ] = [] + + for user_id, permissions in self.users.items(): + assignee = CatalogAssigneeIdentifier( + id=user_id, type=PermissionType.user.value + ) + for declaration in self._permissions_for_target( + permissions, assignee + ): + permission_declarations.append(declaration) + + for ug_id, permissions in self.user_groups.items(): + assignee = CatalogAssigneeIdentifier( + id=ug_id, type=PermissionType.user_group.value + ) + for declaration in self._permissions_for_target( + permissions, assignee + ): + permission_declarations.append(declaration) + + return CatalogDeclarativeWorkspacePermissions( + permissions=permission_declarations + ) + + def add_permission(self, permission: PermissionIncrementalLoad) -> None: + """ + Adds WSPermission object into respective field within the instance. + Handles duplicate permissions and different combinations of input + and upstream is_active permission states. + """ + target_dict = ( + self.users + if permission.type == PermissionType.user + else self.user_groups + ) + + if permission.id not in target_dict: + target_dict[permission.id] = {} + + is_active = permission.is_active + target_permissions = target_dict[permission.id] + permission_value = permission.permission + + if permission_value not in target_permissions: + target_permissions[permission_value] = is_active + elif not is_active and target_permissions[permission_value] is True: + print( + "isActive=False provided after True has been specificed for the " + + f"same input. Skipping '{permission}'" + ) + elif is_active and target_permissions[permission_value] is False: + print( + "isActive=True provided after False has been specified for the " + + f"same input. Overwriting '{permission}'" + ) + target_permissions[permission_value] = is_active + + def upsert(self, other: "PermissionDeclaration") -> None: + """ + Modifies the owner object by merging with the other. + Keeps the unmodified users/userGroups untouched. + If some user/userGroup is modified, it gets overwritten with permissions + defined in the input. + """ + for user_id, permissions in other.users.items(): + self.users[user_id] = permissions + + for ug_id, permissions in other.user_groups.items(): + self.user_groups[ug_id] = permissions + + +WSPermissionsDeclarations: TypeAlias = dict[str, PermissionDeclaration] diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/user_groups.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/user_groups.py new file mode 100644 index 0000000..015b7af --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/user_groups.py @@ -0,0 +1,64 @@ +# (C) 2025 GoodData Corporation + +from typing import Any + +from pydantic import BaseModel + +from gooddata_pipelines.provisioning.utils.utils import SplitMixin + + +class BaseUserGroup(BaseModel, SplitMixin): + user_group_id: str + user_group_name: str + parent_user_groups: list[str] + + @classmethod + def _create_from_dict_data( + cls, user_group_data: dict[str, Any], delimiter: str = "," + ) -> dict[str, Any]: + """Helper method to extract common data from dict.""" + parent_user_groups = cls.split( + user_group_data["parent_user_groups"], delimiter=delimiter + ) + user_group_name = user_group_data["user_group_name"] + if not user_group_name: + user_group_name = user_group_data["user_group_id"] + + return { + "user_group_id": user_group_data["user_group_id"], + "user_group_name": user_group_name, + "parent_user_groups": parent_user_groups, + } + + +class UserGroupIncrementalLoad(BaseUserGroup): + is_active: bool + + @classmethod + def from_list_of_dicts( + cls, data: list[dict[str, Any]], delimiter: str = "," + ) -> list["UserGroupIncrementalLoad"]: + """Creates a list of User objects from list of dicts.""" + user_groups = [] + for user_group in data: + base_data = cls._create_from_dict_data(user_group, delimiter) + base_data["is_active"] = user_group["is_active"] + + user_groups.append(UserGroupIncrementalLoad(**base_data)) + + return user_groups + + +class UserGroupFullLoad(BaseUserGroup): + @classmethod + def from_list_of_dicts( + cls, data: list[dict[str, Any]], delimiter: str = "," + ) -> list["UserGroupFullLoad"]: + """Creates a list of User objects from list of dicts.""" + user_groups = [] + for user_group in data: + base_data = cls._create_from_dict_data(user_group, delimiter) + + user_groups.append(UserGroupFullLoad(**base_data)) + + return user_groups diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/users.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/users.py new file mode 100644 index 0000000..b027834 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/models/users.py @@ -0,0 +1,114 @@ +# (C) 2025 GoodData Corporation + +from typing import Any + +from gooddata_sdk.catalog.user.entity_model.user import CatalogUser +from pydantic import BaseModel + +from gooddata_pipelines.provisioning.utils.utils import SplitMixin + + +class BaseUser(BaseModel, SplitMixin): + """Base class containing shared user fields and functionality.""" + + user_id: str + firstname: str | None + lastname: str | None + email: str | None + auth_id: str | None + user_groups: list[str] + + @classmethod + def _create_from_dict_data( + cls, user_data: dict[str, Any], delimiter: str = "," + ) -> dict[str, Any]: + """Helper method to extract common data from dict.""" + user_groups = cls.split(user_data["user_groups"], delimiter=delimiter) + return { + "user_id": user_data["user_id"], + "firstname": user_data["firstname"], + "lastname": user_data["lastname"], + "email": user_data["email"], + "auth_id": user_data["auth_id"], + "user_groups": user_groups, + } + + @classmethod + def _create_from_sdk_data(cls, obj: CatalogUser) -> dict[str, Any]: + """Helper method to extract common data from SDK object.""" + if obj.attributes: + firstname = obj.attributes.firstname + lastname = obj.attributes.lastname + email = obj.attributes.email + auth_id = obj.attributes.authentication_id + else: + firstname = None + lastname = None + email = None + auth_id = None + + return { + "user_id": obj.id, + "firstname": firstname, + "lastname": lastname, + "email": email, + "auth_id": auth_id, + "user_groups": [ug.id for ug in obj.user_groups], + } + + def to_sdk_obj(self) -> CatalogUser: + """Converts to CatalogUser SDK object.""" + return CatalogUser.init( + user_id=self.user_id, + firstname=self.firstname, + lastname=self.lastname, + email=self.email, + authentication_id=self.auth_id, + user_group_ids=self.user_groups, + ) + + +class UserIncrementalLoad(BaseUser): + """User model for incremental load operations with active status tracking.""" + + is_active: bool + + @classmethod + def from_list_of_dicts( + cls, data: list[dict[str, Any]], delimiter: str = "," + ) -> list["UserIncrementalLoad"]: + """Creates a list of User objects from list of dicts.""" + converted_users = [] + for user in data: + base_data = cls._create_from_dict_data(user, delimiter) + base_data["is_active"] = user["is_active"] + converted_users.append(cls(**base_data)) + return converted_users + + @classmethod + def from_sdk_obj(cls, obj: CatalogUser) -> "UserIncrementalLoad": + """Creates GDUserTarget from CatalogUser SDK object.""" + base_data = cls._create_from_sdk_data(obj) + base_data["is_active"] = True + return cls(**base_data) + + +class UserFullLoad(BaseUser): + """User model for full load operations.""" + + @classmethod + def from_list_of_dicts( + cls, data: list[dict[str, Any]], delimiter: str = "," + ) -> list["UserFullLoad"]: + """Creates a list of User objects from list of dicts.""" + converted_users = [] + for user in data: + base_data = cls._create_from_dict_data(user, delimiter) + converted_users.append(cls(**base_data)) + return converted_users + + @classmethod + def from_sdk_obj(cls, obj: CatalogUser) -> "UserFullLoad": + """Creates GDUserTarget from CatalogUser SDK object.""" + base_data = cls._create_from_sdk_data(obj) + return cls(**base_data) diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/permissions.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/permissions.py new file mode 100644 index 0000000..fa773a7 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/permissions.py @@ -0,0 +1,153 @@ +# (C) 2025 GoodData Corporation + +"""Module for provisioning user permissions in GoodData workspaces.""" + +from gooddata_pipelines.api.exceptions import GoodDataApiException +from gooddata_pipelines.provisioning.entities.users.models.permissions import ( + PermissionDeclaration, + PermissionFullLoad, + PermissionIncrementalLoad, + PermissionType, + TargetsPermissionDict, + WSPermissionsDeclarations, +) +from gooddata_pipelines.provisioning.provisioning import Provisioning +from gooddata_pipelines.provisioning.utils.exceptions import BaseUserException + + +class PermissionProvisioner( + Provisioning[PermissionFullLoad, PermissionIncrementalLoad] +): + """Provisioning class for user permissions in GoodData workspaces. + + This class handles the provisioning of user permissions based on the provided + source data. + """ + + source_group_incremental: list[PermissionIncrementalLoad] + source_group_full: list[PermissionFullLoad] + + def _get_ws_declaration(self, ws_id: str) -> PermissionDeclaration: + users: TargetsPermissionDict = {} + user_groups: TargetsPermissionDict = {} + + upstream_declaration = self._api.get_declarative_permissions(ws_id) + + for permission in upstream_declaration.permissions: + permission_type, id = ( + permission.assignee.type, + permission.assignee.id, + ) + target_dict = ( + users + if permission_type == PermissionType.user.value + else user_groups + ) + + id_permissions = target_dict.get(id) + if not id_permissions: + target_dict[id] = dict() + + target_dict[id][permission.name] = True + + return PermissionDeclaration(users, user_groups) + + def _get_upstream_declaration( + self, ws_id: str + ) -> PermissionDeclaration | None: + """Retrieves upstream permission declaration for a workspace.""" + declaration = self._api.get_declarative_permissions(ws_id) + return PermissionDeclaration.from_sdk_api(declaration) + + def _get_upstream_declarations( + self, input_ws_ids: list[str] + ) -> WSPermissionsDeclarations: + """Retrieves upstream permission declarations for a list of workspaces.""" + ws_dict: WSPermissionsDeclarations = {} + for ws_id in input_ws_ids: + declaration = self._get_upstream_declaration(ws_id) + if declaration: + ws_dict[ws_id] = declaration + return ws_dict + + @staticmethod + def _construct_declarations( + permissions: list[PermissionIncrementalLoad], + ) -> WSPermissionsDeclarations: + """Constructs workspace permission declarations from the input permissions.""" + ws_dict: WSPermissionsDeclarations = {} + for permission in permissions: + ws_id = permission.workspace_id + + if ws_id not in ws_dict: + ws_dict[ws_id] = PermissionDeclaration({}, {}) + + ws_dict[ws_id].add_permission(permission) + return ws_dict + + def _check_user_group_exists(self, ug_id: str) -> None: + """Checks if user group with provided ID exists.""" + self._api._sdk.catalog_user.get_user_group(ug_id) + + def _validate_permission( + self, permission: PermissionIncrementalLoad + ) -> None: + """Validates if the permission is correctly defined.""" + if permission.type == PermissionType.user: + self._api.get_user(permission.id, error_message="User not found") + else: + self._api.get_user_group( + permission.id, error_message="User group not found" + ) + + self._api.get_workspace( + permission.workspace_id, error_message="Workspace not found" + ) + + def _filter_invalid_permissions( + self, permissions: list[PermissionIncrementalLoad] + ) -> list[PermissionIncrementalLoad]: + """Filters out invalid permissions from the input list.""" + valid_permissions: list[PermissionIncrementalLoad] = [] + for permission in permissions: + try: + self._validate_permission(permission) + except (BaseUserException, GoodDataApiException) as e: + self.logger.error( + f"Skipping {permission}. Error: {e.error_message} " + + f"Context: {permission.__dict__}" + ) + continue + valid_permissions.append(permission) + return valid_permissions + + def _manage_permissions( + self, permissions: list[PermissionIncrementalLoad] + ) -> None: + """Manages permissions for a list of workspaces. + Modify upstream workspace declarations for each input workspace and skip non-existent ws_ids + """ + valid_permissions = self._filter_invalid_permissions(permissions) + + input_declarations = self._construct_declarations(valid_permissions) + + input_ws_ids = list(input_declarations.keys()) + upstream_declarations = self._get_upstream_declarations(input_ws_ids) + + for ws_id, declaration in input_declarations.items(): + if ws_id not in upstream_declarations: + continue + + upstream_declarations[ws_id].upsert(declaration) + + ws_permissions = upstream_declarations[ws_id].to_sdk_api() + + self._api.put_declarative_permissions(ws_id, ws_permissions) + self.logger.info(f"Updated permissions for workspace {ws_id}") + + def _provision_incremental_load(self) -> None: + """Provision permissions based on the source group.""" + self._manage_permissions(self.source_group_incremental) + + def _provision_full_load(self) -> None: + raise NotImplementedError("Not implemented yet.") diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/user_groups.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/user_groups.py new file mode 100644 index 0000000..44463f6 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/user_groups.py @@ -0,0 +1,212 @@ +# (C) 2025 GoodData Corporation + +"""Module for provisioning user groups in GoodData workspaces.""" + +from typing import Sequence, TypeAlias + +from gooddata_sdk.catalog.user.entity_model.user_group import CatalogUserGroup + +from gooddata_pipelines.provisioning.entities.users.models.user_groups import ( + UserGroupFullLoad, + UserGroupIncrementalLoad, +) +from gooddata_pipelines.provisioning.provisioning import Provisioning + +UserGroupModel: TypeAlias = UserGroupFullLoad | UserGroupIncrementalLoad + + +class UserGroupProvisioner( + Provisioning[UserGroupFullLoad, UserGroupIncrementalLoad] +): + """Provisioning class for user groups in GoodData workspaces. + + This class handles the creation, update, and deletion of user groups + based on the provided source data. + """ + + source_group_incremental: list[UserGroupIncrementalLoad] + source_group_full: list[UserGroupFullLoad] + upstream_user_groups: list[CatalogUserGroup] + + @staticmethod + def _is_changed( + group: UserGroupModel, existing_group: CatalogUserGroup + ) -> bool: + """Checks if user group has some changes and needs to be updated.""" + group.parent_user_groups.sort() + parents_changed = group.parent_user_groups != existing_group.get_parents + name_changed = group.user_group_name != existing_group.name + return parents_changed or name_changed + + def _create_or_update_user_group( + self, + group_id: str, + group_name: str, + parent_user_groups: list[str], + ) -> None: + """Creates or updates user group in the project.""" + catalog_user_group = CatalogUserGroup.init( + user_group_id=group_id, + user_group_name=group_name, + user_group_parent_ids=parent_user_groups, + ) + try: + self._api.create_or_update_user_group( + catalog_user_group=catalog_user_group + ) + self.logger.info( + f"Created/Updated user group: {group_id} - {group_name}" + ) + except Exception as e: + self.logger.error( + f"Failed to create/update user group. Error: {e} " + + f"Context: {catalog_user_group.__dict__}" + ) + + def _create_missing_user_groups( + self, + groups_to_create: Sequence[UserGroupModel], + ) -> None: + """Provisions user groups that don't exist.""" + # Sort user groups to create those without parents first + sorted_groups = sorted( + groups_to_create, key=lambda x: 1 if x.parent_user_groups else 0 + ) + + for group in sorted_groups: + self._create_or_update_user_group( + group.user_group_id, + group.user_group_name, + group.parent_user_groups, + ) + + def _update_existing_user_groups( + self, + groups_to_update: Sequence[UserGroupModel], + upstream_user_groups: list[CatalogUserGroup], + ) -> None: + """Update existing user groups and update ws_permissions.""" + existing_groups = {group.id: group for group in upstream_user_groups} + + for group in groups_to_update: + existing_group = existing_groups[group.user_group_id] + if self._is_changed(group, existing_group): + self._create_or_update_user_group( + group.user_group_id, + group.user_group_name, + group.parent_user_groups, + ) + + def _delete_user_group(self, group_ids_to_delete: set[str]) -> None: + """Deletes user group from the project.""" + for group_id in group_ids_to_delete: + try: + self._api.delete_user_group(group_id) + self.logger.info(f"Deleted user group: {group_id}") + except Exception as e: + self.logger.error( + f"Failed to delete user group. Error: {e} " + + f"Context: {{'user_group_id': {group_id}}}" + ) + + def _provision_incremental_load(self) -> None: + """Runs incremental provisioning of user groups.""" + # Get existing user groups from GoodData Cloud + self.upstream_user_groups = self._api.list_user_groups() + + # Create a set of upstream user group IDs + upstream_group_ids: set[str] = { + group.id for group in self.upstream_user_groups + } + + # Create a set of active source user group IDs + active_source_groups: set[str] = { + group.user_group_id + for group in self.source_group_incremental + if group.is_active is True + } + + # Create a set of inactive source user group IDs + inactive_source_groups: set[str] = { + group.user_group_id + for group in self.source_group_incremental + if group.is_active is False + } + + # Create a set of user group IDs to create as the difference between active + # source groups and upstream groups - i.e, we are creating groups marked + # as active in the source data but which are missing upstream in GoodData Cloud. + group_ids_to_create: set[str] = active_source_groups.difference( + upstream_group_ids + ) + + # Create a set of user group IDs to update as the intersection between active + # source groups and upstream groups - i.e, we are updating groups marked + # as active in the source data and which are present upstream in GoodData Cloud. + # The `_update_existing_user_groups` method will check if the upstream group + # definition differs from the source and if so, it will update the group. + group_ids_to_update: set[str] = active_source_groups.intersection( + upstream_group_ids + ) + + # Create a set of user group IDs to delete as the intersection between + # inactive source groups and upstream groups - i.e, we are deleting groups + # marked as inactive in the source data and which are present upstream in + # GoodData Cloud. + group_ids_to_delete: set[str] = inactive_source_groups.intersection( + upstream_group_ids + ) + + # create lists of groups to create, update and delete based on the sets + groups_to_create: list[UserGroupIncrementalLoad] = [] + groups_to_update: list[UserGroupIncrementalLoad] = [] + + for group in self.source_group_incremental: + if group.user_group_id in group_ids_to_create: + groups_to_create.append(group) + elif group.user_group_id in group_ids_to_update: + groups_to_update.append(group) + + self._create_missing_user_groups(groups_to_create) + self._update_existing_user_groups( + groups_to_update, self.upstream_user_groups + ) + self._delete_user_group(group_ids_to_delete) + + def _provision_full_load(self) -> None: + """Runs full load provisioning of user groups.""" + # Get upsream user groups + self.upstream_user_groups = self._api.list_user_groups() + + # Create a set of upstream user group IDs + upstream_group_ids: set[str] = { + group.id for group in self.upstream_user_groups + } + + # Create a set of source user group IDs + source_group_ids: set[str] = { + group.user_group_id for group in self.source_group_full + } + + # Figure out which ids are to be created, deleted or exist in both systems + id_groups = self._create_groups(source_group_ids, upstream_group_ids) + + groups_to_create: list[UserGroupFullLoad] = [] + groups_to_update: list[UserGroupFullLoad] = [] + + for group in self.source_group_full: + if group.user_group_id in id_groups.ids_to_create: + groups_to_create.append(group) + elif group.user_group_id in id_groups.ids_in_both_systems: + groups_to_update.append(group) + + # Create user groups + self._create_missing_user_groups(groups_to_create) + + # Update user groups + self._update_existing_user_groups( + groups_to_update, self.upstream_user_groups + ) + + # Delete user groups + self._delete_user_group(id_groups.ids_to_delete) diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/users.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/users.py new file mode 100644 index 0000000..e443bcf --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/users/users.py @@ -0,0 +1,179 @@ +# (C) 2025 GoodData Corporation + +"""Module for provisioning users in GoodData workspaces.""" + +from typing import TypeAlias + +from gooddata_api_client.exceptions import NotFoundException # type: ignore +from gooddata_sdk.catalog.user.entity_model.user import CatalogUser +from gooddata_sdk.catalog.user.entity_model.user_group import CatalogUserGroup + +from gooddata_pipelines.provisioning.entities.users.models.users import ( + UserFullLoad, + UserIncrementalLoad, +) +from gooddata_pipelines.provisioning.provisioning import Provisioning +from gooddata_pipelines.provisioning.utils.context_objects import UserContext + +# Type alias for user model instances +UserModel: TypeAlias = UserFullLoad | UserIncrementalLoad +UserId: TypeAlias = str + + +class UserProvisioner(Provisioning[UserFullLoad, UserIncrementalLoad]): + """Provisioning class for users in GoodData workspaces. + + This class handles the creation, update, and deletion of users + based on the provided source data. + """ + + source_group_incremental: list[UserIncrementalLoad] + source_group_full: list[UserFullLoad] + + def __init__(self, host: str, token: str) -> None: + super().__init__(host, token) + self.upstream_user_cache: dict[UserId, UserModel] = {} + + def _try_get_user( + self, user: UserModel, model: type[UserModel] + ) -> UserModel | None: + try: + if user.user_id in self.upstream_user_cache: + return self.upstream_user_cache[user.user_id] + + user_sdk_obj = self._api._sdk.catalog_user.get_user(user.user_id) + return model.from_sdk_obj(user_sdk_obj) + except NotFoundException: + return None + + def _get_or_create_user_groups(self, groups: list[str]) -> None: + """Ensures that all user groups exist in the project.""" + for group in groups: + try: + self._api._sdk.catalog_user.get_user_group(group) + except NotFoundException: + # Create the user gtoup if it does not exist + self._api.create_or_update_user_group( + CatalogUserGroup.init( + user_group_id=group, user_group_name=group + ), + ) + self.logger.info(f"Created user group: {group}") + + def _user_is_equal_upstream( + self, + user: UserModel, + upstream_user: UserModel | None, + ) -> bool: + """ + Checks if the user is different from the upstream user. Lists are checked by converting to sets. + """ + if not upstream_user: + return False + + user_data = user.model_dump() + upstream_data = upstream_user.model_dump() + + for attr, source_value in user_data.items(): + upstream_value = upstream_data.get(attr) + + if isinstance(source_value, list): + if set(source_value) != set(upstream_value or []): + return False + else: + if source_value != upstream_value: + return False + return True + + def _create_or_update_user( + self, user: UserModel, model: type[UserModel] + ) -> None: + """Creates or updates user in the project. + + Determines if the user needs to be updated or created by getting the + upstream user from GoodData Cloud and comparing it with the source user. + If user is supposed to be placed in a User Group, the function will check + for its existence and create it if needed. + + """ + user_context = UserContext( + user_id=user.user_id, + user_groups=user.user_groups, + ) + + upstream_user = self._try_get_user(user, model) + + if self._user_is_equal_upstream(user, upstream_user): + return + + self._get_or_create_user_groups(user.user_groups) + + self._api.create_or_update_user( + user.to_sdk_obj(), **user_context.__dict__ + ) + self.logger.info(f"User {user.user_id} created/updated successfully.") + + def _delete_user(self, user_id: str) -> None: + """Deletes user from the project.""" + try: + self._api._sdk.catalog_user.get_user(user_id) + except NotFoundException: + return + + self._api.delete_user(user_id) + self.logger.info(f"Deleted user: {user_id}") + + def _manage_user(self, user: UserIncrementalLoad) -> None: + """Manages user based on the provided GDUserTarget.""" + if user.is_active: + self._create_or_update_user(user, UserIncrementalLoad) + else: + self._delete_user(user.user_id) + + def _provision_incremental_load(self) -> None: + """Runs the incremental provisioning logic.""" + for user in self.source_group_incremental: + # Attempt to process each user. On failure, log the error and continue + try: + self._manage_user(user) + except Exception as e: + self.logger.error( + f"Failed to manage user {user.user_id}. Error: {e} Context: {user.__dict__}" + ) + + def _provision_full_load(self) -> None: + """Runs the full load provisioning logic.""" + # Get all upstream users + catalog_upstream_users: list[CatalogUser] = self._api.list_users() + + # Convert catalog users to user models + upstream_users: list[UserFullLoad] = [ + UserFullLoad.from_sdk_obj(user) for user in catalog_upstream_users + ] + + # Cache the upstream users in a dict. It will be reused in `_try_get_user` + self.upstream_user_cache = { + user.user_id: user for user in upstream_users + } + # Get source IDs + source_ids: set[str] = {user.user_id for user in self.source_group_full} + + # Get upstream IDs + upstream_ids: set[str] = {user.user_id for user in upstream_users} + + # Create groups of IDs to delete, create, and in both systems + id_groups = self._create_groups(source_ids, upstream_ids) + + # Iterate over source users and create/update + for user in self.source_group_full: + user_id = user.user_id + + if ( + user_id in id_groups.ids_to_create + or user_id in id_groups.ids_in_both_systems + ): + self._create_or_update_user(user, UserFullLoad) + + # Delete users marked for deletion + for user_id in id_groups.ids_to_delete: + self._delete_user(user_id) diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/__init__.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/models.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/models.py new file mode 100644 index 0000000..788c7b4 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/models.py @@ -0,0 +1,78 @@ +# (C) 2025 GoodData Corporation +"""Module containing models related to workspace provisioning in GoodData Cloud.""" + +from dataclasses import dataclass, field +from typing import Literal + +from pydantic import BaseModel, ConfigDict + + +@dataclass +class WorkspaceDataMaps: + """Dataclass to hold various mappings related to workspace data.""" + + child_to_parent_id_map: dict[str, str] = field(default_factory=dict) + workspace_id_to_wdf_map: dict[str, dict[str, list[str]]] = field( + default_factory=dict + ) + parent_ids: set[str] = field(default_factory=set) + source_ids: set[str] = field(default_factory=set) + workspace_id_to_name_map: dict[str, str] = field(default_factory=dict) + upstream_ids: set[str] = field(default_factory=set) + + +class WorkspaceFullLoad(BaseModel): + """Model representing input for provisioning of workspaces in GoodData Cloud.""" + + model_config = ConfigDict(coerce_numbers_to_str=True) + + parent_id: str + workspace_id: str + workspace_name: str + workspace_data_filter_id: str | None = None + workspace_data_filter_values: list[str] | None = None + + +class WorkspaceIncrementalLoad(WorkspaceFullLoad): + """Model representing input for incremental provisioning of workspaces in GoodData Cloud.""" + + # TODO: double check that the model loads the data correctly, write a test + is_active: bool + + +class WDFSettingAttributes(BaseModel): + title: str + filterValues: list[str] + + +class WDFSettingRelationshipsData(BaseModel): + id: str + type: Literal["workspaceDataFilter"] + + +class WDFSettingRelationships(BaseModel): + workspaceDataFilter: dict[str, WDFSettingRelationshipsData] + + +class WDFSettingLinks(BaseModel): + self: str + + +class WDFSettingMetaOrigin(BaseModel): + originType: str + originId: str + + +class WDFSettingMeta(BaseModel): + origin: WDFSettingMetaOrigin + + +class WDFSetting(BaseModel): + """Model representing a workspace data filter setting in GoodData Cloud.""" + + id: str + type: Literal["workspaceDataFilterSetting"] + attributes: WDFSettingAttributes + relationships: WDFSettingRelationships + links: WDFSettingLinks + meta: WDFSettingMeta diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py new file mode 100644 index 0000000..7324d41 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace.py @@ -0,0 +1,263 @@ +# (C) 2025 GoodData Corporation +"""Module for provisioning workspaces in GoodData Cloud.""" + +from typing import Literal + +from gooddata_sdk.catalog.workspace.entity_model.workspace import ( + CatalogWorkspace, +) + +from gooddata_pipelines.api.exceptions import GoodDataApiException +from gooddata_pipelines.provisioning.entities.workspaces.models import ( + WorkspaceDataMaps, + WorkspaceFullLoad, + WorkspaceIncrementalLoad, +) +from gooddata_pipelines.provisioning.entities.workspaces.workspace_data_filters import ( + WorkspaceDataFilterManager, +) +from gooddata_pipelines.provisioning.entities.workspaces.workspace_data_parser import ( + WorkspaceDataParser, +) +from gooddata_pipelines.provisioning.entities.workspaces.workspace_data_validator import ( + WorkspaceDataValidator, +) +from gooddata_pipelines.provisioning.provisioning import Provisioning +from gooddata_pipelines.provisioning.utils.context_objects import ( + WorkspaceContext, +) +from gooddata_pipelines.provisioning.utils.exceptions import WorkspaceException + + +class WorkspaceProvisioner( + Provisioning[WorkspaceFullLoad, WorkspaceIncrementalLoad] +): + source_group_full: list[WorkspaceFullLoad] + source_group_incremental: list[WorkspaceIncrementalLoad] + + def __init__(self, *args: str, **kwargs: str) -> None: + """Creates an instance of the WorkspaceProvisioner. + + Calls the superclass constructor and initializes the validator, parser, + and maps for workspace data. + """ + super().__init__(*args, **kwargs) + self.validator: WorkspaceDataValidator = WorkspaceDataValidator( + self._api + ) + self.parser: WorkspaceDataParser = WorkspaceDataParser() + self.maps: WorkspaceDataMaps = WorkspaceDataMaps() + + def _find_workspaces_to_update( + self, + source_group: list[WorkspaceFullLoad], + panther_group: list[CatalogWorkspace], + ids_in_both_systems: set[str], + ) -> set[str]: + """ + Inspects existing Panther workspaces and compares them to workspaces from + the source database. If the ID exists in both systems but the workspace + name in GoodData Cloud is different from the source, the workspace will + be updated. The rest of the workspaces will be ignored. + """ + existing_workspaces: dict[str, CatalogWorkspace] = { + workspace.id: workspace for workspace in panther_group + } + + ids_to_update: set[str] = set() + + for source_workspace in source_group: + source_id = source_workspace.workspace_id + source_name = source_workspace.workspace_name + + if source_id not in ids_in_both_systems: + continue + + if existing_workspaces.get(source_id): + panther_name = existing_workspaces[source_id].name + else: + continue + + if source_name == panther_name: + continue + + ids_to_update.add(source_id) + + return ids_to_update + + def _create_or_update_panther_workspaces( + self, + workspace_ids_to_create: set[str], + workspace_ids_to_update: set[str], + child_to_parent_map: dict[str, str], + workspace_id_to_wdf_map: dict[str, dict[str, list[str]]], + ) -> None: + action: Literal["CREATE", "UPDATE"] + + for source_workspace in self.source_group_full: + if source_workspace.workspace_id in workspace_ids_to_update: + action = "UPDATE" + elif source_workspace.workspace_id in workspace_ids_to_create: + action = "CREATE" + else: + continue + + context: WorkspaceContext = WorkspaceContext( + workspace_id=source_workspace.workspace_id, + workspace_name=source_workspace.workspace_name, + wdf_id=source_workspace.workspace_data_filter_id, + wdf_values=source_workspace.workspace_data_filter_values, + ) + + parent_workspace_id: str = child_to_parent_map[context.workspace_id] + + try: + self._api.create_or_update_panther_workspace( + workspace_id=context.workspace_id, + workspace_name=str(context.workspace_name), + parent_id=parent_workspace_id, + ) + self.logger.info( + f"{action.title()}d workspace: {context.workspace_id}" + ) + + except GoodDataApiException as e: + combined_context = {**context.__dict__, **e.__dict__} + self.logger.error( + f"Failed to {action.title()} workspace: {context.workspace_id}. " + + f"Error: {e} Context: {combined_context}" + ) + + # If child workspace has WDF settings, apply them + child_wdfs: dict[str, list[str]] = workspace_id_to_wdf_map.get( + context.workspace_id, {} + ) + if child_wdfs: + self.wdf_manager.check_wdf_settings( + context, + ) + + def delete_panther_workspaces( + self, ids_to_delete: set[str], workspace_id_to_name_map: dict[str, str] + ) -> None: + for workspace_id in ids_to_delete: + workspace_context: WorkspaceContext = WorkspaceContext( + workspace_id=workspace_id, + workspace_name=workspace_id_to_name_map.get(workspace_id), + ) + try: + self._api.delete_panther_workspace(workspace_id) + self.logger.info( + f"Deleted workspace: {workspace_context.workspace_id}" + ) + + except GoodDataApiException as e: + exception_context = {**workspace_context.__dict__, **e.__dict__} + self.logger.error( + f"Failed to delete workspace: {workspace_context.workspace_id}. " + + f"Error: {e} Context: {exception_context}" + ) + + def verify_workspace_provisioning( + self, + source_group: list[WorkspaceFullLoad], + parent_workspace_ids: set[str], + ) -> None: + """Verifies that upstream content is equal to the source data.""" + source_ids_names: set[tuple[str, str]] = { + (item.workspace_id, item.workspace_name) for item in source_group + } + + panther_workspaces: list[CatalogWorkspace] = ( + self._api.get_panther_children_workspaces(parent_workspace_ids) + ) + + panther_ids_names: set[tuple[str, str]] = { + (workspace.workspace_id, workspace.name) + for workspace in panther_workspaces + } + + diff: set[tuple[str, str]] = source_ids_names.symmetric_difference( + panther_ids_names + ) + + if diff: + raise WorkspaceException( + "Provisioning failed. The source and Panther workspaces do not " + + f"match. Difference: {diff}" + ) + + def _provision_full_load(self) -> None: + """Full load workspace provisioning.""" + + # Validate the source data. + self.validator.validate_source_data(self.source_group_full) + + # Set the maps based on the source data. + self.maps = self.parser.set_maps_based_on_source( + self.maps, self.source_group_full + ) + + # Get upstream children of all parent workspaces. + self.upstream_group: list[CatalogWorkspace] = ( + self._api.get_panther_children_workspaces(self.maps.parent_ids) + ) + + # Set maps that require upstream data. + self.maps = self.parser.set_maps_with_upstream_data( + self.maps, self.source_group_full, self.upstream_group + ) + + # Create an instance of WDF manager with the created maps. + self.wdf_manager = WorkspaceDataFilterManager(self._api, self.maps) + + # Sort the ids to groups based on provisioning logic. + id_groups = self._create_groups( + self.maps.source_ids, self.maps.upstream_ids + ) + + # Find out which workspaces to update. + self.ids_to_update: set[str] = self._find_workspaces_to_update( + self.source_group_full, + self.upstream_group, + id_groups.ids_in_both_systems, + ) + + # Delete the workspaces that are not in the source. + self.delete_panther_workspaces( + id_groups.ids_to_delete, self.maps.workspace_id_to_name_map + ) + + # Create or update selected workspaces. + self._create_or_update_panther_workspaces( + id_groups.ids_to_create, + self.ids_to_update, + self.maps.child_to_parent_id_map, + self.maps.workspace_id_to_wdf_map, + ) + + # Check WDF settings of ignored workspaces. + ignored_workspace_ids: set[str] = self.maps.source_ids.difference( + id_groups.ids_to_create.union(self.ids_to_update).union( + id_groups.ids_to_delete + ) + ) + + for ignored_workspace_id in ignored_workspace_ids: + ignored_workspace_context: WorkspaceContext = WorkspaceContext( + workspace_id=ignored_workspace_id, + workspace_name=self.maps.workspace_id_to_name_map.get( + ignored_workspace_id + ), + ) + self.wdf_manager.check_wdf_settings(ignored_workspace_context) + + # Verify the provisioning by queries to GoodData Cloud. + self.verify_workspace_provisioning( + self.source_group_full, self.maps.parent_ids + ) + + def _provision_incremental_load(self) -> None: + """Incremental workspace provisioning.""" + + raise NotImplementedError("Not implemented yet.") diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_filters.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_filters.py new file mode 100644 index 0000000..26bbea6 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_filters.py @@ -0,0 +1,286 @@ +# (C) 2025 GoodData Corporation + +"""Module for managing workspace data filter settings in GoodData Cloud.""" + +import json +from typing import Any +from uuid import uuid4 + +from requests import Response + +from gooddata_pipelines.api import GoodDataAPI +from gooddata_pipelines.logger.logger import LogObserver +from gooddata_pipelines.provisioning.entities.workspaces.models import ( + WDFSetting, + WorkspaceDataMaps, +) +from gooddata_pipelines.provisioning.utils.context_objects import ( + WorkspaceContext, +) +from gooddata_pipelines.provisioning.utils.exceptions import WorkspaceException + + +class WorkspaceDataFilterManager: + """ + Helper class to manage workspace data filter settings. Note that Workspace + Data Filters themselves are not managed here. The Workspace Data Filter + Setting object represents the relationship of values in a WDF column and + a specific workspace. + """ + + def __init__(self, api: GoodDataAPI, maps: WorkspaceDataMaps) -> None: + self.api: GoodDataAPI = api + self.maps: WorkspaceDataMaps = maps + self.logger: LogObserver = LogObserver() + + @staticmethod + def _create_wdf_setting_dict( + wdf_setting_id: str, wdf_id: str, wdf_values: list[str] + ) -> dict[str, Any]: + """Loads a JSON template of a WDF setting and fills it with the given values.""" + values = [str(value) for value in wdf_values] + + import os + + wdf_setting_path = os.path.join( + os.path.dirname(__file__), "../../assets/wdf_setting.json" + ) + with open(os.path.abspath(wdf_setting_path)) as file: + wdf_setting: dict[str, Any] = json.load(file) + + wdf_setting["data"]["attributes"]["filterValues"] = values + wdf_setting["data"]["id"] = wdf_setting_id + wdf_setting["data"]["relationships"]["workspaceDataFilter"]["data"][ + "id" + ] = wdf_id + + return wdf_setting + + def _get_wdf_settings_for_workspace( + self, workspace_id: str + ) -> list[WDFSetting]: + """Gets all workspace data filter settings for a given workspace.""" + wdf_settings_response: Response = ( + self.api.get_workspace_data_filter_settings(workspace_id) + ) + + if not wdf_settings_response.ok: + raise WorkspaceException( + f"Failed to get WDF settings: {wdf_settings_response.text}", + workspace_id=workspace_id, + http_status=str(wdf_settings_response.status_code), + ) + + raw_wdf_settings: dict[str, Any] = wdf_settings_response.json() + + data: list[dict[str, Any]] = raw_wdf_settings["data"] + + wdf_settings: list[WDFSetting] = [ + WDFSetting(**wdf_setting) for wdf_setting in data + ] + + return wdf_settings + + @staticmethod + def _get_actual_wdf_setting_id_and_values( + actual_wdf_settings: list[WDFSetting], actual_wdf_id: str + ) -> tuple[str, list[str]]: + """Returns WDF setting ID and values for given WDF ID.""" + for actual_wdf_setting in actual_wdf_settings: + if ( + actual_wdf_setting.relationships.workspaceDataFilter["data"].id + == actual_wdf_id + ): + actual_wdf_setting_id = actual_wdf_setting.id + actual_wdf_values = actual_wdf_setting.attributes.filterValues + + return actual_wdf_setting_id, actual_wdf_values + + raise WorkspaceException( + "Could not find WDF setting for WDF in actual WDF settings.", + wdf_id=actual_wdf_id, + ) + + def _delete_redundant_wdf_setting( + self, + workspace_context: WorkspaceContext, + actual_wdf_id: str, + actual_wdf_settings: list[WDFSetting], + ) -> None: + """Deletes a WDF setting.""" + actual_wdf_setting_id, actual_wdf_values = ( + self._get_actual_wdf_setting_id_and_values( + actual_wdf_settings, actual_wdf_id + ) + ) + # Update context with actual values + workspace_context.wdf_id = actual_wdf_id + workspace_context.wdf_values = actual_wdf_values + + # If there is a WDF setting for a WDF that should not be associated with + # the workspace, then delete the setting + delete_response: Response = ( + self.api.delete_workspace_data_filter_setting( + workspace_context.workspace_id, + actual_wdf_setting_id, + ) + ) + if delete_response.ok: + self.logger.info( + f"Deleted WDF setting for WDF {workspace_context.wdf_id} in " + + f"workspace {workspace_context.workspace_id}" + ) + else: + raise WorkspaceException( + f"Failed to delete WDF setting: {delete_response.text}", + delete_response, + workspace_context, + ) + + def _post_wdf_setting( + self, + workspace_context: WorkspaceContext, + ) -> None: + """Posts a WDF setting to Panther.""" + wdf_setting = self._create_wdf_setting_dict( + str(uuid4()), + str(workspace_context.wdf_id), + workspace_context.wdf_values + if workspace_context.wdf_values + else [], + ) + post_response: Response = self.api.post_workspace_data_filter_setting( + workspace_context.workspace_id, + wdf_setting, + ) + if post_response.ok: + self.logger.info( + f"Created WDF setting for WDF {workspace_context.wdf_id} in workspace {workspace_context.workspace_id}" + ) + else: + raise WorkspaceException( + f"Failed to create WDF setting: {post_response.text}", + post_response, + workspace_context, + ) + + def _put_wdf_setting( + self, + workspace_context: WorkspaceContext, + actual_wdf_settings: list[WDFSetting], + ) -> None: + # get Panther WDF setting ID + actual_wdf_setting_id, _ = self._get_actual_wdf_setting_id_and_values( + actual_wdf_settings, str(workspace_context.wdf_id) + ) + + wdf_setting = self._create_wdf_setting_dict( + actual_wdf_setting_id, + str(workspace_context.wdf_id), + workspace_context.wdf_values + if workspace_context.wdf_values + else [], + ) + + put_response: Response = self.api.put_workspace_data_filter_setting( + workspace_context.workspace_id, + wdf_setting, + ) + if put_response.ok: + self.logger.info( + f"Updated WDF setting for WDF {workspace_context.wdf_id} in workspace {workspace_context.workspace_id}" + ) + else: + raise WorkspaceException( + f"Failed to update WDF setting: {put_response.text}", + put_response, + workspace_context, + ) + + def _compare_wdf_settings( + self, + workspace_context: WorkspaceContext, + source_wdf_config: dict[str, list[str]], + upstream_wdf_settings: list[WDFSetting], + ) -> None: + """ + Compares WDF settings as extracted from the source with the actual WDF + settings in Panther. We do not know the WDF setting IDs from the outset, + which is why we need to check the WDF IDs and then the settings values + in a roundabout way. I.e., we know that a WDF should have some setting + with an unknown ID, but certain values. In this case, we don't care about + the setting ID, but need to make sure that the workspace has the correct + values for the WDF. + """ + upstream_wdf_ids: set[str] = { + upstream_wdf_setting.relationships.workspaceDataFilter["data"].id + for upstream_wdf_setting in upstream_wdf_settings + } + + source_wdf_ids: set[str] = set(source_wdf_config.keys()) + + # Create map of upstream WDF_ID : WDF values + upstream_wdf_ids_and_values: dict[str, list[str]] = {} + for upstream_wdf_setting in upstream_wdf_settings: + upstream_wdf_ids_and_values[ + upstream_wdf_setting.relationships.workspaceDataFilter[ + "data" + ].id + ] = upstream_wdf_setting.attributes.filterValues + + # Iterate through source WDF settings + for source_wdf_id in source_wdf_ids: + # Update WDF information in context to make sure we have the correct + # data -> there can be multiple WDFs per workspace + source_values: list[str] = source_wdf_config[source_wdf_id] + workspace_context.wdf_id = source_wdf_id + workspace_context.wdf_values = source_values + + # Post WDF setting if missing for a WDF, when it should be there + if source_wdf_id not in upstream_wdf_ids: + self._post_wdf_setting(workspace_context) + + # If settings exist for a WDF that should be there, then compare values + elif source_wdf_id in upstream_wdf_ids: + actual_values: list[str] = upstream_wdf_ids_and_values[ + source_wdf_id + ] + + # If values are different, then update the WDF settings + if set(source_values) != set(actual_values): + self._put_wdf_setting( + workspace_context, + upstream_wdf_settings, + ) + + # Go through Panther WDF settings and check if there are any that should + # not be there. Delete them if so. + for actual_wdf_id in upstream_wdf_ids: + if actual_wdf_id not in source_wdf_ids: + self._delete_redundant_wdf_setting( + workspace_context, + actual_wdf_id, + upstream_wdf_settings, + ) + + def check_wdf_settings( + self, + workspace_context: WorkspaceContext, + ) -> None: + """ + Checks WDF settings for a given workspace. + Creates WDF settings defined in source if they are missing in Panther. + Updates WDF setting values if they are different in source and Panther. + Deletes WDF settings from Panther if they are not defined in source. + """ + actual_wdf_settings: list[WDFSetting] = ( + self._get_wdf_settings_for_workspace(workspace_context.workspace_id) + ) + + source_wdf_config: dict[str, list[str]] = ( + self.maps.workspace_id_to_wdf_map[workspace_context.workspace_id] + ) + + self._compare_wdf_settings( + workspace_context, source_wdf_config, actual_wdf_settings + ) diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_parser.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_parser.py new file mode 100644 index 0000000..32bea22 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_parser.py @@ -0,0 +1,123 @@ +# (C) 2025 GoodData Corporation + +"""Module for parsing and processing workspace data in GoodData Cloud.""" + +from gooddata_sdk.catalog.workspace.entity_model.workspace import ( + CatalogWorkspace, +) + +from gooddata_pipelines.provisioning.entities.workspaces.models import ( + WorkspaceDataMaps, + WorkspaceFullLoad, +) + + +class WorkspaceDataParser: + """Helper class to process workspace data retrieved from Panther and source DB.""" + + @staticmethod + def _get_id_to_name_map( + source_group: list[WorkspaceFullLoad], + upstream_group: list[CatalogWorkspace], + ) -> dict[str, str]: + """Creates a map of workspace IDs to their names for all known workspaces.""" + source_map: dict[str, str] = { + workspace.workspace_id: workspace.workspace_name + for workspace in source_group + } + upstream_map: dict[str, str] = { + item.workspace_id: item.name for item in upstream_group + } + + return {**upstream_map, **source_map} + + @staticmethod + def _get_child_to_parent_map( + source_group: list[WorkspaceFullLoad], + ) -> dict[str, str]: + """Creates a map of child workspace IDs to their parent workspace IDs.""" + child_to_parent_map: dict[str, str] = { + workspace.workspace_id: workspace.parent_id + for workspace in source_group + } + + return child_to_parent_map + + @staticmethod + def _get_set_of_ids_from_source( + source_group: list[WorkspaceFullLoad], column_name: str + ) -> set[str]: + """Creates a set of unique parent workspace IDs.""" + set_of_ids: set[str] = { + getattr(workspace, column_name) + for workspace in source_group + if getattr(workspace, column_name) + } + return set_of_ids + + @staticmethod + def get_set_of_upstream_workspace_ids( + upstream_group: list[CatalogWorkspace], + ) -> set[str]: + """Creates a set of unique upstream workspace IDs.""" + set_of_ids: set[str] = {item.workspace_id for item in upstream_group} + return set_of_ids + + def _get_child_to_wdfs_map( + self, source_group: list[WorkspaceFullLoad] + ) -> dict[str, dict[str, list[str]]]: + """Creates a map of child workspace IDs to their WDF IDs.""" + # TODO: Use objects or a more transparent data structure instead of this. + child_to_wdf_map: dict[str, dict[str, list[str]]] = {} + + # For each child, get its possible WDF IDs and values for each id + for workspace in source_group: + child_id: str = workspace.workspace_id + wdf_id: str | None = workspace.workspace_data_filter_id + wdf_values: list[str] | None = ( + workspace.workspace_data_filter_values + ) + + if wdf_values and wdf_id: + if not child_to_wdf_map.get(child_id): + child_to_wdf_map[child_id] = {} + child_to_wdf_map[child_id][wdf_id] = wdf_values + + return child_to_wdf_map + + def set_maps_based_on_source( + self, + map_object: WorkspaceDataMaps, + source_group: list[WorkspaceFullLoad], + ) -> WorkspaceDataMaps: + """Creates maps which are dependent on the source group only.""" + map_object.child_to_parent_id_map = self._get_child_to_parent_map( + source_group + ) + map_object.workspace_id_to_wdf_map = self._get_child_to_wdfs_map( + source_group + ) + map_object.parent_ids = self._get_set_of_ids_from_source( + source_group, "parent_id" + ) + map_object.source_ids = self._get_set_of_ids_from_source( + source_group, "workspace_id" + ) + + return map_object + + def set_maps_with_upstream_data( + self, + map_object: WorkspaceDataMaps, + source_group: list[WorkspaceFullLoad], + upstream_group: list[CatalogWorkspace], + ) -> WorkspaceDataMaps: + """Creates maps which are dependent on both the source group and upstream group.""" + map_object.workspace_id_to_name_map = self._get_id_to_name_map( + source_group, upstream_group + ) + map_object.upstream_ids = self.get_set_of_upstream_workspace_ids( + upstream_group + ) + + return map_object diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_validator.py b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_validator.py new file mode 100644 index 0000000..d86851f --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/entities/workspaces/workspace_data_validator.py @@ -0,0 +1,188 @@ +# (C) 2025 GoodData Corporation + +"""Module for validating workspace data integrity in GoodData Cloud.""" + +from typing import Any + +from requests import Response + +from gooddata_pipelines.api import GoodDataAPI +from gooddata_pipelines.logger.logger import LogObserver +from gooddata_pipelines.provisioning.entities.workspaces.models import ( + WorkspaceFullLoad, +) +from gooddata_pipelines.provisioning.utils.context_objects import ( + WorkspaceContext, +) +from gooddata_pipelines.provisioning.utils.exceptions import ( + WorkspaceDataIntegrityException, + WorkspaceException, +) + + +class WorkspaceDataValidator: + """Class for validating workspace data integrity before provisioning.""" + + def __init__(self, api: GoodDataAPI): + """ + Initializes the WorkspaceDataValidator with the GoodData API instance. + + Args: + api (GoodDataAPI): An instance of the GoodData API client. + """ + self.api = api + self.logger = LogObserver() + + def _check_basic_integrity( + self, + source_group: list[WorkspaceFullLoad], + ) -> tuple[set[str], dict[str, list[str]]]: + """ + Checks that mandatory fields are not empty and that that the combinations + of values are unique. + + Returns a set of parent workspaces and a dictionary of parent-wdf mappings. + """ + parent_workspaces: set[str] = set() + parent_wdf_map: dict[str, list[str]] = {} + parent_child_wdf_ids: list[tuple[str, str, str | None]] = [] + + # Check that fields are not empty + for workspace in source_group: + parent_id: str | None = workspace.parent_id + workspace_id: str | None = workspace.workspace_id + workspace_name: str | None = workspace.workspace_name + wdf_id: str | None = workspace.workspace_data_filter_id + wdf_values: list[str] | None = ( + workspace.workspace_data_filter_values + ) + + # Create a context for the workspace validation + validation_context: WorkspaceContext = WorkspaceContext( + workspace_id=workspace_id, + workspace_name=workspace_name, + wdf_id=wdf_id, + wdf_values=wdf_values, + ) + + # Raise specific error if both parent_id and workspace_id are not defined + if (parent_id is None or parent_id == "") and ( + workspace_id is None or workspace_id == "" + ): + raise WorkspaceDataIntegrityException( + "Parent ID and workspace ID are not defined for at least one row. Please check the source data." + ) + + # Raise error if parent_id is not defined + if parent_id is None or parent_id == "": + raise WorkspaceDataIntegrityException( + "Parent ID is not defined in source data.", + validation_context, + ) + + # Add parent_id to the set of unique parent workspaces + parent_workspaces.add(parent_id) + + # Raise error if workspace_id is not defined + if workspace_id is None or workspace_id == "": + raise WorkspaceDataIntegrityException( + f"Workspace ID is not defined for parent {parent_id}" + ) + + # Raise error if wdf_id is not defined but has values + if wdf_id is not None and wdf_id != "": + if wdf_values is None or wdf_values == []: + raise WorkspaceDataIntegrityException( + "WDF ID is defined but no WDF values are provided", + validation_context, + ) + + # Add wdf_id to the parent-wdf dict if the value is defined + if not parent_wdf_map.get(parent_id): + parent_wdf_map[parent_id] = [] + + parent_wdf_map[parent_id].append(wdf_id) + + # Raise error if wdf_values are defined but wdf_id is not defined + if wdf_values is not None and wdf_values != []: + if wdf_id is None or wdf_id == "": + raise WorkspaceDataIntegrityException( + "WDF values are provided but WDF ID is not defined.", + validation_context, + ) + + parent_child_wdf_ids.append((parent_id, workspace_id, wdf_id)) + + # Check whether there are non-unique combinations in data + if len(parent_child_wdf_ids) != len(set(parent_child_wdf_ids)): + # Log the error to the database as a warning, but continue execution + self.logger.warning( + "Duplicate combinations of parent_id, workspace_id, " + + "wdf_id exist in the source data." + ) + + return parent_workspaces, parent_wdf_map + + def _check_parent_exist(self, parent_id: str) -> None: + """ + Raises an error if a parent workspace does not exist in Panther. + """ + if not self.api.check_workspace_exists(parent_id): + raise WorkspaceException( + f"Parent workspace {parent_id} does not exist in Panther.", + workspace_id=parent_id, + ) + + def _check_wdf_is_set_on_parent( + self, parent_id: str, source_wdf_ids: list[str] + ) -> None: + """Raises an error if the parent workspace does not contain any of the defined wdf_id.""" + wdf_response: Response = self.api.get_all_workspace_data_filters( + parent_id + ) + wdf_json: dict[str, Any] = wdf_response.json() + wdf_data: list[dict[str, Any]] = wdf_json.get("data", []) + wdf_ids_on_parent: set[str] = {wdf["id"] for wdf in wdf_data} + + for source_wdf_id in source_wdf_ids: + if source_wdf_id not in wdf_ids_on_parent: + raise WorkspaceException( + f"WDF is not set on parent workspace {parent_id}.", + wdf_id=source_wdf_id, + workspace_id=parent_id, + ) + + def validate_source_data( + self, source_group: list[WorkspaceFullLoad] + ) -> None: + """ + Validates the source data integrity. + + **Raises error when**: + - the list of workspaces is empty + - parent_id is not defined + - workspace_id is not defined + - The parent workspace does not exist + - The parent workspace does not contain defined wdf_id. + - wdf_id is defined but wdf_values is not defined + - wdf_values are defined but wdf_id is not defined + + **Logs a warning when**: + - There are more values for the parent_id, workspace_id, wdf_id combination. + """ + if not source_group: + # Raise error if source is empty + raise WorkspaceException( + "No workspaces found in the source database." + ) + + unique_parents, parent_wdf_map = self._check_basic_integrity( + source_group + ) + + for parent_id in unique_parents: + self._check_parent_exist(parent_id) + + self._check_wdf_is_set_on_parent( + parent_id, parent_wdf_map[parent_id] + ) diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py b/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py new file mode 100644 index 0000000..2d74b54 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/provisioning.py @@ -0,0 +1,132 @@ +# (C) 2025 GoodData Corporation + +"""Provisioning base class for GoodData Pipelines.""" + +from pathlib import Path +from typing import Generic, Type, TypeVar + +from gooddata_sdk.utils import PROFILES_FILE_PATH, profile_content + +from gooddata_pipelines.api import GoodDataAPI +from gooddata_pipelines.logger.logger import ( + LogObserver, +) +from gooddata_pipelines.provisioning.utils.utils import EntityGroupIds + +TFullLoadSourceData = TypeVar("TFullLoadSourceData") +TIncrementalSourceData = TypeVar("TIncrementalSourceData") + + +class Provisioning(Generic[TFullLoadSourceData, TIncrementalSourceData]): + """Base provisioning class.""" + + TProvisioning = TypeVar("TProvisioning", bound="Provisioning") + source_group_full: list[TFullLoadSourceData] + source_group_incremental: list[TIncrementalSourceData] + + def __init__(self, host: str, token: str) -> None: + self.source_id: set[str] = set() + self.upstream_id: set[str] = set() + self._api = GoodDataAPI(host, token) + self.logger: LogObserver = LogObserver() + self.fatal_exception: str = "" + + @classmethod + def create( + cls: Type[TProvisioning], host: str, token: str + ) -> TProvisioning: + """Creates a provisioner instance using provided host and token.""" + return cls(host=host, token=token) + + @classmethod + def create_from_profile( + cls: Type[TProvisioning], + profile: str = "default", + profiles_path: Path = PROFILES_FILE_PATH, + ) -> TProvisioning: + """Creates a provisioner instance using a GoodData profile file.""" + content = profile_content(profile, profiles_path) + return cls(**content) + + @staticmethod + def _create_groups( + source_id: set[str], panther_id: set[str] + ) -> EntityGroupIds: + """Creates groups for provisioning as sets of IDs. + + Sorts the IDs into three categories: + - IDs that exist both source and upstream (to be checked further) + - IDs that exist upstream but not in source (to be deleted) + - IDs that exist in source but not upstream (to be created) + """ + ids_in_both_systems: set[str] = source_id.intersection(panther_id) + ids_to_delete: set[str] = panther_id.difference(source_id) + ids_to_create: set[str] = source_id.difference(panther_id) + + return EntityGroupIds( + ids_in_both_systems=ids_in_both_systems, + ids_to_delete=ids_to_delete, + ids_to_create=ids_to_create, + ) + + def _provision_incremental_load(self) -> None: + raise NotImplementedError( + "Provisioning method to be implemented in the subclass." + ) + + def _provision_full_load(self) -> None: + raise NotImplementedError( + "Provisioning method to be implemented in the subclass." + ) + + def full_load(self, source_data: list[TFullLoadSourceData]) -> None: + """Runs full provisioning workflow with the provided source data. + + Full provisioning is a full load of the source data, where the source data + is assumed to a single source of truth and the upstream workspaces are updated + to match it. + + That means: + - All workspaces declared in the source data are created if missing, or + updated to match the source data + - All workspaces not declared in the source data are deleted + """ + self.source_group_full = source_data + + try: + self._provision_full_load() + self.logger.info("Provisioning completed successfully.") + except Exception as e: + self.fatal_exception = str(e) + self.logger.error( + f"Provisioning failed. Error: {self.fatal_exception} " + + f"Context: {e.__dict__}" + ) + + def incremental_load( + self, source_data: list[TIncrementalSourceData] + ) -> None: + """Runs incremental provisioning workflow with the provided source data. + + Incremental provisioning is used to modify a subset of the upstream workspaces + based on the source data provided. + """ + self.source_group_incremental = source_data + + try: + self._provision_incremental_load() + self.logger.info("Provisioning completed successfully.") + except Exception as e: + self.fatal_exception = str(e) + self.logger.error( + f"Provisioning failed. Error: {self.fatal_exception} " + + f"Context: {e.__dict__}" + ) + + # TODO: implement a sceond provisioning method and name the two differently: + # 1) provision_incremental - will use the is_active logic, such as user provisioning now + # 2) provision_full - full load of the source data, like workspaces now + # Each will have its own implementation and source data model. + # Both use cases are required and need to be supported. + # This will also improve the clarity of the code as now provisioning of each + # entity works differently, leading to confusion. diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/utils/__init__.py b/gooddata-pipelines/gooddata_pipelines/provisioning/utils/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/utils/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/utils/context_objects.py b/gooddata-pipelines/gooddata_pipelines/provisioning/utils/context_objects.py new file mode 100644 index 0000000..b54894a --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/utils/context_objects.py @@ -0,0 +1,32 @@ +# (C) 2025 GoodData Corporation + +"""Module for context objects used in GoodData Pipelines provisioning.""" + + +class WorkspaceContext: + workspace_id: str + workspace_name: str | None + wdf_id: str | None + wdf_values: list[str] | None + + def __init__( + self, + workspace_id: str | None, + workspace_name: str | None, + wdf_id: str | None = None, + wdf_values: list[str] | None = None, + ): + self.workspace_id: str = workspace_id if workspace_id else "NA" + self.workspace_name: str | None = workspace_name + self.wdf_id: str | None = wdf_id + self.wdf_values: list[str] | None = wdf_values + + +class UserContext: + user_id: str + user_groups: str + + def __init__(self, user_id: str, user_groups: list[str]): + """User context object, stringifies list of user groups""" + self.user_id: str = user_id + self.user_groups: str = ",".join(user_groups) diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/utils/exceptions.py b/gooddata-pipelines/gooddata_pipelines/provisioning/utils/exceptions.py new file mode 100644 index 0000000..bdb5c68 --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/utils/exceptions.py @@ -0,0 +1,95 @@ +# (C) 2025 GoodData Corporation + +"""Module for exceptions used in GoodData Pipelines provisioning.""" + +from gooddata_pipelines.provisioning.utils.utils import AttributesMixin + + +# TODO: Use the generic context exception and phase out the specific ones +# - we don't need to conform to process-specific schema anymore +class ContextException(Exception, AttributesMixin): + def __init__( + self, message: str, *context_objects: object, **kwargs: str + ) -> None: + """Exception raised during context processing.""" + super().__init__(message) + attributes = self.get_attrs(*context_objects, overrides=kwargs) + + for key, value in attributes.items(): + setattr(self, key, value) + + +class ProvisioningException(Exception, AttributesMixin): + def __init__( + self, message: str, *context_objects: object, **kwargs: str + ) -> None: + """Exception raised during provisioning.""" + super().__init__(message) + self.attributes = self.get_attrs(*context_objects, overrides=kwargs) + self.error_message: str = message + + +class WorkspaceException(ProvisioningException): + def __init__( + self, + message: str, + *context_objects: object, + **kwargs: str, + ) -> None: + """Exception raised during workspace provisioning.""" + super().__init__(message, *context_objects, **kwargs) + + self.http_status: str = self.attributes.get( + "http_status", "500 Internal Server Error" + ) + self.http_method: str | None = self.attributes.get("http_method", "NA") + self.workspace_id: str = self.attributes.get("workspace_id", "NA") + self.workspace_name: str | None = self.attributes.get( + "workspace_name", "NA" + ) + self.wdf_id: str | None = self.attributes.get("wdf_id", None) + self.wdf_values: str | None = self.attributes.get("wdf_values", None) + self.api_endpoint: str = self.attributes.get( + "api_endpoint", "workspace_provisioning" + ) + + +class WorkspaceDataIntegrityException(WorkspaceException): + def __init__( + self, message: str, *context_objects: object, **kwargs: str + ) -> None: + """Exception raised during workspace validation.""" + super().__init__(message, *context_objects, **kwargs) + + self.workspace_id: str = self.attributes.get("workspace_id", "NA") + self.workspace_name: str | None = self.attributes.get( + "workspace_name", None + ) + self.wdf_id: str | None = self.attributes.get("wdf_id", None) + self.wdf_values: str | None = self.attributes.get("wdf_values", None) + self.api_endpoint: str = self.attributes.get( + "api_endpoint", "workspace_data_validation" + ) + + +class BaseUserException(ProvisioningException): + def __init__( + self, message: str, *context_objects: object, **kwargs: str + ) -> None: + """Exception raised during user provisioning.""" + super().__init__(message, *context_objects, **kwargs) + + self.http_status: str = self.attributes.get( + "http_status", "500 Internal Server Error" + ) + self.http_method: str | None = self.attributes.get("http_method", None) + self.workspace_id: str | None = self.attributes.get( + "workspace_id", None + ) + self.user_id: str | None = self.attributes.get("user_id", None) + self.user_group_id: str | None = self.attributes.get( + "user_group_id", None + ) + self.api_endpoint: str = self.attributes.get( + "api_endpoint", "user_provisioning" + ) diff --git a/gooddata-pipelines/gooddata_pipelines/provisioning/utils/utils.py b/gooddata-pipelines/gooddata_pipelines/provisioning/utils/utils.py new file mode 100644 index 0000000..a9064cf --- /dev/null +++ b/gooddata-pipelines/gooddata_pipelines/provisioning/utils/utils.py @@ -0,0 +1,79 @@ +# (C) 2025 GoodData Corporation + +"""Module for utilities used in GoodData Pipelines provisioning.""" + +from pydantic import BaseModel +from requests import Response + + +class AttributesMixin: + """ + Mixin class to provide a method for getting attributes of an object which may or may not exist. + """ + + @staticmethod + def get_attrs( + *objects: object, overrides: dict[str, str] | None = None + ) -> dict[str, str]: + """ + Returns a dictionary of attributes from the given objects. + + Args: + objects: The objects to get the attributes from. Special handling is implemented for + requests.Response, __dict__ attribute is used for general objects. + overrides: A dictionary of attributes to override the object's attributes. + Returns: + dict: Returns a dictionary of the objects' attributes. + """ + # TODO: This might not work great with nested objects, values which are lists of objects etc. + # If we care about parsing the logs back from the string, we should consider some other approach + attrs: dict[str, str] = {} + for context_object in objects: + if isinstance(context_object, Response): + # for request.Response objects, keys need to be renamed to match the log schema + attrs.update( + { + "http_status": str(context_object.status_code), + "http_method": getattr( + context_object.request, "method", "NA" + ), + "api_endpoint": getattr( + context_object.request, "url", "NA" + ), + } + ) + else: + # Generic handling for other objects + for key, value in context_object.__dict__.items(): + if isinstance(value, list): + attrs[key] = ", ".join( + str(list_item) for list_item in value + ) + elif value is None: + continue + else: + attrs[key] = str(value) + + if overrides: + attrs.update(overrides) + + return attrs + + +class SplitMixin: + @staticmethod + def split(string_value: str, delimiter: str = ",") -> list[str]: + """ + Splits a string by the given delimiter and returns a list of stripped values. + If the input is empty, returns an empty list. + """ + if not string_value: + return [] + + return [value.strip() for value in string_value.split(delimiter)] + + +class EntityGroupIds(BaseModel): + ids_in_both_systems: set[str] + ids_to_delete: set[str] + ids_to_create: set[str] diff --git a/gooddata-pipelines/gooddata_pipelines/py.typed b/gooddata-pipelines/gooddata_pipelines/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/poetry.lock b/gooddata-pipelines/poetry.lock new file mode 100644 index 0000000..5f7c84e --- /dev/null +++ b/gooddata-pipelines/poetry.lock @@ -0,0 +1,1507 @@ +# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. + +[[package]] +name = "annotated-types" +version = "0.7.0" +description = "Reusable constraint types to use with typing.Annotated" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53"}, + {file = "annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89"}, +] + +[[package]] +name = "attrs" +version = "24.2.0" +description = "Classes Without Boilerplate" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "attrs-24.2.0-py3-none-any.whl", hash = "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2"}, + {file = "attrs-24.2.0.tar.gz", hash = "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346"}, +] + +[package.extras] +benchmark = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-codspeed", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +cov = ["cloudpickle ; platform_python_implementation == \"CPython\"", "coverage[toml] (>=5.3)", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +dev = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +docs = ["cogapp", "furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +tests = ["cloudpickle ; platform_python_implementation == \"CPython\"", "hypothesis", "mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\"", "pytest-xdist[psutil]"] +tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\"", "pytest-mypy-plugins ; platform_python_implementation == \"CPython\" and python_version >= \"3.9\" and python_version < \"3.13\""] + +[[package]] +name = "boto3" +version = "1.39.3" +description = "The AWS SDK for Python" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "boto3-1.39.3-py3-none-any.whl", hash = "sha256:056cfa2440fe1a157a7c2be897c749c83e1a322144aa4dad889f2fca66571019"}, + {file = "boto3-1.39.3.tar.gz", hash = "sha256:0a367106497649ae3d8a7b571b8c3be01b7b935a0fe303d4cc2574ed03aecbb4"}, +] + +[package.dependencies] +botocore = ">=1.39.3,<1.40.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.13.0,<0.14.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "boto3-stubs" +version = "1.39.3" +description = "Type annotations for boto3 1.39.3 generated with mypy-boto3-builder 8.11.0" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "boto3_stubs-1.39.3-py3-none-any.whl", hash = "sha256:4daddb19374efa6d1bef7aded9cede0075f380722a9e60ab129ebba14ae66b69"}, + {file = "boto3_stubs-1.39.3.tar.gz", hash = "sha256:9aad443b1d690951fd9ccb6fa20ad387bd0b1054c704566ff65dd0043a63fc26"}, +] + +[package.dependencies] +botocore-stubs = "*" +types-s3transfer = "*" +typing-extensions = {version = ">=4.1.0", markers = "python_version < \"3.12\""} + +[package.extras] +accessanalyzer = ["mypy-boto3-accessanalyzer (>=1.39.0,<1.40.0)"] +account = ["mypy-boto3-account (>=1.39.0,<1.40.0)"] +acm = ["mypy-boto3-acm (>=1.39.0,<1.40.0)"] +acm-pca = ["mypy-boto3-acm-pca (>=1.39.0,<1.40.0)"] +aiops = ["mypy-boto3-aiops (>=1.39.0,<1.40.0)"] +all = ["mypy-boto3-accessanalyzer (>=1.39.0,<1.40.0)", "mypy-boto3-account (>=1.39.0,<1.40.0)", "mypy-boto3-acm (>=1.39.0,<1.40.0)", "mypy-boto3-acm-pca (>=1.39.0,<1.40.0)", "mypy-boto3-aiops (>=1.39.0,<1.40.0)", "mypy-boto3-amp (>=1.39.0,<1.40.0)", "mypy-boto3-amplify (>=1.39.0,<1.40.0)", "mypy-boto3-amplifybackend (>=1.39.0,<1.40.0)", "mypy-boto3-amplifyuibuilder (>=1.39.0,<1.40.0)", "mypy-boto3-apigateway (>=1.39.0,<1.40.0)", "mypy-boto3-apigatewaymanagementapi (>=1.39.0,<1.40.0)", "mypy-boto3-apigatewayv2 (>=1.39.0,<1.40.0)", "mypy-boto3-appconfig (>=1.39.0,<1.40.0)", "mypy-boto3-appconfigdata (>=1.39.0,<1.40.0)", "mypy-boto3-appfabric (>=1.39.0,<1.40.0)", "mypy-boto3-appflow (>=1.39.0,<1.40.0)", "mypy-boto3-appintegrations (>=1.39.0,<1.40.0)", "mypy-boto3-application-autoscaling (>=1.39.0,<1.40.0)", "mypy-boto3-application-insights (>=1.39.0,<1.40.0)", "mypy-boto3-application-signals (>=1.39.0,<1.40.0)", "mypy-boto3-applicationcostprofiler (>=1.39.0,<1.40.0)", "mypy-boto3-appmesh (>=1.39.0,<1.40.0)", "mypy-boto3-apprunner (>=1.39.0,<1.40.0)", "mypy-boto3-appstream (>=1.39.0,<1.40.0)", "mypy-boto3-appsync (>=1.39.0,<1.40.0)", "mypy-boto3-apptest (>=1.39.0,<1.40.0)", "mypy-boto3-arc-zonal-shift (>=1.39.0,<1.40.0)", "mypy-boto3-artifact (>=1.39.0,<1.40.0)", "mypy-boto3-athena (>=1.39.0,<1.40.0)", "mypy-boto3-auditmanager (>=1.39.0,<1.40.0)", "mypy-boto3-autoscaling (>=1.39.0,<1.40.0)", "mypy-boto3-autoscaling-plans (>=1.39.0,<1.40.0)", "mypy-boto3-b2bi (>=1.39.0,<1.40.0)", "mypy-boto3-backup (>=1.39.0,<1.40.0)", "mypy-boto3-backup-gateway (>=1.39.0,<1.40.0)", "mypy-boto3-backupsearch (>=1.39.0,<1.40.0)", "mypy-boto3-batch (>=1.39.0,<1.40.0)", "mypy-boto3-bcm-data-exports (>=1.39.0,<1.40.0)", "mypy-boto3-bcm-pricing-calculator (>=1.39.0,<1.40.0)", "mypy-boto3-bedrock (>=1.39.0,<1.40.0)", "mypy-boto3-bedrock-agent (>=1.39.0,<1.40.0)", "mypy-boto3-bedrock-agent-runtime (>=1.39.0,<1.40.0)", "mypy-boto3-bedrock-data-automation (>=1.39.0,<1.40.0)", "mypy-boto3-bedrock-data-automation-runtime (>=1.39.0,<1.40.0)", "mypy-boto3-bedrock-runtime (>=1.39.0,<1.40.0)", "mypy-boto3-billing (>=1.39.0,<1.40.0)", "mypy-boto3-billingconductor (>=1.39.0,<1.40.0)", "mypy-boto3-braket (>=1.39.0,<1.40.0)", "mypy-boto3-budgets (>=1.39.0,<1.40.0)", "mypy-boto3-ce (>=1.39.0,<1.40.0)", "mypy-boto3-chatbot (>=1.39.0,<1.40.0)", "mypy-boto3-chime (>=1.39.0,<1.40.0)", "mypy-boto3-chime-sdk-identity (>=1.39.0,<1.40.0)", "mypy-boto3-chime-sdk-media-pipelines (>=1.39.0,<1.40.0)", "mypy-boto3-chime-sdk-meetings (>=1.39.0,<1.40.0)", "mypy-boto3-chime-sdk-messaging (>=1.39.0,<1.40.0)", "mypy-boto3-chime-sdk-voice (>=1.39.0,<1.40.0)", "mypy-boto3-cleanrooms (>=1.39.0,<1.40.0)", "mypy-boto3-cleanroomsml (>=1.39.0,<1.40.0)", "mypy-boto3-cloud9 (>=1.39.0,<1.40.0)", "mypy-boto3-cloudcontrol (>=1.39.0,<1.40.0)", "mypy-boto3-clouddirectory (>=1.39.0,<1.40.0)", "mypy-boto3-cloudformation (>=1.39.0,<1.40.0)", "mypy-boto3-cloudfront (>=1.39.0,<1.40.0)", "mypy-boto3-cloudfront-keyvaluestore (>=1.39.0,<1.40.0)", "mypy-boto3-cloudhsm (>=1.39.0,<1.40.0)", "mypy-boto3-cloudhsmv2 (>=1.39.0,<1.40.0)", "mypy-boto3-cloudsearch (>=1.39.0,<1.40.0)", "mypy-boto3-cloudsearchdomain (>=1.39.0,<1.40.0)", "mypy-boto3-cloudtrail (>=1.39.0,<1.40.0)", "mypy-boto3-cloudtrail-data (>=1.39.0,<1.40.0)", "mypy-boto3-cloudwatch (>=1.39.0,<1.40.0)", "mypy-boto3-codeartifact (>=1.39.0,<1.40.0)", "mypy-boto3-codebuild (>=1.39.0,<1.40.0)", "mypy-boto3-codecatalyst (>=1.39.0,<1.40.0)", "mypy-boto3-codecommit (>=1.39.0,<1.40.0)", "mypy-boto3-codeconnections (>=1.39.0,<1.40.0)", "mypy-boto3-codedeploy (>=1.39.0,<1.40.0)", "mypy-boto3-codeguru-reviewer (>=1.39.0,<1.40.0)", "mypy-boto3-codeguru-security (>=1.39.0,<1.40.0)", "mypy-boto3-codeguruprofiler (>=1.39.0,<1.40.0)", "mypy-boto3-codepipeline (>=1.39.0,<1.40.0)", "mypy-boto3-codestar-connections (>=1.39.0,<1.40.0)", "mypy-boto3-codestar-notifications (>=1.39.0,<1.40.0)", "mypy-boto3-cognito-identity (>=1.39.0,<1.40.0)", "mypy-boto3-cognito-idp (>=1.39.0,<1.40.0)", "mypy-boto3-cognito-sync (>=1.39.0,<1.40.0)", "mypy-boto3-comprehend (>=1.39.0,<1.40.0)", "mypy-boto3-comprehendmedical (>=1.39.0,<1.40.0)", "mypy-boto3-compute-optimizer (>=1.39.0,<1.40.0)", "mypy-boto3-config (>=1.39.0,<1.40.0)", "mypy-boto3-connect (>=1.39.0,<1.40.0)", "mypy-boto3-connect-contact-lens (>=1.39.0,<1.40.0)", "mypy-boto3-connectcampaigns (>=1.39.0,<1.40.0)", "mypy-boto3-connectcampaignsv2 (>=1.39.0,<1.40.0)", "mypy-boto3-connectcases (>=1.39.0,<1.40.0)", "mypy-boto3-connectparticipant (>=1.39.0,<1.40.0)", "mypy-boto3-controlcatalog (>=1.39.0,<1.40.0)", "mypy-boto3-controltower (>=1.39.0,<1.40.0)", "mypy-boto3-cost-optimization-hub (>=1.39.0,<1.40.0)", "mypy-boto3-cur (>=1.39.0,<1.40.0)", "mypy-boto3-customer-profiles (>=1.39.0,<1.40.0)", "mypy-boto3-databrew (>=1.39.0,<1.40.0)", "mypy-boto3-dataexchange (>=1.39.0,<1.40.0)", "mypy-boto3-datapipeline (>=1.39.0,<1.40.0)", "mypy-boto3-datasync (>=1.39.0,<1.40.0)", "mypy-boto3-datazone (>=1.39.0,<1.40.0)", "mypy-boto3-dax (>=1.39.0,<1.40.0)", "mypy-boto3-deadline (>=1.39.0,<1.40.0)", "mypy-boto3-detective (>=1.39.0,<1.40.0)", "mypy-boto3-devicefarm (>=1.39.0,<1.40.0)", "mypy-boto3-devops-guru (>=1.39.0,<1.40.0)", "mypy-boto3-directconnect (>=1.39.0,<1.40.0)", "mypy-boto3-discovery (>=1.39.0,<1.40.0)", "mypy-boto3-dlm (>=1.39.0,<1.40.0)", "mypy-boto3-dms (>=1.39.0,<1.40.0)", "mypy-boto3-docdb (>=1.39.0,<1.40.0)", "mypy-boto3-docdb-elastic (>=1.39.0,<1.40.0)", "mypy-boto3-drs (>=1.39.0,<1.40.0)", "mypy-boto3-ds (>=1.39.0,<1.40.0)", "mypy-boto3-ds-data (>=1.39.0,<1.40.0)", "mypy-boto3-dsql (>=1.39.0,<1.40.0)", "mypy-boto3-dynamodb (>=1.39.0,<1.40.0)", "mypy-boto3-dynamodbstreams (>=1.39.0,<1.40.0)", "mypy-boto3-ebs (>=1.39.0,<1.40.0)", "mypy-boto3-ec2 (>=1.39.0,<1.40.0)", "mypy-boto3-ec2-instance-connect (>=1.39.0,<1.40.0)", "mypy-boto3-ecr (>=1.39.0,<1.40.0)", "mypy-boto3-ecr-public (>=1.39.0,<1.40.0)", "mypy-boto3-ecs (>=1.39.0,<1.40.0)", "mypy-boto3-efs (>=1.39.0,<1.40.0)", "mypy-boto3-eks (>=1.39.0,<1.40.0)", "mypy-boto3-eks-auth (>=1.39.0,<1.40.0)", "mypy-boto3-elasticache (>=1.39.0,<1.40.0)", "mypy-boto3-elasticbeanstalk (>=1.39.0,<1.40.0)", "mypy-boto3-elastictranscoder (>=1.39.0,<1.40.0)", "mypy-boto3-elb (>=1.39.0,<1.40.0)", "mypy-boto3-elbv2 (>=1.39.0,<1.40.0)", "mypy-boto3-emr (>=1.39.0,<1.40.0)", "mypy-boto3-emr-containers (>=1.39.0,<1.40.0)", "mypy-boto3-emr-serverless (>=1.39.0,<1.40.0)", "mypy-boto3-entityresolution (>=1.39.0,<1.40.0)", "mypy-boto3-es (>=1.39.0,<1.40.0)", "mypy-boto3-events (>=1.39.0,<1.40.0)", "mypy-boto3-evidently (>=1.39.0,<1.40.0)", "mypy-boto3-evs (>=1.39.0,<1.40.0)", "mypy-boto3-finspace (>=1.39.0,<1.40.0)", "mypy-boto3-finspace-data (>=1.39.0,<1.40.0)", "mypy-boto3-firehose (>=1.39.0,<1.40.0)", "mypy-boto3-fis (>=1.39.0,<1.40.0)", "mypy-boto3-fms (>=1.39.0,<1.40.0)", "mypy-boto3-forecast (>=1.39.0,<1.40.0)", "mypy-boto3-forecastquery (>=1.39.0,<1.40.0)", "mypy-boto3-frauddetector (>=1.39.0,<1.40.0)", "mypy-boto3-freetier (>=1.39.0,<1.40.0)", "mypy-boto3-fsx (>=1.39.0,<1.40.0)", "mypy-boto3-gamelift (>=1.39.0,<1.40.0)", "mypy-boto3-gameliftstreams (>=1.39.0,<1.40.0)", "mypy-boto3-geo-maps (>=1.39.0,<1.40.0)", "mypy-boto3-geo-places (>=1.39.0,<1.40.0)", "mypy-boto3-geo-routes (>=1.39.0,<1.40.0)", "mypy-boto3-glacier (>=1.39.0,<1.40.0)", "mypy-boto3-globalaccelerator (>=1.39.0,<1.40.0)", "mypy-boto3-glue (>=1.39.0,<1.40.0)", "mypy-boto3-grafana (>=1.39.0,<1.40.0)", "mypy-boto3-greengrass (>=1.39.0,<1.40.0)", "mypy-boto3-greengrassv2 (>=1.39.0,<1.40.0)", "mypy-boto3-groundstation (>=1.39.0,<1.40.0)", "mypy-boto3-guardduty (>=1.39.0,<1.40.0)", "mypy-boto3-health (>=1.39.0,<1.40.0)", "mypy-boto3-healthlake (>=1.39.0,<1.40.0)", "mypy-boto3-iam (>=1.39.0,<1.40.0)", "mypy-boto3-identitystore (>=1.39.0,<1.40.0)", "mypy-boto3-imagebuilder (>=1.39.0,<1.40.0)", "mypy-boto3-importexport (>=1.39.0,<1.40.0)", "mypy-boto3-inspector (>=1.39.0,<1.40.0)", "mypy-boto3-inspector-scan (>=1.39.0,<1.40.0)", "mypy-boto3-inspector2 (>=1.39.0,<1.40.0)", "mypy-boto3-internetmonitor (>=1.39.0,<1.40.0)", "mypy-boto3-invoicing (>=1.39.0,<1.40.0)", "mypy-boto3-iot (>=1.39.0,<1.40.0)", "mypy-boto3-iot-data (>=1.39.0,<1.40.0)", "mypy-boto3-iot-jobs-data (>=1.39.0,<1.40.0)", "mypy-boto3-iot-managed-integrations (>=1.39.0,<1.40.0)", "mypy-boto3-iotanalytics (>=1.39.0,<1.40.0)", "mypy-boto3-iotdeviceadvisor (>=1.39.0,<1.40.0)", "mypy-boto3-iotevents (>=1.39.0,<1.40.0)", "mypy-boto3-iotevents-data (>=1.39.0,<1.40.0)", "mypy-boto3-iotfleethub (>=1.39.0,<1.40.0)", "mypy-boto3-iotfleetwise (>=1.39.0,<1.40.0)", "mypy-boto3-iotsecuretunneling (>=1.39.0,<1.40.0)", "mypy-boto3-iotsitewise (>=1.39.0,<1.40.0)", "mypy-boto3-iotthingsgraph (>=1.39.0,<1.40.0)", "mypy-boto3-iottwinmaker (>=1.39.0,<1.40.0)", "mypy-boto3-iotwireless (>=1.39.0,<1.40.0)", "mypy-boto3-ivs (>=1.39.0,<1.40.0)", "mypy-boto3-ivs-realtime (>=1.39.0,<1.40.0)", "mypy-boto3-ivschat (>=1.39.0,<1.40.0)", "mypy-boto3-kafka (>=1.39.0,<1.40.0)", "mypy-boto3-kafkaconnect (>=1.39.0,<1.40.0)", "mypy-boto3-kendra (>=1.39.0,<1.40.0)", "mypy-boto3-kendra-ranking (>=1.39.0,<1.40.0)", "mypy-boto3-keyspaces (>=1.39.0,<1.40.0)", "mypy-boto3-keyspacesstreams (>=1.39.0,<1.40.0)", "mypy-boto3-kinesis (>=1.39.0,<1.40.0)", "mypy-boto3-kinesis-video-archived-media (>=1.39.0,<1.40.0)", "mypy-boto3-kinesis-video-media (>=1.39.0,<1.40.0)", "mypy-boto3-kinesis-video-signaling (>=1.39.0,<1.40.0)", "mypy-boto3-kinesis-video-webrtc-storage (>=1.39.0,<1.40.0)", "mypy-boto3-kinesisanalytics (>=1.39.0,<1.40.0)", "mypy-boto3-kinesisanalyticsv2 (>=1.39.0,<1.40.0)", "mypy-boto3-kinesisvideo (>=1.39.0,<1.40.0)", "mypy-boto3-kms (>=1.39.0,<1.40.0)", "mypy-boto3-lakeformation (>=1.39.0,<1.40.0)", "mypy-boto3-lambda (>=1.39.0,<1.40.0)", "mypy-boto3-launch-wizard (>=1.39.0,<1.40.0)", "mypy-boto3-lex-models (>=1.39.0,<1.40.0)", "mypy-boto3-lex-runtime (>=1.39.0,<1.40.0)", "mypy-boto3-lexv2-models (>=1.39.0,<1.40.0)", "mypy-boto3-lexv2-runtime (>=1.39.0,<1.40.0)", "mypy-boto3-license-manager (>=1.39.0,<1.40.0)", "mypy-boto3-license-manager-linux-subscriptions (>=1.39.0,<1.40.0)", "mypy-boto3-license-manager-user-subscriptions (>=1.39.0,<1.40.0)", "mypy-boto3-lightsail (>=1.39.0,<1.40.0)", "mypy-boto3-location (>=1.39.0,<1.40.0)", "mypy-boto3-logs (>=1.39.0,<1.40.0)", "mypy-boto3-lookoutequipment (>=1.39.0,<1.40.0)", "mypy-boto3-lookoutmetrics (>=1.39.0,<1.40.0)", "mypy-boto3-lookoutvision (>=1.39.0,<1.40.0)", "mypy-boto3-m2 (>=1.39.0,<1.40.0)", "mypy-boto3-machinelearning (>=1.39.0,<1.40.0)", "mypy-boto3-macie2 (>=1.39.0,<1.40.0)", "mypy-boto3-mailmanager (>=1.39.0,<1.40.0)", "mypy-boto3-managedblockchain (>=1.39.0,<1.40.0)", "mypy-boto3-managedblockchain-query (>=1.39.0,<1.40.0)", "mypy-boto3-marketplace-agreement (>=1.39.0,<1.40.0)", "mypy-boto3-marketplace-catalog (>=1.39.0,<1.40.0)", "mypy-boto3-marketplace-deployment (>=1.39.0,<1.40.0)", "mypy-boto3-marketplace-entitlement (>=1.39.0,<1.40.0)", "mypy-boto3-marketplace-reporting (>=1.39.0,<1.40.0)", "mypy-boto3-marketplacecommerceanalytics (>=1.39.0,<1.40.0)", "mypy-boto3-mediaconnect (>=1.39.0,<1.40.0)", "mypy-boto3-mediaconvert (>=1.39.0,<1.40.0)", "mypy-boto3-medialive (>=1.39.0,<1.40.0)", "mypy-boto3-mediapackage (>=1.39.0,<1.40.0)", "mypy-boto3-mediapackage-vod (>=1.39.0,<1.40.0)", "mypy-boto3-mediapackagev2 (>=1.39.0,<1.40.0)", "mypy-boto3-mediastore (>=1.39.0,<1.40.0)", "mypy-boto3-mediastore-data (>=1.39.0,<1.40.0)", "mypy-boto3-mediatailor (>=1.39.0,<1.40.0)", "mypy-boto3-medical-imaging (>=1.39.0,<1.40.0)", "mypy-boto3-memorydb (>=1.39.0,<1.40.0)", "mypy-boto3-meteringmarketplace (>=1.39.0,<1.40.0)", "mypy-boto3-mgh (>=1.39.0,<1.40.0)", "mypy-boto3-mgn (>=1.39.0,<1.40.0)", "mypy-boto3-migration-hub-refactor-spaces (>=1.39.0,<1.40.0)", "mypy-boto3-migrationhub-config (>=1.39.0,<1.40.0)", "mypy-boto3-migrationhuborchestrator (>=1.39.0,<1.40.0)", "mypy-boto3-migrationhubstrategy (>=1.39.0,<1.40.0)", "mypy-boto3-mpa (>=1.39.0,<1.40.0)", "mypy-boto3-mq (>=1.39.0,<1.40.0)", "mypy-boto3-mturk (>=1.39.0,<1.40.0)", "mypy-boto3-mwaa (>=1.39.0,<1.40.0)", "mypy-boto3-neptune (>=1.39.0,<1.40.0)", "mypy-boto3-neptune-graph (>=1.39.0,<1.40.0)", "mypy-boto3-neptunedata (>=1.39.0,<1.40.0)", "mypy-boto3-network-firewall (>=1.39.0,<1.40.0)", "mypy-boto3-networkflowmonitor (>=1.39.0,<1.40.0)", "mypy-boto3-networkmanager (>=1.39.0,<1.40.0)", "mypy-boto3-networkmonitor (>=1.39.0,<1.40.0)", "mypy-boto3-notifications (>=1.39.0,<1.40.0)", "mypy-boto3-notificationscontacts (>=1.39.0,<1.40.0)", "mypy-boto3-oam (>=1.39.0,<1.40.0)", "mypy-boto3-observabilityadmin (>=1.39.0,<1.40.0)", "mypy-boto3-odb (>=1.39.0,<1.40.0)", "mypy-boto3-omics (>=1.39.0,<1.40.0)", "mypy-boto3-opensearch (>=1.39.0,<1.40.0)", "mypy-boto3-opensearchserverless (>=1.39.0,<1.40.0)", "mypy-boto3-opsworks (>=1.39.0,<1.40.0)", "mypy-boto3-opsworkscm (>=1.39.0,<1.40.0)", "mypy-boto3-organizations (>=1.39.0,<1.40.0)", "mypy-boto3-osis (>=1.39.0,<1.40.0)", "mypy-boto3-outposts (>=1.39.0,<1.40.0)", "mypy-boto3-panorama (>=1.39.0,<1.40.0)", "mypy-boto3-partnercentral-selling (>=1.39.0,<1.40.0)", "mypy-boto3-payment-cryptography (>=1.39.0,<1.40.0)", "mypy-boto3-payment-cryptography-data (>=1.39.0,<1.40.0)", "mypy-boto3-pca-connector-ad (>=1.39.0,<1.40.0)", "mypy-boto3-pca-connector-scep (>=1.39.0,<1.40.0)", "mypy-boto3-pcs (>=1.39.0,<1.40.0)", "mypy-boto3-personalize (>=1.39.0,<1.40.0)", "mypy-boto3-personalize-events (>=1.39.0,<1.40.0)", "mypy-boto3-personalize-runtime (>=1.39.0,<1.40.0)", "mypy-boto3-pi (>=1.39.0,<1.40.0)", "mypy-boto3-pinpoint (>=1.39.0,<1.40.0)", "mypy-boto3-pinpoint-email (>=1.39.0,<1.40.0)", "mypy-boto3-pinpoint-sms-voice (>=1.39.0,<1.40.0)", "mypy-boto3-pinpoint-sms-voice-v2 (>=1.39.0,<1.40.0)", "mypy-boto3-pipes (>=1.39.0,<1.40.0)", "mypy-boto3-polly (>=1.39.0,<1.40.0)", "mypy-boto3-pricing (>=1.39.0,<1.40.0)", "mypy-boto3-proton (>=1.39.0,<1.40.0)", "mypy-boto3-qapps (>=1.39.0,<1.40.0)", "mypy-boto3-qbusiness (>=1.39.0,<1.40.0)", "mypy-boto3-qconnect (>=1.39.0,<1.40.0)", "mypy-boto3-qldb (>=1.39.0,<1.40.0)", "mypy-boto3-qldb-session (>=1.39.0,<1.40.0)", "mypy-boto3-quicksight (>=1.39.0,<1.40.0)", "mypy-boto3-ram (>=1.39.0,<1.40.0)", "mypy-boto3-rbin (>=1.39.0,<1.40.0)", "mypy-boto3-rds (>=1.39.0,<1.40.0)", "mypy-boto3-rds-data (>=1.39.0,<1.40.0)", "mypy-boto3-redshift (>=1.39.0,<1.40.0)", "mypy-boto3-redshift-data (>=1.39.0,<1.40.0)", "mypy-boto3-redshift-serverless (>=1.39.0,<1.40.0)", "mypy-boto3-rekognition (>=1.39.0,<1.40.0)", "mypy-boto3-repostspace (>=1.39.0,<1.40.0)", "mypy-boto3-resiliencehub (>=1.39.0,<1.40.0)", "mypy-boto3-resource-explorer-2 (>=1.39.0,<1.40.0)", "mypy-boto3-resource-groups (>=1.39.0,<1.40.0)", "mypy-boto3-resourcegroupstaggingapi (>=1.39.0,<1.40.0)", "mypy-boto3-robomaker (>=1.39.0,<1.40.0)", "mypy-boto3-rolesanywhere (>=1.39.0,<1.40.0)", "mypy-boto3-route53 (>=1.39.0,<1.40.0)", "mypy-boto3-route53-recovery-cluster (>=1.39.0,<1.40.0)", "mypy-boto3-route53-recovery-control-config (>=1.39.0,<1.40.0)", "mypy-boto3-route53-recovery-readiness (>=1.39.0,<1.40.0)", "mypy-boto3-route53domains (>=1.39.0,<1.40.0)", "mypy-boto3-route53profiles (>=1.39.0,<1.40.0)", "mypy-boto3-route53resolver (>=1.39.0,<1.40.0)", "mypy-boto3-rum (>=1.39.0,<1.40.0)", "mypy-boto3-s3 (>=1.39.0,<1.40.0)", "mypy-boto3-s3control (>=1.39.0,<1.40.0)", "mypy-boto3-s3outposts (>=1.39.0,<1.40.0)", "mypy-boto3-s3tables (>=1.39.0,<1.40.0)", "mypy-boto3-sagemaker (>=1.39.0,<1.40.0)", "mypy-boto3-sagemaker-a2i-runtime (>=1.39.0,<1.40.0)", "mypy-boto3-sagemaker-edge (>=1.39.0,<1.40.0)", "mypy-boto3-sagemaker-featurestore-runtime (>=1.39.0,<1.40.0)", "mypy-boto3-sagemaker-geospatial (>=1.39.0,<1.40.0)", "mypy-boto3-sagemaker-metrics (>=1.39.0,<1.40.0)", "mypy-boto3-sagemaker-runtime (>=1.39.0,<1.40.0)", "mypy-boto3-savingsplans (>=1.39.0,<1.40.0)", "mypy-boto3-scheduler (>=1.39.0,<1.40.0)", "mypy-boto3-schemas (>=1.39.0,<1.40.0)", "mypy-boto3-sdb (>=1.39.0,<1.40.0)", "mypy-boto3-secretsmanager (>=1.39.0,<1.40.0)", "mypy-boto3-security-ir (>=1.39.0,<1.40.0)", "mypy-boto3-securityhub (>=1.39.0,<1.40.0)", "mypy-boto3-securitylake (>=1.39.0,<1.40.0)", "mypy-boto3-serverlessrepo (>=1.39.0,<1.40.0)", "mypy-boto3-service-quotas (>=1.39.0,<1.40.0)", "mypy-boto3-servicecatalog (>=1.39.0,<1.40.0)", "mypy-boto3-servicecatalog-appregistry (>=1.39.0,<1.40.0)", "mypy-boto3-servicediscovery (>=1.39.0,<1.40.0)", "mypy-boto3-ses (>=1.39.0,<1.40.0)", "mypy-boto3-sesv2 (>=1.39.0,<1.40.0)", "mypy-boto3-shield (>=1.39.0,<1.40.0)", "mypy-boto3-signer (>=1.39.0,<1.40.0)", "mypy-boto3-simspaceweaver (>=1.39.0,<1.40.0)", "mypy-boto3-sms (>=1.39.0,<1.40.0)", "mypy-boto3-snow-device-management (>=1.39.0,<1.40.0)", "mypy-boto3-snowball (>=1.39.0,<1.40.0)", "mypy-boto3-sns (>=1.39.0,<1.40.0)", "mypy-boto3-socialmessaging (>=1.39.0,<1.40.0)", "mypy-boto3-sqs (>=1.39.0,<1.40.0)", "mypy-boto3-ssm (>=1.39.0,<1.40.0)", "mypy-boto3-ssm-contacts (>=1.39.0,<1.40.0)", "mypy-boto3-ssm-guiconnect (>=1.39.0,<1.40.0)", "mypy-boto3-ssm-incidents (>=1.39.0,<1.40.0)", "mypy-boto3-ssm-quicksetup (>=1.39.0,<1.40.0)", "mypy-boto3-ssm-sap (>=1.39.0,<1.40.0)", "mypy-boto3-sso (>=1.39.0,<1.40.0)", "mypy-boto3-sso-admin (>=1.39.0,<1.40.0)", "mypy-boto3-sso-oidc (>=1.39.0,<1.40.0)", "mypy-boto3-stepfunctions (>=1.39.0,<1.40.0)", "mypy-boto3-storagegateway (>=1.39.0,<1.40.0)", "mypy-boto3-sts (>=1.39.0,<1.40.0)", "mypy-boto3-supplychain (>=1.39.0,<1.40.0)", "mypy-boto3-support (>=1.39.0,<1.40.0)", "mypy-boto3-support-app (>=1.39.0,<1.40.0)", "mypy-boto3-swf (>=1.39.0,<1.40.0)", "mypy-boto3-synthetics (>=1.39.0,<1.40.0)", "mypy-boto3-taxsettings (>=1.39.0,<1.40.0)", "mypy-boto3-textract (>=1.39.0,<1.40.0)", "mypy-boto3-timestream-influxdb (>=1.39.0,<1.40.0)", "mypy-boto3-timestream-query (>=1.39.0,<1.40.0)", "mypy-boto3-timestream-write (>=1.39.0,<1.40.0)", "mypy-boto3-tnb (>=1.39.0,<1.40.0)", "mypy-boto3-transcribe (>=1.39.0,<1.40.0)", "mypy-boto3-transfer (>=1.39.0,<1.40.0)", "mypy-boto3-translate (>=1.39.0,<1.40.0)", "mypy-boto3-trustedadvisor (>=1.39.0,<1.40.0)", "mypy-boto3-verifiedpermissions (>=1.39.0,<1.40.0)", "mypy-boto3-voice-id (>=1.39.0,<1.40.0)", "mypy-boto3-vpc-lattice (>=1.39.0,<1.40.0)", "mypy-boto3-waf (>=1.39.0,<1.40.0)", "mypy-boto3-waf-regional (>=1.39.0,<1.40.0)", "mypy-boto3-wafv2 (>=1.39.0,<1.40.0)", "mypy-boto3-wellarchitected (>=1.39.0,<1.40.0)", "mypy-boto3-wisdom (>=1.39.0,<1.40.0)", "mypy-boto3-workdocs (>=1.39.0,<1.40.0)", "mypy-boto3-workmail (>=1.39.0,<1.40.0)", "mypy-boto3-workmailmessageflow (>=1.39.0,<1.40.0)", "mypy-boto3-workspaces (>=1.39.0,<1.40.0)", "mypy-boto3-workspaces-instances (>=1.39.0,<1.40.0)", "mypy-boto3-workspaces-thin-client (>=1.39.0,<1.40.0)", "mypy-boto3-workspaces-web (>=1.39.0,<1.40.0)", "mypy-boto3-xray (>=1.39.0,<1.40.0)"] +amp = ["mypy-boto3-amp (>=1.39.0,<1.40.0)"] +amplify = ["mypy-boto3-amplify (>=1.39.0,<1.40.0)"] +amplifybackend = ["mypy-boto3-amplifybackend (>=1.39.0,<1.40.0)"] +amplifyuibuilder = ["mypy-boto3-amplifyuibuilder (>=1.39.0,<1.40.0)"] +apigateway = ["mypy-boto3-apigateway (>=1.39.0,<1.40.0)"] +apigatewaymanagementapi = ["mypy-boto3-apigatewaymanagementapi (>=1.39.0,<1.40.0)"] +apigatewayv2 = ["mypy-boto3-apigatewayv2 (>=1.39.0,<1.40.0)"] +appconfig = ["mypy-boto3-appconfig (>=1.39.0,<1.40.0)"] +appconfigdata = ["mypy-boto3-appconfigdata (>=1.39.0,<1.40.0)"] +appfabric = ["mypy-boto3-appfabric (>=1.39.0,<1.40.0)"] +appflow = ["mypy-boto3-appflow (>=1.39.0,<1.40.0)"] +appintegrations = ["mypy-boto3-appintegrations (>=1.39.0,<1.40.0)"] +application-autoscaling = ["mypy-boto3-application-autoscaling (>=1.39.0,<1.40.0)"] +application-insights = ["mypy-boto3-application-insights (>=1.39.0,<1.40.0)"] +application-signals = ["mypy-boto3-application-signals (>=1.39.0,<1.40.0)"] +applicationcostprofiler = ["mypy-boto3-applicationcostprofiler (>=1.39.0,<1.40.0)"] +appmesh = ["mypy-boto3-appmesh (>=1.39.0,<1.40.0)"] +apprunner = ["mypy-boto3-apprunner (>=1.39.0,<1.40.0)"] +appstream = ["mypy-boto3-appstream (>=1.39.0,<1.40.0)"] +appsync = ["mypy-boto3-appsync (>=1.39.0,<1.40.0)"] +apptest = ["mypy-boto3-apptest (>=1.39.0,<1.40.0)"] +arc-zonal-shift = ["mypy-boto3-arc-zonal-shift (>=1.39.0,<1.40.0)"] +artifact = ["mypy-boto3-artifact (>=1.39.0,<1.40.0)"] +athena = ["mypy-boto3-athena (>=1.39.0,<1.40.0)"] +auditmanager = ["mypy-boto3-auditmanager (>=1.39.0,<1.40.0)"] +autoscaling = ["mypy-boto3-autoscaling (>=1.39.0,<1.40.0)"] +autoscaling-plans = ["mypy-boto3-autoscaling-plans (>=1.39.0,<1.40.0)"] +b2bi = ["mypy-boto3-b2bi (>=1.39.0,<1.40.0)"] +backup = ["mypy-boto3-backup (>=1.39.0,<1.40.0)"] +backup-gateway = ["mypy-boto3-backup-gateway (>=1.39.0,<1.40.0)"] +backupsearch = ["mypy-boto3-backupsearch (>=1.39.0,<1.40.0)"] +batch = ["mypy-boto3-batch (>=1.39.0,<1.40.0)"] +bcm-data-exports = ["mypy-boto3-bcm-data-exports (>=1.39.0,<1.40.0)"] +bcm-pricing-calculator = ["mypy-boto3-bcm-pricing-calculator (>=1.39.0,<1.40.0)"] +bedrock = ["mypy-boto3-bedrock (>=1.39.0,<1.40.0)"] +bedrock-agent = ["mypy-boto3-bedrock-agent (>=1.39.0,<1.40.0)"] +bedrock-agent-runtime = ["mypy-boto3-bedrock-agent-runtime (>=1.39.0,<1.40.0)"] +bedrock-data-automation = ["mypy-boto3-bedrock-data-automation (>=1.39.0,<1.40.0)"] +bedrock-data-automation-runtime = ["mypy-boto3-bedrock-data-automation-runtime (>=1.39.0,<1.40.0)"] +bedrock-runtime = ["mypy-boto3-bedrock-runtime (>=1.39.0,<1.40.0)"] +billing = ["mypy-boto3-billing (>=1.39.0,<1.40.0)"] +billingconductor = ["mypy-boto3-billingconductor (>=1.39.0,<1.40.0)"] +boto3 = ["boto3 (==1.39.3)"] +braket = ["mypy-boto3-braket (>=1.39.0,<1.40.0)"] +budgets = ["mypy-boto3-budgets (>=1.39.0,<1.40.0)"] +ce = ["mypy-boto3-ce (>=1.39.0,<1.40.0)"] +chatbot = ["mypy-boto3-chatbot (>=1.39.0,<1.40.0)"] +chime = ["mypy-boto3-chime (>=1.39.0,<1.40.0)"] +chime-sdk-identity = ["mypy-boto3-chime-sdk-identity (>=1.39.0,<1.40.0)"] +chime-sdk-media-pipelines = ["mypy-boto3-chime-sdk-media-pipelines (>=1.39.0,<1.40.0)"] +chime-sdk-meetings = ["mypy-boto3-chime-sdk-meetings (>=1.39.0,<1.40.0)"] +chime-sdk-messaging = ["mypy-boto3-chime-sdk-messaging (>=1.39.0,<1.40.0)"] +chime-sdk-voice = ["mypy-boto3-chime-sdk-voice (>=1.39.0,<1.40.0)"] +cleanrooms = ["mypy-boto3-cleanrooms (>=1.39.0,<1.40.0)"] +cleanroomsml = ["mypy-boto3-cleanroomsml (>=1.39.0,<1.40.0)"] +cloud9 = ["mypy-boto3-cloud9 (>=1.39.0,<1.40.0)"] +cloudcontrol = ["mypy-boto3-cloudcontrol (>=1.39.0,<1.40.0)"] +clouddirectory = ["mypy-boto3-clouddirectory (>=1.39.0,<1.40.0)"] +cloudformation = ["mypy-boto3-cloudformation (>=1.39.0,<1.40.0)"] +cloudfront = ["mypy-boto3-cloudfront (>=1.39.0,<1.40.0)"] +cloudfront-keyvaluestore = ["mypy-boto3-cloudfront-keyvaluestore (>=1.39.0,<1.40.0)"] +cloudhsm = ["mypy-boto3-cloudhsm (>=1.39.0,<1.40.0)"] +cloudhsmv2 = ["mypy-boto3-cloudhsmv2 (>=1.39.0,<1.40.0)"] +cloudsearch = ["mypy-boto3-cloudsearch (>=1.39.0,<1.40.0)"] +cloudsearchdomain = ["mypy-boto3-cloudsearchdomain (>=1.39.0,<1.40.0)"] +cloudtrail = ["mypy-boto3-cloudtrail (>=1.39.0,<1.40.0)"] +cloudtrail-data = ["mypy-boto3-cloudtrail-data (>=1.39.0,<1.40.0)"] +cloudwatch = ["mypy-boto3-cloudwatch (>=1.39.0,<1.40.0)"] +codeartifact = ["mypy-boto3-codeartifact (>=1.39.0,<1.40.0)"] +codebuild = ["mypy-boto3-codebuild (>=1.39.0,<1.40.0)"] +codecatalyst = ["mypy-boto3-codecatalyst (>=1.39.0,<1.40.0)"] +codecommit = ["mypy-boto3-codecommit (>=1.39.0,<1.40.0)"] +codeconnections = ["mypy-boto3-codeconnections (>=1.39.0,<1.40.0)"] +codedeploy = ["mypy-boto3-codedeploy (>=1.39.0,<1.40.0)"] +codeguru-reviewer = ["mypy-boto3-codeguru-reviewer (>=1.39.0,<1.40.0)"] +codeguru-security = ["mypy-boto3-codeguru-security (>=1.39.0,<1.40.0)"] +codeguruprofiler = ["mypy-boto3-codeguruprofiler (>=1.39.0,<1.40.0)"] +codepipeline = ["mypy-boto3-codepipeline (>=1.39.0,<1.40.0)"] +codestar-connections = ["mypy-boto3-codestar-connections (>=1.39.0,<1.40.0)"] +codestar-notifications = ["mypy-boto3-codestar-notifications (>=1.39.0,<1.40.0)"] +cognito-identity = ["mypy-boto3-cognito-identity (>=1.39.0,<1.40.0)"] +cognito-idp = ["mypy-boto3-cognito-idp (>=1.39.0,<1.40.0)"] +cognito-sync = ["mypy-boto3-cognito-sync (>=1.39.0,<1.40.0)"] +comprehend = ["mypy-boto3-comprehend (>=1.39.0,<1.40.0)"] +comprehendmedical = ["mypy-boto3-comprehendmedical (>=1.39.0,<1.40.0)"] +compute-optimizer = ["mypy-boto3-compute-optimizer (>=1.39.0,<1.40.0)"] +config = ["mypy-boto3-config (>=1.39.0,<1.40.0)"] +connect = ["mypy-boto3-connect (>=1.39.0,<1.40.0)"] +connect-contact-lens = ["mypy-boto3-connect-contact-lens (>=1.39.0,<1.40.0)"] +connectcampaigns = ["mypy-boto3-connectcampaigns (>=1.39.0,<1.40.0)"] +connectcampaignsv2 = ["mypy-boto3-connectcampaignsv2 (>=1.39.0,<1.40.0)"] +connectcases = ["mypy-boto3-connectcases (>=1.39.0,<1.40.0)"] +connectparticipant = ["mypy-boto3-connectparticipant (>=1.39.0,<1.40.0)"] +controlcatalog = ["mypy-boto3-controlcatalog (>=1.39.0,<1.40.0)"] +controltower = ["mypy-boto3-controltower (>=1.39.0,<1.40.0)"] +cost-optimization-hub = ["mypy-boto3-cost-optimization-hub (>=1.39.0,<1.40.0)"] +cur = ["mypy-boto3-cur (>=1.39.0,<1.40.0)"] +customer-profiles = ["mypy-boto3-customer-profiles (>=1.39.0,<1.40.0)"] +databrew = ["mypy-boto3-databrew (>=1.39.0,<1.40.0)"] +dataexchange = ["mypy-boto3-dataexchange (>=1.39.0,<1.40.0)"] +datapipeline = ["mypy-boto3-datapipeline (>=1.39.0,<1.40.0)"] +datasync = ["mypy-boto3-datasync (>=1.39.0,<1.40.0)"] +datazone = ["mypy-boto3-datazone (>=1.39.0,<1.40.0)"] +dax = ["mypy-boto3-dax (>=1.39.0,<1.40.0)"] +deadline = ["mypy-boto3-deadline (>=1.39.0,<1.40.0)"] +detective = ["mypy-boto3-detective (>=1.39.0,<1.40.0)"] +devicefarm = ["mypy-boto3-devicefarm (>=1.39.0,<1.40.0)"] +devops-guru = ["mypy-boto3-devops-guru (>=1.39.0,<1.40.0)"] +directconnect = ["mypy-boto3-directconnect (>=1.39.0,<1.40.0)"] +discovery = ["mypy-boto3-discovery (>=1.39.0,<1.40.0)"] +dlm = ["mypy-boto3-dlm (>=1.39.0,<1.40.0)"] +dms = ["mypy-boto3-dms (>=1.39.0,<1.40.0)"] +docdb = ["mypy-boto3-docdb (>=1.39.0,<1.40.0)"] +docdb-elastic = ["mypy-boto3-docdb-elastic (>=1.39.0,<1.40.0)"] +drs = ["mypy-boto3-drs (>=1.39.0,<1.40.0)"] +ds = ["mypy-boto3-ds (>=1.39.0,<1.40.0)"] +ds-data = ["mypy-boto3-ds-data (>=1.39.0,<1.40.0)"] +dsql = ["mypy-boto3-dsql (>=1.39.0,<1.40.0)"] +dynamodb = ["mypy-boto3-dynamodb (>=1.39.0,<1.40.0)"] +dynamodbstreams = ["mypy-boto3-dynamodbstreams (>=1.39.0,<1.40.0)"] +ebs = ["mypy-boto3-ebs (>=1.39.0,<1.40.0)"] +ec2 = ["mypy-boto3-ec2 (>=1.39.0,<1.40.0)"] +ec2-instance-connect = ["mypy-boto3-ec2-instance-connect (>=1.39.0,<1.40.0)"] +ecr = ["mypy-boto3-ecr (>=1.39.0,<1.40.0)"] +ecr-public = ["mypy-boto3-ecr-public (>=1.39.0,<1.40.0)"] +ecs = ["mypy-boto3-ecs (>=1.39.0,<1.40.0)"] +efs = ["mypy-boto3-efs (>=1.39.0,<1.40.0)"] +eks = ["mypy-boto3-eks (>=1.39.0,<1.40.0)"] +eks-auth = ["mypy-boto3-eks-auth (>=1.39.0,<1.40.0)"] +elasticache = ["mypy-boto3-elasticache (>=1.39.0,<1.40.0)"] +elasticbeanstalk = ["mypy-boto3-elasticbeanstalk (>=1.39.0,<1.40.0)"] +elastictranscoder = ["mypy-boto3-elastictranscoder (>=1.39.0,<1.40.0)"] +elb = ["mypy-boto3-elb (>=1.39.0,<1.40.0)"] +elbv2 = ["mypy-boto3-elbv2 (>=1.39.0,<1.40.0)"] +emr = ["mypy-boto3-emr (>=1.39.0,<1.40.0)"] +emr-containers = ["mypy-boto3-emr-containers (>=1.39.0,<1.40.0)"] +emr-serverless = ["mypy-boto3-emr-serverless (>=1.39.0,<1.40.0)"] +entityresolution = ["mypy-boto3-entityresolution (>=1.39.0,<1.40.0)"] +es = ["mypy-boto3-es (>=1.39.0,<1.40.0)"] +essential = ["mypy-boto3-cloudformation (>=1.39.0,<1.40.0)", "mypy-boto3-dynamodb (>=1.39.0,<1.40.0)", "mypy-boto3-ec2 (>=1.39.0,<1.40.0)", "mypy-boto3-lambda (>=1.39.0,<1.40.0)", "mypy-boto3-rds (>=1.39.0,<1.40.0)", "mypy-boto3-s3 (>=1.39.0,<1.40.0)", "mypy-boto3-sqs (>=1.39.0,<1.40.0)"] +events = ["mypy-boto3-events (>=1.39.0,<1.40.0)"] +evidently = ["mypy-boto3-evidently (>=1.39.0,<1.40.0)"] +evs = ["mypy-boto3-evs (>=1.39.0,<1.40.0)"] +finspace = ["mypy-boto3-finspace (>=1.39.0,<1.40.0)"] +finspace-data = ["mypy-boto3-finspace-data (>=1.39.0,<1.40.0)"] +firehose = ["mypy-boto3-firehose (>=1.39.0,<1.40.0)"] +fis = ["mypy-boto3-fis (>=1.39.0,<1.40.0)"] +fms = ["mypy-boto3-fms (>=1.39.0,<1.40.0)"] +forecast = ["mypy-boto3-forecast (>=1.39.0,<1.40.0)"] +forecastquery = ["mypy-boto3-forecastquery (>=1.39.0,<1.40.0)"] +frauddetector = ["mypy-boto3-frauddetector (>=1.39.0,<1.40.0)"] +freetier = ["mypy-boto3-freetier (>=1.39.0,<1.40.0)"] +fsx = ["mypy-boto3-fsx (>=1.39.0,<1.40.0)"] +full = ["boto3-stubs-full (>=1.39.0,<1.40.0)"] +gamelift = ["mypy-boto3-gamelift (>=1.39.0,<1.40.0)"] +gameliftstreams = ["mypy-boto3-gameliftstreams (>=1.39.0,<1.40.0)"] +geo-maps = ["mypy-boto3-geo-maps (>=1.39.0,<1.40.0)"] +geo-places = ["mypy-boto3-geo-places (>=1.39.0,<1.40.0)"] +geo-routes = ["mypy-boto3-geo-routes (>=1.39.0,<1.40.0)"] +glacier = ["mypy-boto3-glacier (>=1.39.0,<1.40.0)"] +globalaccelerator = ["mypy-boto3-globalaccelerator (>=1.39.0,<1.40.0)"] +glue = ["mypy-boto3-glue (>=1.39.0,<1.40.0)"] +grafana = ["mypy-boto3-grafana (>=1.39.0,<1.40.0)"] +greengrass = ["mypy-boto3-greengrass (>=1.39.0,<1.40.0)"] +greengrassv2 = ["mypy-boto3-greengrassv2 (>=1.39.0,<1.40.0)"] +groundstation = ["mypy-boto3-groundstation (>=1.39.0,<1.40.0)"] +guardduty = ["mypy-boto3-guardduty (>=1.39.0,<1.40.0)"] +health = ["mypy-boto3-health (>=1.39.0,<1.40.0)"] +healthlake = ["mypy-boto3-healthlake (>=1.39.0,<1.40.0)"] +iam = ["mypy-boto3-iam (>=1.39.0,<1.40.0)"] +identitystore = ["mypy-boto3-identitystore (>=1.39.0,<1.40.0)"] +imagebuilder = ["mypy-boto3-imagebuilder (>=1.39.0,<1.40.0)"] +importexport = ["mypy-boto3-importexport (>=1.39.0,<1.40.0)"] +inspector = ["mypy-boto3-inspector (>=1.39.0,<1.40.0)"] +inspector-scan = ["mypy-boto3-inspector-scan (>=1.39.0,<1.40.0)"] +inspector2 = ["mypy-boto3-inspector2 (>=1.39.0,<1.40.0)"] +internetmonitor = ["mypy-boto3-internetmonitor (>=1.39.0,<1.40.0)"] +invoicing = ["mypy-boto3-invoicing (>=1.39.0,<1.40.0)"] +iot = ["mypy-boto3-iot (>=1.39.0,<1.40.0)"] +iot-data = ["mypy-boto3-iot-data (>=1.39.0,<1.40.0)"] +iot-jobs-data = ["mypy-boto3-iot-jobs-data (>=1.39.0,<1.40.0)"] +iot-managed-integrations = ["mypy-boto3-iot-managed-integrations (>=1.39.0,<1.40.0)"] +iotanalytics = ["mypy-boto3-iotanalytics (>=1.39.0,<1.40.0)"] +iotdeviceadvisor = ["mypy-boto3-iotdeviceadvisor (>=1.39.0,<1.40.0)"] +iotevents = ["mypy-boto3-iotevents (>=1.39.0,<1.40.0)"] +iotevents-data = ["mypy-boto3-iotevents-data (>=1.39.0,<1.40.0)"] +iotfleethub = ["mypy-boto3-iotfleethub (>=1.39.0,<1.40.0)"] +iotfleetwise = ["mypy-boto3-iotfleetwise (>=1.39.0,<1.40.0)"] +iotsecuretunneling = ["mypy-boto3-iotsecuretunneling (>=1.39.0,<1.40.0)"] +iotsitewise = ["mypy-boto3-iotsitewise (>=1.39.0,<1.40.0)"] +iotthingsgraph = ["mypy-boto3-iotthingsgraph (>=1.39.0,<1.40.0)"] +iottwinmaker = ["mypy-boto3-iottwinmaker (>=1.39.0,<1.40.0)"] +iotwireless = ["mypy-boto3-iotwireless (>=1.39.0,<1.40.0)"] +ivs = ["mypy-boto3-ivs (>=1.39.0,<1.40.0)"] +ivs-realtime = ["mypy-boto3-ivs-realtime (>=1.39.0,<1.40.0)"] +ivschat = ["mypy-boto3-ivschat (>=1.39.0,<1.40.0)"] +kafka = ["mypy-boto3-kafka (>=1.39.0,<1.40.0)"] +kafkaconnect = ["mypy-boto3-kafkaconnect (>=1.39.0,<1.40.0)"] +kendra = ["mypy-boto3-kendra (>=1.39.0,<1.40.0)"] +kendra-ranking = ["mypy-boto3-kendra-ranking (>=1.39.0,<1.40.0)"] +keyspaces = ["mypy-boto3-keyspaces (>=1.39.0,<1.40.0)"] +keyspacesstreams = ["mypy-boto3-keyspacesstreams (>=1.39.0,<1.40.0)"] +kinesis = ["mypy-boto3-kinesis (>=1.39.0,<1.40.0)"] +kinesis-video-archived-media = ["mypy-boto3-kinesis-video-archived-media (>=1.39.0,<1.40.0)"] +kinesis-video-media = ["mypy-boto3-kinesis-video-media (>=1.39.0,<1.40.0)"] +kinesis-video-signaling = ["mypy-boto3-kinesis-video-signaling (>=1.39.0,<1.40.0)"] +kinesis-video-webrtc-storage = ["mypy-boto3-kinesis-video-webrtc-storage (>=1.39.0,<1.40.0)"] +kinesisanalytics = ["mypy-boto3-kinesisanalytics (>=1.39.0,<1.40.0)"] +kinesisanalyticsv2 = ["mypy-boto3-kinesisanalyticsv2 (>=1.39.0,<1.40.0)"] +kinesisvideo = ["mypy-boto3-kinesisvideo (>=1.39.0,<1.40.0)"] +kms = ["mypy-boto3-kms (>=1.39.0,<1.40.0)"] +lakeformation = ["mypy-boto3-lakeformation (>=1.39.0,<1.40.0)"] +lambda = ["mypy-boto3-lambda (>=1.39.0,<1.40.0)"] +launch-wizard = ["mypy-boto3-launch-wizard (>=1.39.0,<1.40.0)"] +lex-models = ["mypy-boto3-lex-models (>=1.39.0,<1.40.0)"] +lex-runtime = ["mypy-boto3-lex-runtime (>=1.39.0,<1.40.0)"] +lexv2-models = ["mypy-boto3-lexv2-models (>=1.39.0,<1.40.0)"] +lexv2-runtime = ["mypy-boto3-lexv2-runtime (>=1.39.0,<1.40.0)"] +license-manager = ["mypy-boto3-license-manager (>=1.39.0,<1.40.0)"] +license-manager-linux-subscriptions = ["mypy-boto3-license-manager-linux-subscriptions (>=1.39.0,<1.40.0)"] +license-manager-user-subscriptions = ["mypy-boto3-license-manager-user-subscriptions (>=1.39.0,<1.40.0)"] +lightsail = ["mypy-boto3-lightsail (>=1.39.0,<1.40.0)"] +location = ["mypy-boto3-location (>=1.39.0,<1.40.0)"] +logs = ["mypy-boto3-logs (>=1.39.0,<1.40.0)"] +lookoutequipment = ["mypy-boto3-lookoutequipment (>=1.39.0,<1.40.0)"] +lookoutmetrics = ["mypy-boto3-lookoutmetrics (>=1.39.0,<1.40.0)"] +lookoutvision = ["mypy-boto3-lookoutvision (>=1.39.0,<1.40.0)"] +m2 = ["mypy-boto3-m2 (>=1.39.0,<1.40.0)"] +machinelearning = ["mypy-boto3-machinelearning (>=1.39.0,<1.40.0)"] +macie2 = ["mypy-boto3-macie2 (>=1.39.0,<1.40.0)"] +mailmanager = ["mypy-boto3-mailmanager (>=1.39.0,<1.40.0)"] +managedblockchain = ["mypy-boto3-managedblockchain (>=1.39.0,<1.40.0)"] +managedblockchain-query = ["mypy-boto3-managedblockchain-query (>=1.39.0,<1.40.0)"] +marketplace-agreement = ["mypy-boto3-marketplace-agreement (>=1.39.0,<1.40.0)"] +marketplace-catalog = ["mypy-boto3-marketplace-catalog (>=1.39.0,<1.40.0)"] +marketplace-deployment = ["mypy-boto3-marketplace-deployment (>=1.39.0,<1.40.0)"] +marketplace-entitlement = ["mypy-boto3-marketplace-entitlement (>=1.39.0,<1.40.0)"] +marketplace-reporting = ["mypy-boto3-marketplace-reporting (>=1.39.0,<1.40.0)"] +marketplacecommerceanalytics = ["mypy-boto3-marketplacecommerceanalytics (>=1.39.0,<1.40.0)"] +mediaconnect = ["mypy-boto3-mediaconnect (>=1.39.0,<1.40.0)"] +mediaconvert = ["mypy-boto3-mediaconvert (>=1.39.0,<1.40.0)"] +medialive = ["mypy-boto3-medialive (>=1.39.0,<1.40.0)"] +mediapackage = ["mypy-boto3-mediapackage (>=1.39.0,<1.40.0)"] +mediapackage-vod = ["mypy-boto3-mediapackage-vod (>=1.39.0,<1.40.0)"] +mediapackagev2 = ["mypy-boto3-mediapackagev2 (>=1.39.0,<1.40.0)"] +mediastore = ["mypy-boto3-mediastore (>=1.39.0,<1.40.0)"] +mediastore-data = ["mypy-boto3-mediastore-data (>=1.39.0,<1.40.0)"] +mediatailor = ["mypy-boto3-mediatailor (>=1.39.0,<1.40.0)"] +medical-imaging = ["mypy-boto3-medical-imaging (>=1.39.0,<1.40.0)"] +memorydb = ["mypy-boto3-memorydb (>=1.39.0,<1.40.0)"] +meteringmarketplace = ["mypy-boto3-meteringmarketplace (>=1.39.0,<1.40.0)"] +mgh = ["mypy-boto3-mgh (>=1.39.0,<1.40.0)"] +mgn = ["mypy-boto3-mgn (>=1.39.0,<1.40.0)"] +migration-hub-refactor-spaces = ["mypy-boto3-migration-hub-refactor-spaces (>=1.39.0,<1.40.0)"] +migrationhub-config = ["mypy-boto3-migrationhub-config (>=1.39.0,<1.40.0)"] +migrationhuborchestrator = ["mypy-boto3-migrationhuborchestrator (>=1.39.0,<1.40.0)"] +migrationhubstrategy = ["mypy-boto3-migrationhubstrategy (>=1.39.0,<1.40.0)"] +mpa = ["mypy-boto3-mpa (>=1.39.0,<1.40.0)"] +mq = ["mypy-boto3-mq (>=1.39.0,<1.40.0)"] +mturk = ["mypy-boto3-mturk (>=1.39.0,<1.40.0)"] +mwaa = ["mypy-boto3-mwaa (>=1.39.0,<1.40.0)"] +neptune = ["mypy-boto3-neptune (>=1.39.0,<1.40.0)"] +neptune-graph = ["mypy-boto3-neptune-graph (>=1.39.0,<1.40.0)"] +neptunedata = ["mypy-boto3-neptunedata (>=1.39.0,<1.40.0)"] +network-firewall = ["mypy-boto3-network-firewall (>=1.39.0,<1.40.0)"] +networkflowmonitor = ["mypy-boto3-networkflowmonitor (>=1.39.0,<1.40.0)"] +networkmanager = ["mypy-boto3-networkmanager (>=1.39.0,<1.40.0)"] +networkmonitor = ["mypy-boto3-networkmonitor (>=1.39.0,<1.40.0)"] +notifications = ["mypy-boto3-notifications (>=1.39.0,<1.40.0)"] +notificationscontacts = ["mypy-boto3-notificationscontacts (>=1.39.0,<1.40.0)"] +oam = ["mypy-boto3-oam (>=1.39.0,<1.40.0)"] +observabilityadmin = ["mypy-boto3-observabilityadmin (>=1.39.0,<1.40.0)"] +odb = ["mypy-boto3-odb (>=1.39.0,<1.40.0)"] +omics = ["mypy-boto3-omics (>=1.39.0,<1.40.0)"] +opensearch = ["mypy-boto3-opensearch (>=1.39.0,<1.40.0)"] +opensearchserverless = ["mypy-boto3-opensearchserverless (>=1.39.0,<1.40.0)"] +opsworks = ["mypy-boto3-opsworks (>=1.39.0,<1.40.0)"] +opsworkscm = ["mypy-boto3-opsworkscm (>=1.39.0,<1.40.0)"] +organizations = ["mypy-boto3-organizations (>=1.39.0,<1.40.0)"] +osis = ["mypy-boto3-osis (>=1.39.0,<1.40.0)"] +outposts = ["mypy-boto3-outposts (>=1.39.0,<1.40.0)"] +panorama = ["mypy-boto3-panorama (>=1.39.0,<1.40.0)"] +partnercentral-selling = ["mypy-boto3-partnercentral-selling (>=1.39.0,<1.40.0)"] +payment-cryptography = ["mypy-boto3-payment-cryptography (>=1.39.0,<1.40.0)"] +payment-cryptography-data = ["mypy-boto3-payment-cryptography-data (>=1.39.0,<1.40.0)"] +pca-connector-ad = ["mypy-boto3-pca-connector-ad (>=1.39.0,<1.40.0)"] +pca-connector-scep = ["mypy-boto3-pca-connector-scep (>=1.39.0,<1.40.0)"] +pcs = ["mypy-boto3-pcs (>=1.39.0,<1.40.0)"] +personalize = ["mypy-boto3-personalize (>=1.39.0,<1.40.0)"] +personalize-events = ["mypy-boto3-personalize-events (>=1.39.0,<1.40.0)"] +personalize-runtime = ["mypy-boto3-personalize-runtime (>=1.39.0,<1.40.0)"] +pi = ["mypy-boto3-pi (>=1.39.0,<1.40.0)"] +pinpoint = ["mypy-boto3-pinpoint (>=1.39.0,<1.40.0)"] +pinpoint-email = ["mypy-boto3-pinpoint-email (>=1.39.0,<1.40.0)"] +pinpoint-sms-voice = ["mypy-boto3-pinpoint-sms-voice (>=1.39.0,<1.40.0)"] +pinpoint-sms-voice-v2 = ["mypy-boto3-pinpoint-sms-voice-v2 (>=1.39.0,<1.40.0)"] +pipes = ["mypy-boto3-pipes (>=1.39.0,<1.40.0)"] +polly = ["mypy-boto3-polly (>=1.39.0,<1.40.0)"] +pricing = ["mypy-boto3-pricing (>=1.39.0,<1.40.0)"] +proton = ["mypy-boto3-proton (>=1.39.0,<1.40.0)"] +qapps = ["mypy-boto3-qapps (>=1.39.0,<1.40.0)"] +qbusiness = ["mypy-boto3-qbusiness (>=1.39.0,<1.40.0)"] +qconnect = ["mypy-boto3-qconnect (>=1.39.0,<1.40.0)"] +qldb = ["mypy-boto3-qldb (>=1.39.0,<1.40.0)"] +qldb-session = ["mypy-boto3-qldb-session (>=1.39.0,<1.40.0)"] +quicksight = ["mypy-boto3-quicksight (>=1.39.0,<1.40.0)"] +ram = ["mypy-boto3-ram (>=1.39.0,<1.40.0)"] +rbin = ["mypy-boto3-rbin (>=1.39.0,<1.40.0)"] +rds = ["mypy-boto3-rds (>=1.39.0,<1.40.0)"] +rds-data = ["mypy-boto3-rds-data (>=1.39.0,<1.40.0)"] +redshift = ["mypy-boto3-redshift (>=1.39.0,<1.40.0)"] +redshift-data = ["mypy-boto3-redshift-data (>=1.39.0,<1.40.0)"] +redshift-serverless = ["mypy-boto3-redshift-serverless (>=1.39.0,<1.40.0)"] +rekognition = ["mypy-boto3-rekognition (>=1.39.0,<1.40.0)"] +repostspace = ["mypy-boto3-repostspace (>=1.39.0,<1.40.0)"] +resiliencehub = ["mypy-boto3-resiliencehub (>=1.39.0,<1.40.0)"] +resource-explorer-2 = ["mypy-boto3-resource-explorer-2 (>=1.39.0,<1.40.0)"] +resource-groups = ["mypy-boto3-resource-groups (>=1.39.0,<1.40.0)"] +resourcegroupstaggingapi = ["mypy-boto3-resourcegroupstaggingapi (>=1.39.0,<1.40.0)"] +robomaker = ["mypy-boto3-robomaker (>=1.39.0,<1.40.0)"] +rolesanywhere = ["mypy-boto3-rolesanywhere (>=1.39.0,<1.40.0)"] +route53 = ["mypy-boto3-route53 (>=1.39.0,<1.40.0)"] +route53-recovery-cluster = ["mypy-boto3-route53-recovery-cluster (>=1.39.0,<1.40.0)"] +route53-recovery-control-config = ["mypy-boto3-route53-recovery-control-config (>=1.39.0,<1.40.0)"] +route53-recovery-readiness = ["mypy-boto3-route53-recovery-readiness (>=1.39.0,<1.40.0)"] +route53domains = ["mypy-boto3-route53domains (>=1.39.0,<1.40.0)"] +route53profiles = ["mypy-boto3-route53profiles (>=1.39.0,<1.40.0)"] +route53resolver = ["mypy-boto3-route53resolver (>=1.39.0,<1.40.0)"] +rum = ["mypy-boto3-rum (>=1.39.0,<1.40.0)"] +s3 = ["mypy-boto3-s3 (>=1.39.0,<1.40.0)"] +s3control = ["mypy-boto3-s3control (>=1.39.0,<1.40.0)"] +s3outposts = ["mypy-boto3-s3outposts (>=1.39.0,<1.40.0)"] +s3tables = ["mypy-boto3-s3tables (>=1.39.0,<1.40.0)"] +sagemaker = ["mypy-boto3-sagemaker (>=1.39.0,<1.40.0)"] +sagemaker-a2i-runtime = ["mypy-boto3-sagemaker-a2i-runtime (>=1.39.0,<1.40.0)"] +sagemaker-edge = ["mypy-boto3-sagemaker-edge (>=1.39.0,<1.40.0)"] +sagemaker-featurestore-runtime = ["mypy-boto3-sagemaker-featurestore-runtime (>=1.39.0,<1.40.0)"] +sagemaker-geospatial = ["mypy-boto3-sagemaker-geospatial (>=1.39.0,<1.40.0)"] +sagemaker-metrics = ["mypy-boto3-sagemaker-metrics (>=1.39.0,<1.40.0)"] +sagemaker-runtime = ["mypy-boto3-sagemaker-runtime (>=1.39.0,<1.40.0)"] +savingsplans = ["mypy-boto3-savingsplans (>=1.39.0,<1.40.0)"] +scheduler = ["mypy-boto3-scheduler (>=1.39.0,<1.40.0)"] +schemas = ["mypy-boto3-schemas (>=1.39.0,<1.40.0)"] +sdb = ["mypy-boto3-sdb (>=1.39.0,<1.40.0)"] +secretsmanager = ["mypy-boto3-secretsmanager (>=1.39.0,<1.40.0)"] +security-ir = ["mypy-boto3-security-ir (>=1.39.0,<1.40.0)"] +securityhub = ["mypy-boto3-securityhub (>=1.39.0,<1.40.0)"] +securitylake = ["mypy-boto3-securitylake (>=1.39.0,<1.40.0)"] +serverlessrepo = ["mypy-boto3-serverlessrepo (>=1.39.0,<1.40.0)"] +service-quotas = ["mypy-boto3-service-quotas (>=1.39.0,<1.40.0)"] +servicecatalog = ["mypy-boto3-servicecatalog (>=1.39.0,<1.40.0)"] +servicecatalog-appregistry = ["mypy-boto3-servicecatalog-appregistry (>=1.39.0,<1.40.0)"] +servicediscovery = ["mypy-boto3-servicediscovery (>=1.39.0,<1.40.0)"] +ses = ["mypy-boto3-ses (>=1.39.0,<1.40.0)"] +sesv2 = ["mypy-boto3-sesv2 (>=1.39.0,<1.40.0)"] +shield = ["mypy-boto3-shield (>=1.39.0,<1.40.0)"] +signer = ["mypy-boto3-signer (>=1.39.0,<1.40.0)"] +simspaceweaver = ["mypy-boto3-simspaceweaver (>=1.39.0,<1.40.0)"] +sms = ["mypy-boto3-sms (>=1.39.0,<1.40.0)"] +snow-device-management = ["mypy-boto3-snow-device-management (>=1.39.0,<1.40.0)"] +snowball = ["mypy-boto3-snowball (>=1.39.0,<1.40.0)"] +sns = ["mypy-boto3-sns (>=1.39.0,<1.40.0)"] +socialmessaging = ["mypy-boto3-socialmessaging (>=1.39.0,<1.40.0)"] +sqs = ["mypy-boto3-sqs (>=1.39.0,<1.40.0)"] +ssm = ["mypy-boto3-ssm (>=1.39.0,<1.40.0)"] +ssm-contacts = ["mypy-boto3-ssm-contacts (>=1.39.0,<1.40.0)"] +ssm-guiconnect = ["mypy-boto3-ssm-guiconnect (>=1.39.0,<1.40.0)"] +ssm-incidents = ["mypy-boto3-ssm-incidents (>=1.39.0,<1.40.0)"] +ssm-quicksetup = ["mypy-boto3-ssm-quicksetup (>=1.39.0,<1.40.0)"] +ssm-sap = ["mypy-boto3-ssm-sap (>=1.39.0,<1.40.0)"] +sso = ["mypy-boto3-sso (>=1.39.0,<1.40.0)"] +sso-admin = ["mypy-boto3-sso-admin (>=1.39.0,<1.40.0)"] +sso-oidc = ["mypy-boto3-sso-oidc (>=1.39.0,<1.40.0)"] +stepfunctions = ["mypy-boto3-stepfunctions (>=1.39.0,<1.40.0)"] +storagegateway = ["mypy-boto3-storagegateway (>=1.39.0,<1.40.0)"] +sts = ["mypy-boto3-sts (>=1.39.0,<1.40.0)"] +supplychain = ["mypy-boto3-supplychain (>=1.39.0,<1.40.0)"] +support = ["mypy-boto3-support (>=1.39.0,<1.40.0)"] +support-app = ["mypy-boto3-support-app (>=1.39.0,<1.40.0)"] +swf = ["mypy-boto3-swf (>=1.39.0,<1.40.0)"] +synthetics = ["mypy-boto3-synthetics (>=1.39.0,<1.40.0)"] +taxsettings = ["mypy-boto3-taxsettings (>=1.39.0,<1.40.0)"] +textract = ["mypy-boto3-textract (>=1.39.0,<1.40.0)"] +timestream-influxdb = ["mypy-boto3-timestream-influxdb (>=1.39.0,<1.40.0)"] +timestream-query = ["mypy-boto3-timestream-query (>=1.39.0,<1.40.0)"] +timestream-write = ["mypy-boto3-timestream-write (>=1.39.0,<1.40.0)"] +tnb = ["mypy-boto3-tnb (>=1.39.0,<1.40.0)"] +transcribe = ["mypy-boto3-transcribe (>=1.39.0,<1.40.0)"] +transfer = ["mypy-boto3-transfer (>=1.39.0,<1.40.0)"] +translate = ["mypy-boto3-translate (>=1.39.0,<1.40.0)"] +trustedadvisor = ["mypy-boto3-trustedadvisor (>=1.39.0,<1.40.0)"] +verifiedpermissions = ["mypy-boto3-verifiedpermissions (>=1.39.0,<1.40.0)"] +voice-id = ["mypy-boto3-voice-id (>=1.39.0,<1.40.0)"] +vpc-lattice = ["mypy-boto3-vpc-lattice (>=1.39.0,<1.40.0)"] +waf = ["mypy-boto3-waf (>=1.39.0,<1.40.0)"] +waf-regional = ["mypy-boto3-waf-regional (>=1.39.0,<1.40.0)"] +wafv2 = ["mypy-boto3-wafv2 (>=1.39.0,<1.40.0)"] +wellarchitected = ["mypy-boto3-wellarchitected (>=1.39.0,<1.40.0)"] +wisdom = ["mypy-boto3-wisdom (>=1.39.0,<1.40.0)"] +workdocs = ["mypy-boto3-workdocs (>=1.39.0,<1.40.0)"] +workmail = ["mypy-boto3-workmail (>=1.39.0,<1.40.0)"] +workmailmessageflow = ["mypy-boto3-workmailmessageflow (>=1.39.0,<1.40.0)"] +workspaces = ["mypy-boto3-workspaces (>=1.39.0,<1.40.0)"] +workspaces-instances = ["mypy-boto3-workspaces-instances (>=1.39.0,<1.40.0)"] +workspaces-thin-client = ["mypy-boto3-workspaces-thin-client (>=1.39.0,<1.40.0)"] +workspaces-web = ["mypy-boto3-workspaces-web (>=1.39.0,<1.40.0)"] +xray = ["mypy-boto3-xray (>=1.39.0,<1.40.0)"] + +[[package]] +name = "botocore" +version = "1.39.3" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "botocore-1.39.3-py3-none-any.whl", hash = "sha256:66a81cfac18ad5e9f47696c73fdf44cdbd8f8ca51ab3fca1effca0aabf61f02f"}, + {file = "botocore-1.39.3.tar.gz", hash = "sha256:da8f477e119f9f8a3aaa8b3c99d9c6856ed0a243680aa3a3fbbfc15a8d4093fb"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = {version = ">=1.25.4,<2.2.0 || >2.2.0,<3", markers = "python_version >= \"3.10\""} + +[package.extras] +crt = ["awscrt (==0.23.8)"] + +[[package]] +name = "botocore-stubs" +version = "1.38.46" +description = "Type annotations and code completion for botocore" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "botocore_stubs-1.38.46-py3-none-any.whl", hash = "sha256:cc21d9a7dd994bdd90872db4664d817c4719b51cda8004fd507a4bf65b085a75"}, + {file = "botocore_stubs-1.38.46.tar.gz", hash = "sha256:a04e69766ab8bae338911c1897492f88d05cd489cd75f06e6eb4f135f9da8c7b"}, +] + +[package.dependencies] +types-awscrt = "*" + +[package.extras] +botocore = ["botocore"] + +[[package]] +name = "brotli" +version = "1.1.0" +description = "Python bindings for the Brotli compression library" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "Brotli-1.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e1140c64812cb9b06c922e77f1c26a75ec5e3f0fb2bf92cc8c58720dec276752"}, + {file = "Brotli-1.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c8fd5270e906eef71d4a8d19b7c6a43760c6abcfcc10c9101d14eb2357418de9"}, + {file = "Brotli-1.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ae56aca0402a0f9a3431cddda62ad71666ca9d4dc3a10a142b9dce2e3c0cda3"}, + {file = "Brotli-1.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:43ce1b9935bfa1ede40028054d7f48b5469cd02733a365eec8a329ffd342915d"}, + {file = "Brotli-1.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:7c4855522edb2e6ae7fdb58e07c3ba9111e7621a8956f481c68d5d979c93032e"}, + {file = "Brotli-1.1.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:38025d9f30cf4634f8309c6874ef871b841eb3c347e90b0851f63d1ded5212da"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e6a904cb26bfefc2f0a6f240bdf5233be78cd2488900a2f846f3c3ac8489ab80"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a37b8f0391212d29b3a91a799c8e4a2855e0576911cdfb2515487e30e322253d"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:e84799f09591700a4154154cab9787452925578841a94321d5ee8fb9a9a328f0"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f66b5337fa213f1da0d9000bc8dc0cb5b896b726eefd9c6046f699b169c41b9e"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5dab0844f2cf82be357a0eb11a9087f70c5430b2c241493fc122bb6f2bb0917c"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:e4fe605b917c70283db7dfe5ada75e04561479075761a0b3866c081d035b01c1"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:1e9a65b5736232e7a7f91ff3d02277f11d339bf34099a56cdab6a8b3410a02b2"}, + {file = "Brotli-1.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:58d4b711689366d4a03ac7957ab8c28890415e267f9b6589969e74b6e42225ec"}, + {file = "Brotli-1.1.0-cp310-cp310-win32.whl", hash = "sha256:be36e3d172dc816333f33520154d708a2657ea63762ec16b62ece02ab5e4daf2"}, + {file = "Brotli-1.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c6244521dda65ea562d5a69b9a26120769b7a9fb3db2fe9545935ed6735b128"}, + {file = "Brotli-1.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:a3daabb76a78f829cafc365531c972016e4aa8d5b4bf60660ad8ecee19df7ccc"}, + {file = "Brotli-1.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c8146669223164fc87a7e3de9f81e9423c67a79d6b3447994dfb9c95da16e2d6"}, + {file = "Brotli-1.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:30924eb4c57903d5a7526b08ef4a584acc22ab1ffa085faceb521521d2de32dd"}, + {file = "Brotli-1.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ceb64bbc6eac5a140ca649003756940f8d6a7c444a68af170b3187623b43bebf"}, + {file = "Brotli-1.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a469274ad18dc0e4d316eefa616d1d0c2ff9da369af19fa6f3daa4f09671fd61"}, + {file = "Brotli-1.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:524f35912131cc2cabb00edfd8d573b07f2d9f21fa824bd3fb19725a9cf06327"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:5b3cc074004d968722f51e550b41a27be656ec48f8afaeeb45ebf65b561481dd"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:19c116e796420b0cee3da1ccec3b764ed2952ccfcc298b55a10e5610ad7885f9"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:510b5b1bfbe20e1a7b3baf5fed9e9451873559a976c1a78eebaa3b86c57b4265"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a1fd8a29719ccce974d523580987b7f8229aeace506952fa9ce1d53a033873c8"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c247dd99d39e0338a604f8c2b3bc7061d5c2e9e2ac7ba9cc1be5a69cb6cd832f"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1b2c248cd517c222d89e74669a4adfa5577e06ab68771a529060cf5a156e9757"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:2a24c50840d89ded6c9a8fdc7b6ed3692ed4e86f1c4a4a938e1e92def92933e0"}, + {file = "Brotli-1.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f31859074d57b4639318523d6ffdca586ace54271a73ad23ad021acd807eb14b"}, + {file = "Brotli-1.1.0-cp311-cp311-win32.whl", hash = "sha256:39da8adedf6942d76dc3e46653e52df937a3c4d6d18fdc94a7c29d263b1f5b50"}, + {file = "Brotli-1.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:aac0411d20e345dc0920bdec5548e438e999ff68d77564d5e9463a7ca9d3e7b1"}, + {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:32d95b80260d79926f5fab3c41701dbb818fde1c9da590e77e571eefd14abe28"}, + {file = "Brotli-1.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b760c65308ff1e462f65d69c12e4ae085cff3b332d894637f6273a12a482d09f"}, + {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:316cc9b17edf613ac76b1f1f305d2a748f1b976b033b049a6ecdfd5612c70409"}, + {file = "Brotli-1.1.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:caf9ee9a5775f3111642d33b86237b05808dafcd6268faa492250e9b78046eb2"}, + {file = "Brotli-1.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70051525001750221daa10907c77830bc889cb6d865cc0b813d9db7fefc21451"}, + {file = "Brotli-1.1.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7f4bf76817c14aa98cc6697ac02f3972cb8c3da93e9ef16b9c66573a68014f91"}, + {file = "Brotli-1.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d0c5516f0aed654134a2fc936325cc2e642f8a0e096d075209672eb321cff408"}, + {file = "Brotli-1.1.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6c3020404e0b5eefd7c9485ccf8393cfb75ec38ce75586e046573c9dc29967a0"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:4ed11165dd45ce798d99a136808a794a748d5dc38511303239d4e2363c0695dc"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:4093c631e96fdd49e0377a9c167bfd75b6d0bad2ace734c6eb20b348bc3ea180"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7e4c4629ddad63006efa0ef968c8e4751c5868ff0b1c5c40f76524e894c50248"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:861bf317735688269936f755fa136a99d1ed526883859f86e41a5d43c61d8966"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87a3044c3a35055527ac75e419dfa9f4f3667a1e887ee80360589eb8c90aabb9"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:c5529b34c1c9d937168297f2c1fde7ebe9ebdd5e121297ff9c043bdb2ae3d6fb"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:ca63e1890ede90b2e4454f9a65135a4d387a4585ff8282bb72964fab893f2111"}, + {file = "Brotli-1.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e79e6520141d792237c70bcd7a3b122d00f2613769ae0cb61c52e89fd3443839"}, + {file = "Brotli-1.1.0-cp312-cp312-win32.whl", hash = "sha256:5f4d5ea15c9382135076d2fb28dde923352fe02951e66935a9efaac8f10e81b0"}, + {file = "Brotli-1.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:906bc3a79de8c4ae5b86d3d75a8b77e44404b0f4261714306e3ad248d8ab0951"}, + {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:8bf32b98b75c13ec7cf774164172683d6e7891088f6316e54425fde1efc276d5"}, + {file = "Brotli-1.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7bc37c4d6b87fb1017ea28c9508b36bbcb0c3d18b4260fcdf08b200c74a6aee8"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3c0ef38c7a7014ffac184db9e04debe495d317cc9c6fb10071f7fefd93100a4f"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91d7cc2a76b5567591d12c01f019dd7afce6ba8cba6571187e21e2fc418ae648"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a93dde851926f4f2678e704fadeb39e16c35d8baebd5252c9fd94ce8ce68c4a0"}, + {file = "Brotli-1.1.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0db75f47be8b8abc8d9e31bc7aad0547ca26f24a54e6fd10231d623f183d089"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:6967ced6730aed543b8673008b5a391c3b1076d834ca438bbd70635c73775368"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7eedaa5d036d9336c95915035fb57422054014ebdeb6f3b42eac809928e40d0c"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:d487f5432bf35b60ed625d7e1b448e2dc855422e87469e3f450aa5552b0eb284"}, + {file = "Brotli-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:832436e59afb93e1836081a20f324cb185836c617659b07b129141a8426973c7"}, + {file = "Brotli-1.1.0-cp313-cp313-win32.whl", hash = "sha256:43395e90523f9c23a3d5bdf004733246fba087f2948f87ab28015f12359ca6a0"}, + {file = "Brotli-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:9011560a466d2eb3f5a6e4929cf4a09be405c64154e12df0dd72713f6500e32b"}, + {file = "Brotli-1.1.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a090ca607cbb6a34b0391776f0cb48062081f5f60ddcce5d11838e67a01928d1"}, + {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2de9d02f5bda03d27ede52e8cfe7b865b066fa49258cbab568720aa5be80a47d"}, + {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2333e30a5e00fe0fe55903c8832e08ee9c3b1382aacf4db26664a16528d51b4b"}, + {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4d4a848d1837973bf0f4b5e54e3bec977d99be36a7895c61abb659301b02c112"}, + {file = "Brotli-1.1.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:fdc3ff3bfccdc6b9cc7c342c03aa2400683f0cb891d46e94b64a197910dc4064"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:5eeb539606f18a0b232d4ba45adccde4125592f3f636a6182b4a8a436548b914"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:fd5f17ff8f14003595ab414e45fce13d073e0762394f957182e69035c9f3d7c2"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:069a121ac97412d1fe506da790b3e69f52254b9df4eb665cd42460c837193354"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:e93dfc1a1165e385cc8239fab7c036fb2cd8093728cbd85097b284d7b99249a2"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:aea440a510e14e818e67bfc4027880e2fb500c2ccb20ab21c7a7c8b5b4703d75"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_i686.whl", hash = "sha256:6974f52a02321b36847cd19d1b8e381bf39939c21efd6ee2fc13a28b0d99348c"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_ppc64le.whl", hash = "sha256:a7e53012d2853a07a4a79c00643832161a910674a893d296c9f1259859a289d2"}, + {file = "Brotli-1.1.0-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:d7702622a8b40c49bffb46e1e3ba2e81268d5c04a34f460978c6b5517a34dd52"}, + {file = "Brotli-1.1.0-cp36-cp36m-win32.whl", hash = "sha256:a599669fd7c47233438a56936988a2478685e74854088ef5293802123b5b2460"}, + {file = "Brotli-1.1.0-cp36-cp36m-win_amd64.whl", hash = "sha256:d143fd47fad1db3d7c27a1b1d66162e855b5d50a89666af46e1679c496e8e579"}, + {file = "Brotli-1.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:11d00ed0a83fa22d29bc6b64ef636c4552ebafcef57154b4ddd132f5638fbd1c"}, + {file = "Brotli-1.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f733d788519c7e3e71f0855c96618720f5d3d60c3cb829d8bbb722dddce37985"}, + {file = "Brotli-1.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:929811df5462e182b13920da56c6e0284af407d1de637d8e536c5cd00a7daf60"}, + {file = "Brotli-1.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0b63b949ff929fbc2d6d3ce0e924c9b93c9785d877a21a1b678877ffbbc4423a"}, + {file = "Brotli-1.1.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:d192f0f30804e55db0d0e0a35d83a9fead0e9a359a9ed0285dbacea60cc10a84"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:f296c40e23065d0d6650c4aefe7470d2a25fffda489bcc3eb66083f3ac9f6643"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:919e32f147ae93a09fe064d77d5ebf4e35502a8df75c29fb05788528e330fe74"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:23032ae55523cc7bccb4f6a0bf368cd25ad9bcdcc1990b64a647e7bbcce9cb5b"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:224e57f6eac61cc449f498cc5f0e1725ba2071a3d4f48d5d9dffba42db196438"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:cb1dac1770878ade83f2ccdf7d25e494f05c9165f5246b46a621cc849341dc01"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:3ee8a80d67a4334482d9712b8e83ca6b1d9bc7e351931252ebef5d8f7335a547"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:5e55da2c8724191e5b557f8e18943b1b4839b8efc3ef60d65985bcf6f587dd38"}, + {file = "Brotli-1.1.0-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:d342778ef319e1026af243ed0a07c97acf3bad33b9f29e7ae6a1f68fd083e90c"}, + {file = "Brotli-1.1.0-cp37-cp37m-win32.whl", hash = "sha256:587ca6d3cef6e4e868102672d3bd9dc9698c309ba56d41c2b9c85bbb903cdb95"}, + {file = "Brotli-1.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:2954c1c23f81c2eaf0b0717d9380bd348578a94161a65b3a2afc62c86467dd68"}, + {file = "Brotli-1.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:efa8b278894b14d6da122a72fefcebc28445f2d3f880ac59d46c90f4c13be9a3"}, + {file = "Brotli-1.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:03d20af184290887bdea3f0f78c4f737d126c74dc2f3ccadf07e54ceca3bf208"}, + {file = "Brotli-1.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6172447e1b368dcbc458925e5ddaf9113477b0ed542df258d84fa28fc45ceea7"}, + {file = "Brotli-1.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a743e5a28af5f70f9c080380a5f908d4d21d40e8f0e0c8901604d15cfa9ba751"}, + {file = "Brotli-1.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0541e747cce78e24ea12d69176f6a7ddb690e62c425e01d31cc065e69ce55b48"}, + {file = "Brotli-1.1.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:cdbc1fc1bc0bff1cef838eafe581b55bfbffaed4ed0318b724d0b71d4d377619"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:890b5a14ce214389b2cc36ce82f3093f96f4cc730c1cffdbefff77a7c71f2a97"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1ab4fbee0b2d9098c74f3057b2bc055a8bd92ccf02f65944a241b4349229185a"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:141bd4d93984070e097521ed07e2575b46f817d08f9fa42b16b9b5f27b5ac088"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fce1473f3ccc4187f75b4690cfc922628aed4d3dd013d047f95a9b3919a86596"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:d2b35ca2c7f81d173d2fadc2f4f31e88cc5f7a39ae5b6db5513cf3383b0e0ec7"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:af6fa6817889314555aede9a919612b23739395ce767fe7fcbea9a80bf140fe5"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:2feb1d960f760a575dbc5ab3b1c00504b24caaf6986e2dc2b01c09c87866a943"}, + {file = "Brotli-1.1.0-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:4410f84b33374409552ac9b6903507cdb31cd30d2501fc5ca13d18f73548444a"}, + {file = "Brotli-1.1.0-cp38-cp38-win32.whl", hash = "sha256:db85ecf4e609a48f4b29055f1e144231b90edc90af7481aa731ba2d059226b1b"}, + {file = "Brotli-1.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3d7954194c36e304e1523f55d7042c59dc53ec20dd4e9ea9d151f1b62b4415c0"}, + {file = "Brotli-1.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5fb2ce4b8045c78ebbc7b8f3c15062e435d47e7393cc57c25115cfd49883747a"}, + {file = "Brotli-1.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7905193081db9bfa73b1219140b3d315831cbff0d8941f22da695832f0dd188f"}, + {file = "Brotli-1.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a77def80806c421b4b0af06f45d65a136e7ac0bdca3c09d9e2ea4e515367c7e9"}, + {file = "Brotli-1.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8dadd1314583ec0bf2d1379f7008ad627cd6336625d6679cf2f8e67081b83acf"}, + {file = "Brotli-1.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:901032ff242d479a0efa956d853d16875d42157f98951c0230f69e69f9c09bac"}, + {file = "Brotli-1.1.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:22fc2a8549ffe699bfba2256ab2ed0421a7b8fadff114a3d201794e45a9ff578"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ae15b066e5ad21366600ebec29a7ccbc86812ed267e4b28e860b8ca16a2bc474"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:949f3b7c29912693cee0afcf09acd6ebc04c57af949d9bf77d6101ebb61e388c"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:89f4988c7203739d48c6f806f1e87a1d96e0806d44f0fba61dba81392c9e474d"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:de6551e370ef19f8de1807d0a9aa2cdfdce2e85ce88b122fe9f6b2b076837e59"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:0737ddb3068957cf1b054899b0883830bb1fec522ec76b1098f9b6e0f02d9419"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:4f3607b129417e111e30637af1b56f24f7a49e64763253bbc275c75fa887d4b2"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:6c6e0c425f22c1c719c42670d561ad682f7bfeeef918edea971a79ac5252437f"}, + {file = "Brotli-1.1.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:494994f807ba0b92092a163a0a283961369a65f6cbe01e8891132b7a320e61eb"}, + {file = "Brotli-1.1.0-cp39-cp39-win32.whl", hash = "sha256:f0d8a7a6b5983c2496e364b969f0e526647a06b075d034f3297dc66f3b360c64"}, + {file = "Brotli-1.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:cdad5b9014d83ca68c25d2e9444e28e967ef16e80f6b436918c700c117a85467"}, + {file = "Brotli-1.1.0.tar.gz", hash = "sha256:81de08ac11bcb85841e440c13611c00b67d3bf82698314928d0b676362546724"}, +] + +[[package]] +name = "cattrs" +version = "24.1.1" +description = "Composable complex class support for attrs and dataclasses." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "cattrs-24.1.1-py3-none-any.whl", hash = "sha256:ec8ce8fdc725de9d07547cd616f968670687c6fa7a2e263b088370c46d834d97"}, + {file = "cattrs-24.1.1.tar.gz", hash = "sha256:16e94a13f9aaf6438bd5be5df521e072b1b00481b4cf807bcb1acbd49f814c08"}, +] + +[package.dependencies] +attrs = ">=23.1.0" +exceptiongroup = {version = ">=1.1.1", markers = "python_version < \"3.11\""} +typing-extensions = {version = ">=4.1.0,<4.6.3 || >4.6.3", markers = "python_version < \"3.11\""} + +[package.extras] +bson = ["pymongo (>=4.4.0)"] +cbor2 = ["cbor2 (>=5.4.6)"] +msgpack = ["msgpack (>=1.0.5)"] +msgspec = ["msgspec (>=0.18.5) ; implementation_name == \"cpython\""] +orjson = ["orjson (>=3.9.2) ; implementation_name == \"cpython\""] +pyyaml = ["pyyaml (>=6.0)"] +tomlkit = ["tomlkit (>=0.11.8)"] +ujson = ["ujson (>=5.7.0)"] + +[[package]] +name = "certifi" +version = "2025.4.26" +description = "Python package for providing Mozilla's CA Bundle." +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "certifi-2025.4.26-py3-none-any.whl", hash = "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3"}, + {file = "certifi-2025.4.26.tar.gz", hash = "sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6"}, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.2" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "charset_normalizer-3.4.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941"}, + {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd"}, + {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6"}, + {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d"}, + {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86"}, + {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c"}, + {file = "charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0"}, + {file = "charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef"}, + {file = "charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6"}, + {file = "charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366"}, + {file = "charset_normalizer-3.4.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db"}, + {file = "charset_normalizer-3.4.2-cp310-cp310-win32.whl", hash = "sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a"}, + {file = "charset_normalizer-3.4.2-cp310-cp310-win_amd64.whl", hash = "sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-win32.whl", hash = "sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a"}, + {file = "charset_normalizer-3.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-win32.whl", hash = "sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c"}, + {file = "charset_normalizer-3.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-win32.whl", hash = "sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7"}, + {file = "charset_normalizer-3.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980"}, + {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184"}, + {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa"}, + {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344"}, + {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da"}, + {file = "charset_normalizer-3.4.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02"}, + {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d"}, + {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4"}, + {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f"}, + {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_s390x.whl", hash = "sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64"}, + {file = "charset_normalizer-3.4.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f"}, + {file = "charset_normalizer-3.4.2-cp37-cp37m-win32.whl", hash = "sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58"}, + {file = "charset_normalizer-3.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_s390x.whl", hash = "sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-win32.whl", hash = "sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7"}, + {file = "charset_normalizer-3.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_s390x.whl", hash = "sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-win32.whl", hash = "sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471"}, + {file = "charset_normalizer-3.4.2-cp39-cp39-win_amd64.whl", hash = "sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e"}, + {file = "charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0"}, + {file = "charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63"}, +] + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +optional = true +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main"] +markers = "extra == \"dev\" and sys_platform == \"win32\"" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.11\"" +files = [ + {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, + {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "gooddata-api-client" +version = "1.43.0" +description = "OpenAPI definition" +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "gooddata_api_client-1.43.0-py3-none-any.whl", hash = "sha256:9aa0cd7212b2d60213d88c81774204e9b992d868b364ec4b484cb4eb66d11f5e"}, + {file = "gooddata_api_client-1.43.0.tar.gz", hash = "sha256:7115dbbee54b4964c1c679b95d019261283b7d304b7234bfaf2bbbcf73bde87b"}, +] + +[package.dependencies] +python-dateutil = "*" +urllib3 = ">=1.25.3" + +[[package]] +name = "gooddata-sdk" +version = "1.43.0" +description = "GoodData Cloud Python SDK" +optional = false +python-versions = ">=3.9.0" +groups = ["main"] +files = [ + {file = "gooddata_sdk-1.43.0-py3-none-any.whl", hash = "sha256:04e7d7f1e4f4849713198f70af463d5042f3db947dec9251b96972ea8a317a36"}, + {file = "gooddata_sdk-1.43.0.tar.gz", hash = "sha256:f671c919086180ad91162c9ae20c395eae91d46e8d9fc12532505992f0093173"}, +] + +[package.dependencies] +attrs = ">=21.4.0,<=24.2.0" +brotli = "1.1.0" +cattrs = ">=22.1.0,<=24.1.1" +gooddata-api-client = ">=1.43.0,<1.44.0" +python-dateutil = ">=2.5.3" +python-dotenv = ">=1.0.0,<2.0.0" +pyyaml = ">=6.0" +requests = ">=2.32.0,<2.33.0" + +[[package]] +name = "idna" +version = "3.10" +description = "Internationalized Domain Names in Applications (IDNA)" +optional = false +python-versions = ">=3.6" +groups = ["main"] +files = [ + {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, + {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, +] + +[package.extras] +all = ["flake8 (>=7.1.1)", "mypy (>=1.11.2)", "pytest (>=8.3.2)", "ruff (>=0.6.2)"] + +[[package]] +name = "iniconfig" +version = "2.1.0" +description = "brain-dead simple config-ini parsing" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"dev\"" +files = [ + {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, + {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980"}, + {file = "jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe"}, +] + +[[package]] +name = "mypy" +version = "1.16.0" +description = "Optional static typing for Python" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"dev\"" +files = [ + {file = "mypy-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:7909541fef256527e5ee9c0a7e2aeed78b6cda72ba44298d1334fe7881b05c5c"}, + {file = "mypy-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e71d6f0090c2256c713ed3d52711d01859c82608b5d68d4fa01a3fe30df95571"}, + {file = "mypy-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:936ccfdd749af4766be824268bfe22d1db9eb2f34a3ea1d00ffbe5b5265f5491"}, + {file = "mypy-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4086883a73166631307fdd330c4a9080ce24913d4f4c5ec596c601b3a4bdd777"}, + {file = "mypy-1.16.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:feec38097f71797da0231997e0de3a58108c51845399669ebc532c815f93866b"}, + {file = "mypy-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:09a8da6a0ee9a9770b8ff61b39c0bb07971cda90e7297f4213741b48a0cc8d93"}, + {file = "mypy-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9f826aaa7ff8443bac6a494cf743f591488ea940dd360e7dd330e30dd772a5ab"}, + {file = "mypy-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:82d056e6faa508501af333a6af192c700b33e15865bda49611e3d7d8358ebea2"}, + {file = "mypy-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:089bedc02307c2548eb51f426e085546db1fa7dd87fbb7c9fa561575cf6eb1ff"}, + {file = "mypy-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6a2322896003ba66bbd1318c10d3afdfe24e78ef12ea10e2acd985e9d684a666"}, + {file = "mypy-1.16.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:021a68568082c5b36e977d54e8f1de978baf401a33884ffcea09bd8e88a98f4c"}, + {file = "mypy-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:54066fed302d83bf5128632d05b4ec68412e1f03ef2c300434057d66866cea4b"}, + {file = "mypy-1.16.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c5436d11e89a3ad16ce8afe752f0f373ae9620841c50883dc96f8b8805620b13"}, + {file = "mypy-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f2622af30bf01d8fc36466231bdd203d120d7a599a6d88fb22bdcb9dbff84090"}, + {file = "mypy-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d045d33c284e10a038f5e29faca055b90eee87da3fc63b8889085744ebabb5a1"}, + {file = "mypy-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b4968f14f44c62e2ec4a038c8797a87315be8df7740dc3ee8d3bfe1c6bf5dba8"}, + {file = "mypy-1.16.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eb14a4a871bb8efb1e4a50360d4e3c8d6c601e7a31028a2c79f9bb659b63d730"}, + {file = "mypy-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:bd4e1ebe126152a7bbaa4daedd781c90c8f9643c79b9748caa270ad542f12bec"}, + {file = "mypy-1.16.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:a9e056237c89f1587a3be1a3a70a06a698d25e2479b9a2f57325ddaaffc3567b"}, + {file = "mypy-1.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0b07e107affb9ee6ce1f342c07f51552d126c32cd62955f59a7db94a51ad12c0"}, + {file = "mypy-1.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c6fb60cbd85dc65d4d63d37cb5c86f4e3a301ec605f606ae3a9173e5cf34997b"}, + {file = "mypy-1.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a7e32297a437cc915599e0578fa6bc68ae6a8dc059c9e009c628e1c47f91495d"}, + {file = "mypy-1.16.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:afe420c9380ccec31e744e8baff0d406c846683681025db3531b32db56962d52"}, + {file = "mypy-1.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:55f9076c6ce55dd3f8cd0c6fff26a008ca8e5131b89d5ba6d86bd3f47e736eeb"}, + {file = "mypy-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f56236114c425620875c7cf71700e3d60004858da856c6fc78998ffe767b73d3"}, + {file = "mypy-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:15486beea80be24ff067d7d0ede673b001d0d684d0095803b3e6e17a886a2a92"}, + {file = "mypy-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f2ed0e0847a80655afa2c121835b848ed101cc7b8d8d6ecc5205aedc732b1436"}, + {file = "mypy-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eb5fbc8063cb4fde7787e4c0406aa63094a34a2daf4673f359a1fb64050e9cb2"}, + {file = "mypy-1.16.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a5fcfdb7318c6a8dd127b14b1052743b83e97a970f0edb6c913211507a255e20"}, + {file = "mypy-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:2e7e0ad35275e02797323a5aa1be0b14a4d03ffdb2e5f2b0489fa07b89c67b21"}, + {file = "mypy-1.16.0-py3-none-any.whl", hash = "sha256:29e1499864a3888bca5c1542f2d7232c6e586295183320caa95758fc84034031"}, + {file = "mypy-1.16.0.tar.gz", hash = "sha256:84b94283f817e2aa6350a14b4a8fb2a35a53c286f97c9d30f53b63620e7af8ab"}, +] + +[package.dependencies] +mypy_extensions = ">=1.0.0" +pathspec = ">=0.9.0" +tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +typing_extensions = ">=4.6.0" + +[package.extras] +dmypy = ["psutil (>=4.0)"] +faster-cache = ["orjson"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] +reports = ["lxml"] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +description = "Type system extensions for programs checked with the mypy type checker." +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"dev\"" +files = [ + {file = "mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505"}, + {file = "mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558"}, +] + +[[package]] +name = "packaging" +version = "25.0" +description = "Core utilities for Python packages" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"dev\"" +files = [ + {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, + {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +description = "Utility library for gitignore style pattern matching of file paths." +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"dev\"" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +description = "plugin and hook calling mechanisms for python" +optional = true +python-versions = ">=3.9" +groups = ["main"] +markers = "extra == \"dev\"" +files = [ + {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, + {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, +] + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["coverage", "pytest", "pytest-benchmark"] + +[[package]] +name = "pydantic" +version = "2.11.3" +description = "Data validation using Python type hints" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pydantic-2.11.3-py3-none-any.whl", hash = "sha256:a082753436a07f9ba1289c6ffa01cd93db3548776088aa917cc43b63f68fa60f"}, + {file = "pydantic-2.11.3.tar.gz", hash = "sha256:7471657138c16adad9322fe3070c0116dd6c3ad8d649300e3cbdfe91f4db4ec3"}, +] + +[package.dependencies] +annotated-types = ">=0.6.0" +pydantic-core = "2.33.1" +typing-extensions = ">=4.12.2" +typing-inspection = ">=0.4.0" + +[package.extras] +email = ["email-validator (>=2.0.0)"] +timezone = ["tzdata ; python_version >= \"3.9\" and platform_system == \"Windows\""] + +[[package]] +name = "pydantic-core" +version = "2.33.1" +description = "Core functionality for Pydantic validation and serialization" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "pydantic_core-2.33.1-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:3077cfdb6125cc8dab61b155fdd714663e401f0e6883f9632118ec12cf42df26"}, + {file = "pydantic_core-2.33.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8ffab8b2908d152e74862d276cf5017c81a2f3719f14e8e3e8d6b83fda863927"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5183e4f6a2d468787243ebcd70cf4098c247e60d73fb7d68d5bc1e1beaa0c4db"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:398a38d323f37714023be1e0285765f0a27243a8b1506b7b7de87b647b517e48"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87d3776f0001b43acebfa86f8c64019c043b55cc5a6a2e313d728b5c95b46969"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c566dd9c5f63d22226409553531f89de0cac55397f2ab8d97d6f06cfce6d947e"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a0d5f3acc81452c56895e90643a625302bd6be351e7010664151cc55b7b97f89"}, + {file = "pydantic_core-2.33.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:d3a07fadec2a13274a8d861d3d37c61e97a816beae717efccaa4b36dfcaadcde"}, + {file = "pydantic_core-2.33.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:f99aeda58dce827f76963ee87a0ebe75e648c72ff9ba1174a253f6744f518f65"}, + {file = "pydantic_core-2.33.1-cp310-cp310-musllinux_1_1_armv7l.whl", hash = "sha256:902dbc832141aa0ec374f4310f1e4e7febeebc3256f00dc359a9ac3f264a45dc"}, + {file = "pydantic_core-2.33.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fe44d56aa0b00d66640aa84a3cbe80b7a3ccdc6f0b1ca71090696a6d4777c091"}, + {file = "pydantic_core-2.33.1-cp310-cp310-win32.whl", hash = "sha256:ed3eb16d51257c763539bde21e011092f127a2202692afaeaccb50db55a31383"}, + {file = "pydantic_core-2.33.1-cp310-cp310-win_amd64.whl", hash = "sha256:694ad99a7f6718c1a498dc170ca430687a39894a60327f548e02a9c7ee4b6504"}, + {file = "pydantic_core-2.33.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:6e966fc3caaf9f1d96b349b0341c70c8d6573bf1bac7261f7b0ba88f96c56c24"}, + {file = "pydantic_core-2.33.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bfd0adeee563d59c598ceabddf2c92eec77abcb3f4a391b19aa7366170bd9e30"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91815221101ad3c6b507804178a7bb5cb7b2ead9ecd600041669c8d805ebd595"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:9fea9c1869bb4742d174a57b4700c6dadea951df8b06de40c2fedb4f02931c2e"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d20eb4861329bb2484c021b9d9a977566ab16d84000a57e28061151c62b349a"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0fb935c5591573ae3201640579f30128ccc10739b45663f93c06796854405505"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c964fd24e6166420d18fb53996d8c9fd6eac9bf5ae3ec3d03015be4414ce497f"}, + {file = "pydantic_core-2.33.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:681d65e9011f7392db5aa002b7423cc442d6a673c635668c227c6c8d0e5a4f77"}, + {file = "pydantic_core-2.33.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e100c52f7355a48413e2999bfb4e139d2977a904495441b374f3d4fb4a170961"}, + {file = "pydantic_core-2.33.1-cp311-cp311-musllinux_1_1_armv7l.whl", hash = "sha256:048831bd363490be79acdd3232f74a0e9951b11b2b4cc058aeb72b22fdc3abe1"}, + {file = "pydantic_core-2.33.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bdc84017d28459c00db6f918a7272a5190bec3090058334e43a76afb279eac7c"}, + {file = "pydantic_core-2.33.1-cp311-cp311-win32.whl", hash = "sha256:32cd11c5914d1179df70406427097c7dcde19fddf1418c787540f4b730289896"}, + {file = "pydantic_core-2.33.1-cp311-cp311-win_amd64.whl", hash = "sha256:2ea62419ba8c397e7da28a9170a16219d310d2cf4970dbc65c32faf20d828c83"}, + {file = "pydantic_core-2.33.1-cp311-cp311-win_arm64.whl", hash = "sha256:fc903512177361e868bc1f5b80ac8c8a6e05fcdd574a5fb5ffeac5a9982b9e89"}, + {file = "pydantic_core-2.33.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:1293d7febb995e9d3ec3ea09caf1a26214eec45b0f29f6074abb004723fc1de8"}, + {file = "pydantic_core-2.33.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:99b56acd433386c8f20be5c4000786d1e7ca0523c8eefc995d14d79c7a081498"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:35a5ec3fa8c2fe6c53e1b2ccc2454398f95d5393ab398478f53e1afbbeb4d939"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b172f7b9d2f3abc0efd12e3386f7e48b576ef309544ac3a63e5e9cdd2e24585d"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9097b9f17f91eea659b9ec58148c0747ec354a42f7389b9d50701610d86f812e"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cc77ec5b7e2118b152b0d886c7514a4653bcb58c6b1d760134a9fab915f777b3"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5e3d15245b08fa4a84cefc6c9222e6f37c98111c8679fbd94aa145f9a0ae23d"}, + {file = "pydantic_core-2.33.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ef99779001d7ac2e2461d8ab55d3373fe7315caefdbecd8ced75304ae5a6fc6b"}, + {file = "pydantic_core-2.33.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:fc6bf8869e193855e8d91d91f6bf59699a5cdfaa47a404e278e776dd7f168b39"}, + {file = "pydantic_core-2.33.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:b1caa0bc2741b043db7823843e1bde8aaa58a55a58fda06083b0569f8b45693a"}, + {file = "pydantic_core-2.33.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:ec259f62538e8bf364903a7d0d0239447059f9434b284f5536e8402b7dd198db"}, + {file = "pydantic_core-2.33.1-cp312-cp312-win32.whl", hash = "sha256:e14f369c98a7c15772b9da98987f58e2b509a93235582838bd0d1d8c08b68fda"}, + {file = "pydantic_core-2.33.1-cp312-cp312-win_amd64.whl", hash = "sha256:1c607801d85e2e123357b3893f82c97a42856192997b95b4d8325deb1cd0c5f4"}, + {file = "pydantic_core-2.33.1-cp312-cp312-win_arm64.whl", hash = "sha256:8d13f0276806ee722e70a1c93da19748594f19ac4299c7e41237fc791d1861ea"}, + {file = "pydantic_core-2.33.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:70af6a21237b53d1fe7b9325b20e65cbf2f0a848cf77bed492b029139701e66a"}, + {file = "pydantic_core-2.33.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:282b3fe1bbbe5ae35224a0dbd05aed9ccabccd241e8e6b60370484234b456266"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4b315e596282bbb5822d0c7ee9d255595bd7506d1cb20c2911a4da0b970187d3"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:1dfae24cf9921875ca0ca6a8ecb4bb2f13c855794ed0d468d6abbec6e6dcd44a"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6dd8ecfde08d8bfadaea669e83c63939af76f4cf5538a72597016edfa3fad516"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2f593494876eae852dc98c43c6f260f45abdbfeec9e4324e31a481d948214764"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:948b73114f47fd7016088e5186d13faf5e1b2fe83f5e320e371f035557fd264d"}, + {file = "pydantic_core-2.33.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e11f3864eb516af21b01e25fac915a82e9ddad3bb0fb9e95a246067398b435a4"}, + {file = "pydantic_core-2.33.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:549150be302428b56fdad0c23c2741dcdb5572413776826c965619a25d9c6bde"}, + {file = "pydantic_core-2.33.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:495bc156026efafd9ef2d82372bd38afce78ddd82bf28ef5276c469e57c0c83e"}, + {file = "pydantic_core-2.33.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ec79de2a8680b1a67a07490bddf9636d5c2fab609ba8c57597e855fa5fa4dacd"}, + {file = "pydantic_core-2.33.1-cp313-cp313-win32.whl", hash = "sha256:ee12a7be1742f81b8a65b36c6921022301d466b82d80315d215c4c691724986f"}, + {file = "pydantic_core-2.33.1-cp313-cp313-win_amd64.whl", hash = "sha256:ede9b407e39949d2afc46385ce6bd6e11588660c26f80576c11c958e6647bc40"}, + {file = "pydantic_core-2.33.1-cp313-cp313-win_arm64.whl", hash = "sha256:aa687a23d4b7871a00e03ca96a09cad0f28f443690d300500603bd0adba4b523"}, + {file = "pydantic_core-2.33.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:401d7b76e1000d0dd5538e6381d28febdcacb097c8d340dde7d7fc6e13e9f95d"}, + {file = "pydantic_core-2.33.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7aeb055a42d734c0255c9e489ac67e75397d59c6fbe60d155851e9782f276a9c"}, + {file = "pydantic_core-2.33.1-cp313-cp313t-win_amd64.whl", hash = "sha256:338ea9b73e6e109f15ab439e62cb3b78aa752c7fd9536794112e14bee02c8d18"}, + {file = "pydantic_core-2.33.1-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:5ab77f45d33d264de66e1884fca158bc920cb5e27fd0764a72f72f5756ae8bdb"}, + {file = "pydantic_core-2.33.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e7aaba1b4b03aaea7bb59e1b5856d734be011d3e6d98f5bcaa98cb30f375f2ad"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7fb66263e9ba8fea2aa85e1e5578980d127fb37d7f2e292773e7bc3a38fb0c7b"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:3f2648b9262607a7fb41d782cc263b48032ff7a03a835581abbf7a3bec62bcf5"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:723c5630c4259400818b4ad096735a829074601805d07f8cafc366d95786d331"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d100e3ae783d2167782391e0c1c7a20a31f55f8015f3293647544df3f9c67824"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:177d50460bc976a0369920b6c744d927b0ecb8606fb56858ff542560251b19e5"}, + {file = "pydantic_core-2.33.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3edde68d1a1f9af1273b2fe798997b33f90308fb6d44d8550c89fc6a3647cf6"}, + {file = "pydantic_core-2.33.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a62c3c3ef6a7e2c45f7853b10b5bc4ddefd6ee3cd31024754a1a5842da7d598d"}, + {file = "pydantic_core-2.33.1-cp39-cp39-musllinux_1_1_armv7l.whl", hash = "sha256:c91dbb0ab683fa0cd64a6e81907c8ff41d6497c346890e26b23de7ee55353f96"}, + {file = "pydantic_core-2.33.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9f466e8bf0a62dc43e068c12166281c2eca72121dd2adc1040f3aa1e21ef8599"}, + {file = "pydantic_core-2.33.1-cp39-cp39-win32.whl", hash = "sha256:ab0277cedb698749caada82e5d099dc9fed3f906a30d4c382d1a21725777a1e5"}, + {file = "pydantic_core-2.33.1-cp39-cp39-win_amd64.whl", hash = "sha256:5773da0ee2d17136b1f1c6fbde543398d452a6ad2a7b54ea1033e2daa739b8d2"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:5c834f54f8f4640fd7e4b193f80eb25a0602bba9e19b3cd2fc7ffe8199f5ae02"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:049e0de24cf23766f12cc5cc71d8abc07d4a9deb9061b334b62093dedc7cb068"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1a28239037b3d6f16916a4c831a5a0eadf856bdd6d2e92c10a0da3a59eadcf3e"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d3da303ab5f378a268fa7d45f37d7d85c3ec19769f28d2cc0c61826a8de21fe"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:25626fb37b3c543818c14821afe0fd3830bc327a43953bc88db924b68c5723f1"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3ab2d36e20fbfcce8f02d73c33a8a7362980cff717926bbae030b93ae46b56c7"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:2f9284e11c751b003fd4215ad92d325d92c9cb19ee6729ebd87e3250072cdcde"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:048c01eee07d37cbd066fc512b9d8b5ea88ceeb4e629ab94b3e56965ad655add"}, + {file = "pydantic_core-2.33.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5ccd429694cf26af7997595d627dd2637e7932214486f55b8a357edaac9dae8c"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:3a371dc00282c4b84246509a5ddc808e61b9864aa1eae9ecc92bb1268b82db4a"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:f59295ecc75a1788af8ba92f2e8c6eeaa5a94c22fc4d151e8d9638814f85c8fc"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:08530b8ac922003033f399128505f513e30ca770527cc8bbacf75a84fcc2c74b"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae370459da6a5466978c0eacf90690cb57ec9d533f8e63e564ef3822bfa04fe"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e3de2777e3b9f4d603112f78006f4ae0acb936e95f06da6cb1a45fbad6bdb4b5"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:3a64e81e8cba118e108d7126362ea30e021291b7805d47e4896e52c791be2761"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:52928d8c1b6bda03cc6d811e8923dffc87a2d3c8b3bfd2ce16471c7147a24850"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:1b30d92c9412beb5ac6b10a3eb7ef92ccb14e3f2a8d7732e2d739f58b3aa7544"}, + {file = "pydantic_core-2.33.1-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:f995719707e0e29f0f41a8aa3bcea6e761a36c9136104d3189eafb83f5cec5e5"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:7edbc454a29fc6aeae1e1eecba4f07b63b8d76e76a748532233c4c167b4cb9ea"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:ad05b683963f69a1d5d2c2bdab1274a31221ca737dbbceaa32bcb67359453cdd"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:df6a94bf9452c6da9b5d76ed229a5683d0306ccb91cca8e1eea883189780d568"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7965c13b3967909a09ecc91f21d09cfc4576bf78140b988904e94f130f188396"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3f1fdb790440a34f6ecf7679e1863b825cb5ffde858a9197f851168ed08371e5"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:5277aec8d879f8d05168fdd17ae811dd313b8ff894aeeaf7cd34ad28b4d77e33"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-musllinux_1_1_armv7l.whl", hash = "sha256:8ab581d3530611897d863d1a649fb0644b860286b4718db919bfd51ece41f10b"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:0483847fa9ad5e3412265c1bd72aad35235512d9ce9d27d81a56d935ef489672"}, + {file = "pydantic_core-2.33.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:de9e06abe3cc5ec6a2d5f75bc99b0bdca4f5c719a5b34026f8c57efbdecd2ee3"}, + {file = "pydantic_core-2.33.1.tar.gz", hash = "sha256:bcc9c6fdb0ced789245b02b7d6603e17d1563064ddcfc36f046b61c0c05dd9df"}, +] + +[package.dependencies] +typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" + +[[package]] +name = "pytest" +version = "8.3.5" +description = "pytest: simple powerful testing with Python" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"dev\"" +files = [ + {file = "pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820"}, + {file = "pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=1.5,<2" +tomli = {version = ">=1", markers = "python_version < \"3.11\""} + +[package.extras] +dev = ["argcomplete", "attrs (>=19.2)", "hypothesis (>=3.56)", "mock", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.14.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"dev\"" +files = [ + {file = "pytest-mock-3.14.0.tar.gz", hash = "sha256:2719255a1efeceadbc056d6bf3df3d1c5015530fb40cf347c0f9afac88410bd0"}, + {file = "pytest_mock-3.14.0-py3-none-any.whl", hash = "sha256:0b72c38033392a5f4621342fe11e9219ac11ec9d375f8e2a0c164539e0d70f6f"}, +] + +[package.dependencies] +pytest = ">=6.2.5" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +description = "Extensions to the standard Python datetime module" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "python-dotenv" +version = "1.1.0" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "python_dotenv-1.1.0-py3-none-any.whl", hash = "sha256:d7c01d9e2293916c18baf562d95698754b0dbbb5e74d457c45d4f6561fb9d55d"}, + {file = "python_dotenv-1.1.0.tar.gz", hash = "sha256:41f90bc6f5f177fb41f53e87666db362025010eb28f60a01c9143bfa33a2b2d5"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + +[[package]] +name = "pyyaml" +version = "6.0.2" +description = "YAML parser and emitter for Python" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, + {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8824b5a04a04a047e72eea5cec3bc266db09e35de6bdfe34c9436ac5ee27d237"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7c36280e6fb8385e520936c3cb3b8042851904eba0e58d277dca80a5cfed590b"}, + {file = "PyYAML-6.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec031d5d2feb36d1d1a24380e4db6d43695f3748343d99434e6f5f9156aaa2ed"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:936d68689298c36b53b29f23c6dbb74de12b4ac12ca6cfe0e047bedceea56180"}, + {file = "PyYAML-6.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:23502f431948090f597378482b4812b0caae32c22213aecf3b55325e049a6c68"}, + {file = "PyYAML-6.0.2-cp310-cp310-win32.whl", hash = "sha256:2e99c6826ffa974fe6e27cdb5ed0021786b03fc98e5ee3c5bfe1fd5015f42b99"}, + {file = "PyYAML-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:a4d3091415f010369ae4ed1fc6b79def9416358877534caf6a0fdd2146c87a3e"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:cc1c1159b3d456576af7a3e4d1ba7e6924cb39de8f67111c735f6fc832082774"}, + {file = "PyYAML-6.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1e2120ef853f59c7419231f3bf4e7021f1b936f6ebd222406c3b60212205d2ee"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d225db5a45f21e78dd9358e58a98702a0302f2659a3c6cd320564b75b86f47c"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5ac9328ec4831237bec75defaf839f7d4564be1e6b25ac710bd1a96321cc8317"}, + {file = "PyYAML-6.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ad2a3decf9aaba3d29c8f537ac4b243e36bef957511b4766cb0057d32b0be85"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4"}, + {file = "PyYAML-6.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:797b4f722ffa07cc8d62053e4cff1486fa6dc094105d13fea7b1de7d8bf71c9e"}, + {file = "PyYAML-6.0.2-cp311-cp311-win32.whl", hash = "sha256:11d8f3dd2b9c1207dcaf2ee0bbbfd5991f571186ec9cc78427ba5bd32afae4b5"}, + {file = "PyYAML-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e10ce637b18caea04431ce14fabcf5c64a1c61ec9c56b071a4b7ca131ca52d44"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:c70c95198c015b85feafc136515252a261a84561b7b1d51e3384e0655ddf25ab"}, + {file = "PyYAML-6.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ce826d6ef20b1bc864f0a68340c8b3287705cae2f8b4b1d932177dcc76721725"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f71ea527786de97d1a0cc0eacd1defc0985dcf6b3f17bb77dcfc8c34bec4dc5"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9b22676e8097e9e22e36d6b7bda33190d0d400f345f23d4065d48f4ca7ae0425"}, + {file = "PyYAML-6.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80bab7bfc629882493af4aa31a4cfa43a4c57c83813253626916b8c7ada83476"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:0833f8694549e586547b576dcfaba4a6b55b9e96098b36cdc7ebefe667dfed48"}, + {file = "PyYAML-6.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8b9c7197f7cb2738065c481a0461e50ad02f18c78cd75775628afb4d7137fb3b"}, + {file = "PyYAML-6.0.2-cp312-cp312-win32.whl", hash = "sha256:ef6107725bd54b262d6dedcc2af448a266975032bc85ef0172c5f059da6325b4"}, + {file = "PyYAML-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:7e7401d0de89a9a855c839bc697c079a4af81cf878373abd7dc625847d25cbd8"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:efdca5630322a10774e8e98e1af481aad470dd62c3170801852d752aa7a783ba"}, + {file = "PyYAML-6.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:50187695423ffe49e2deacb8cd10510bc361faac997de9efef88badc3bb9e2d1"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0ffe8360bab4910ef1b9e87fb812d8bc0a308b0d0eef8c8f44e0254ab3b07133"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:17e311b6c678207928d649faa7cb0d7b4c26a0ba73d41e99c4fff6b6c3276484"}, + {file = "PyYAML-6.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:70b189594dbe54f75ab3a1acec5f1e3faa7e8cf2f1e08d9b561cb41b845f69d5"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:41e4e3953a79407c794916fa277a82531dd93aad34e29c2a514c2c0c5fe971cc"}, + {file = "PyYAML-6.0.2-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:68ccc6023a3400877818152ad9a1033e3db8625d899c72eacb5a668902e4d652"}, + {file = "PyYAML-6.0.2-cp313-cp313-win32.whl", hash = "sha256:bc2fa7c6b47d6bc618dd7fb02ef6fdedb1090ec036abab80d4681424b84c1183"}, + {file = "PyYAML-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:8388ee1976c416731879ac16da0aff3f63b286ffdd57cdeb95f3f2e085687563"}, + {file = "PyYAML-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:24471b829b3bf607e04e88d79542a9d48bb037c2267d7927a874e6c205ca7e9a"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7fded462629cfa4b685c5416b949ebad6cec74af5e2d42905d41e257e0869f5"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d84a1718ee396f54f3a086ea0a66d8e552b2ab2017ef8b420e92edbc841c352d"}, + {file = "PyYAML-6.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9056c1ecd25795207ad294bcf39f2db3d845767be0ea6e6a34d856f006006083"}, + {file = "PyYAML-6.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:82d09873e40955485746739bcb8b4586983670466c23382c19cffecbf1fd8706"}, + {file = "PyYAML-6.0.2-cp38-cp38-win32.whl", hash = "sha256:43fa96a3ca0d6b1812e01ced1044a003533c47f6ee8aca31724f78e93ccc089a"}, + {file = "PyYAML-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:01179a4a8559ab5de078078f37e5c1a30d76bb88519906844fd7bdea1b7729ff"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:688ba32a1cffef67fd2e9398a2efebaea461578b0923624778664cc1c914db5d"}, + {file = "PyYAML-6.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:a8786accb172bd8afb8be14490a16625cbc387036876ab6ba70912730faf8e1f"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8e03406cac8513435335dbab54c0d385e4a49e4945d2909a581c83647ca0290"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12"}, + {file = "PyYAML-6.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3b1fdb9dc17f5a7677423d508ab4f243a726dea51fa5e70992e59a7411c89d19"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:0b69e4ce7a131fe56b7e4d770c67429700908fc0752af059838b1cfb41960e4e"}, + {file = "PyYAML-6.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:a9f8c2e67970f13b16084e04f134610fd1d374bf477b17ec1599185cf611d725"}, + {file = "PyYAML-6.0.2-cp39-cp39-win32.whl", hash = "sha256:6395c297d42274772abc367baaa79683958044e5d3835486c16da75d2a694631"}, + {file = "PyYAML-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:39693e1f8320ae4f43943590b49779ffb98acb81f788220ea932a6b6c51004d8"}, + {file = "pyyaml-6.0.2.tar.gz", hash = "sha256:d584d9ec91ad65861cc08d42e834324ef890a082e591037abe114850ff7bbc3e"}, +] + +[[package]] +name = "requests" +version = "2.32.3" +description = "Python HTTP for Humans." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<3" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "ruff" +version = "0.11.2" +description = "An extremely fast Python linter and code formatter, written in Rust." +optional = true +python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"dev\"" +files = [ + {file = "ruff-0.11.2-py3-none-linux_armv6l.whl", hash = "sha256:c69e20ea49e973f3afec2c06376eb56045709f0212615c1adb0eda35e8a4e477"}, + {file = "ruff-0.11.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:2c5424cc1c4eb1d8ecabe6d4f1b70470b4f24a0c0171356290b1953ad8f0e272"}, + {file = "ruff-0.11.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:ecf20854cc73f42171eedb66f006a43d0a21bfb98a2523a809931cda569552d9"}, + {file = "ruff-0.11.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0c543bf65d5d27240321604cee0633a70c6c25c9a2f2492efa9f6d4b8e4199bb"}, + {file = "ruff-0.11.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20967168cc21195db5830b9224be0e964cc9c8ecf3b5a9e3ce19876e8d3a96e3"}, + {file = "ruff-0.11.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:955a9ce63483999d9f0b8f0b4a3ad669e53484232853054cc8b9d51ab4c5de74"}, + {file = "ruff-0.11.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:86b3a27c38b8fce73bcd262b0de32e9a6801b76d52cdb3ae4c914515f0cef608"}, + {file = "ruff-0.11.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a3b66a03b248c9fcd9d64d445bafdf1589326bee6fc5c8e92d7562e58883e30f"}, + {file = "ruff-0.11.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0397c2672db015be5aa3d4dac54c69aa012429097ff219392c018e21f5085147"}, + {file = "ruff-0.11.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:869bcf3f9abf6457fbe39b5a37333aa4eecc52a3b99c98827ccc371a8e5b6f1b"}, + {file = "ruff-0.11.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:2a2b50ca35457ba785cd8c93ebbe529467594087b527a08d487cf0ee7b3087e9"}, + {file = "ruff-0.11.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:7c69c74bf53ddcfbc22e6eb2f31211df7f65054bfc1f72288fc71e5f82db3eab"}, + {file = "ruff-0.11.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:6e8fb75e14560f7cf53b15bbc55baf5ecbe373dd5f3aab96ff7aa7777edd7630"}, + {file = "ruff-0.11.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:842a472d7b4d6f5924e9297aa38149e5dcb1e628773b70e6387ae2c97a63c58f"}, + {file = "ruff-0.11.2-py3-none-win32.whl", hash = "sha256:aca01ccd0eb5eb7156b324cfaa088586f06a86d9e5314b0eb330cb48415097cc"}, + {file = "ruff-0.11.2-py3-none-win_amd64.whl", hash = "sha256:3170150172a8f994136c0c66f494edf199a0bbea7a409f649e4bc8f4d7084080"}, + {file = "ruff-0.11.2-py3-none-win_arm64.whl", hash = "sha256:52933095158ff328f4c77af3d74f0379e34fd52f175144cefc1b192e7ccd32b4"}, + {file = "ruff-0.11.2.tar.gz", hash = "sha256:ec47591497d5a1050175bdf4e1a4e6272cddff7da88a2ad595e1e326041d8d94"}, +] + +[[package]] +name = "s3transfer" +version = "0.13.0" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "s3transfer-0.13.0-py3-none-any.whl", hash = "sha256:0148ef34d6dd964d0d8cf4311b2b21c474693e57c2e069ec708ce043d2b527be"}, + {file = "s3transfer-0.13.0.tar.gz", hash = "sha256:f5e6db74eb7776a37208001113ea7aa97695368242b364d73e91c981ac522177"}, +] + +[package.dependencies] +botocore = ">=1.37.4,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] + +[[package]] +name = "six" +version = "1.17.0" +description = "Python 2 and 3 compatibility utilities" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main"] +files = [ + {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, + {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, +] + +[[package]] +name = "tomli" +version = "2.2.1" +description = "A lil' TOML parser" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"dev\" and python_version < \"3.11\"" +files = [ + {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, + {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee"}, + {file = "tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106"}, + {file = "tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8"}, + {file = "tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff"}, + {file = "tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea"}, + {file = "tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222"}, + {file = "tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd"}, + {file = "tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e"}, + {file = "tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98"}, + {file = "tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7"}, + {file = "tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281"}, + {file = "tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2"}, + {file = "tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744"}, + {file = "tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec"}, + {file = "tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69"}, + {file = "tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc"}, + {file = "tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff"}, +] + +[[package]] +name = "types-awscrt" +version = "0.27.4" +description = "Type annotations and code completion for awscrt" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "types_awscrt-0.27.4-py3-none-any.whl", hash = "sha256:a8c4b9d9ae66d616755c322aba75ab9bd793c6fef448917e6de2e8b8cdf66fb4"}, + {file = "types_awscrt-0.27.4.tar.gz", hash = "sha256:c019ba91a097e8a31d6948f6176ede1312963f41cdcacf82482ac877cbbcf390"}, +] + +[[package]] +name = "types-requests" +version = "2.32.0.20250602" +description = "Typing stubs for requests" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "types_requests-2.32.0.20250602-py3-none-any.whl", hash = "sha256:f4f335f87779b47ce10b8b8597b409130299f6971ead27fead4fe7ba6ea3e726"}, + {file = "types_requests-2.32.0.20250602.tar.gz", hash = "sha256:ee603aeefec42051195ae62ca7667cd909a2f8128fdf8aad9e8a5219ecfab3bf"}, +] + +[package.dependencies] +urllib3 = ">=2" + +[[package]] +name = "types-s3transfer" +version = "0.13.0" +description = "Type annotations and code completion for s3transfer" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "types_s3transfer-0.13.0-py3-none-any.whl", hash = "sha256:79c8375cbf48a64bff7654c02df1ec4b20d74f8c5672fc13e382f593ca5565b3"}, + {file = "types_s3transfer-0.13.0.tar.gz", hash = "sha256:203dadcb9865c2f68fb44bc0440e1dc05b79197ba4a641c0976c26c9af75ef52"}, +] + +[[package]] +name = "typing-extensions" +version = "4.14.0" +description = "Backported and Experimental Type Hints for Python 3.9+" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af"}, + {file = "typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4"}, +] + +[[package]] +name = "typing-inspection" +version = "0.4.1" +description = "Runtime typing introspection tools" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "typing_inspection-0.4.1-py3-none-any.whl", hash = "sha256:389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51"}, + {file = "typing_inspection-0.4.1.tar.gz", hash = "sha256:6ae134cc0203c33377d43188d4064e9b357dba58cff3185f22924610e70a9d28"}, +] + +[package.dependencies] +typing-extensions = ">=4.12.0" + +[[package]] +name = "urllib3" +version = "2.4.0" +description = "HTTP library with thread-safe connection pooling, file post, and more." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "urllib3-2.4.0-py3-none-any.whl", hash = "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813"}, + {file = "urllib3-2.4.0.tar.gz", hash = "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466"}, +] + +[package.extras] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] +h2 = ["h2 (>=4,<5)"] +socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] +zstd = ["zstandard (>=0.18.0)"] + +[extras] +dev = ["mypy", "pytest", "pytest-mock", "ruff"] + +[metadata] +lock-version = "2.1" +python-versions = ">=3.10" +content-hash = "da17516e468a8a4941b45258d0a72a4c1c76e20004921ce4d638073fe16bc0fe" diff --git a/gooddata-pipelines/pyproject.toml b/gooddata-pipelines/pyproject.toml new file mode 100644 index 0000000..7cda4b7 --- /dev/null +++ b/gooddata-pipelines/pyproject.toml @@ -0,0 +1,34 @@ +# (C) 2025 GoodData Corporation +[project] +name = "gooddata-pipelines" +version = "0.1.0" +description = "" +authors = [{ name = "GoodData", email = "support@gooddata.com" }] +license = { text = "BSD" } +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "pydantic (==2.11.3)", + "requests (==2.32.3)", + "types-requests (==2.32.0.20250602)", + "gooddata-sdk (==1.43.0)", + "boto3 (>=1.39.3,<2.0.0)", + "boto3-stubs (>=1.39.3,<2.0.0)", +] + +[tool.mypy] +disallow_untyped_defs = true +warn_redundant_casts = true +strict_equality = true +no_implicit_optional = true + +[tool.ruff] +exclude = [".venv"] +line-length = 80 + +[project.optional-dependencies] +dev = ["pytest==8.3.5", "pytest-mock==3.14.0", "ruff==0.11.2", "mypy>=1.16.0"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/gooddata-pipelines/tests/__init__.py b/gooddata-pipelines/tests/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/tests/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/tests/backup_and_restore/__init__.py b/gooddata-pipelines/tests/backup_and_restore/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/tests/backup_and_restore/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/tests/backup_and_restore/test_backup.py b/gooddata-pipelines/tests/backup_and_restore/test_backup.py new file mode 100644 index 0000000..d9a2b5e --- /dev/null +++ b/gooddata-pipelines/tests/backup_and_restore/test_backup.py @@ -0,0 +1,390 @@ +# (C) 2025 GoodData Corporation + +import os +import shutil +import tempfile +import threading +from pathlib import Path +from unittest import mock + +import boto3 +import pytest +from moto import mock_aws + +from gooddata_pipelines.backup_and_restore.backup_manager import ( + BackupBatch, + BackupManager, +) +from gooddata_pipelines.backup_and_restore.constants import BackupSettings +from gooddata_pipelines.backup_and_restore.models.storage import ( + BackupRestoreConfig, + S3StorageConfig, + StorageType, +) +from gooddata_pipelines.backup_and_restore.storage.local_storage import ( + LocalStorage, +) +from gooddata_pipelines.backup_and_restore.storage.s3_storage import S3Storage + +S3_BACKUP_PATH = "some/s3/backup/path/org_id/" +S3_BUCKET = "some-s3-bucket" + +LOCAL_CONFIG = BackupRestoreConfig(storage_type=StorageType.LOCAL) + +S3_CONFIG = BackupRestoreConfig( + storage_type=StorageType.S3, + storage=S3StorageConfig( + bucket=S3_BUCKET, + backup_path=S3_BACKUP_PATH, + profile="default", + ), +) + + +@pytest.fixture +def backup_manager(mock_logger): + with ( + mock.patch.object(BackupManager, "_api", create=True), + mock.patch( + "gooddata_pipelines.api.gooddata_api_wrapper.GoodDataAPI.get_organization_id", + return_value="services", + ), + ): + manager = BackupManager.create( + S3_CONFIG, + "host", + "token", + ) + manager.logger.subscribe(mock_logger) + return manager + + +@pytest.fixture() +def s3(aws_credentials): + with mock_aws(): + yield boto3.resource("s3") + + +@pytest.fixture(scope="function") +def s3_bucket(s3): + s3.create_bucket(Bucket=S3_BUCKET) + yield s3.Bucket(S3_BUCKET) + + +@pytest.fixture(scope="function") +def create_backups_in_bucket(s3_bucket): + def create_backups( + ws_ids: list[str], is_e2e: bool = False, suffix: str = "bla" + ): + # If used within e2e test, add some suffix to path + # in order to simulate a more realistic scenario + path_suffix = f"/{suffix}" if is_e2e else "" + + for ws_id in ws_ids: + s3_bucket.put_object( + Bucket=S3_BUCKET, Key=f"{S3_BACKUP_PATH}{ws_id}{path_suffix}/" + ) + s3_bucket.put_object( + Bucket=S3_BUCKET, + Key=f"{S3_BACKUP_PATH}{ws_id}{path_suffix}/gooddata_layouts.zip", + ) + + return create_backups + + +def assert_not_called_with(target, *args, **kwargs): + try: + target.assert_called_with(*args, **kwargs) + except AssertionError: + return + formatted_call = target._format_mock_call_signature(args, kwargs) + raise AssertionError(f"Expected {formatted_call} to not have been called.") + + +def test_get_s3_storage(backup_manager): + """Test get_storage method with literal string as input.""" + s3_storage = backup_manager.get_storage(S3_CONFIG) + assert isinstance(s3_storage, S3Storage) + + +def test_get_local_storage(backup_manager): + """Test get_storage method with literal string as input.""" + local_storage = backup_manager.get_storage(LOCAL_CONFIG) + assert isinstance(local_storage, LocalStorage) + + +# Test that zipping gooddata_layouts folder works +def test_archive_gooddata_layouts_to_zip(backup_manager): + with tempfile.TemporaryDirectory() as tmpdir: + shutil.copytree( + Path("tests/data/backup/test_exports/services/"), + Path(tmpdir + "/services"), + ) + backup_manager.archive_gooddata_layouts_to_zip( + str(Path(tmpdir, "services")) + ) + + zip_exists = os.path.isfile( + Path( + tmpdir, + "services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts.zip", + ) + ) + gooddata_layouts_dir_exists = os.path.isdir( + Path( + tmpdir, + "services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts", + ) + ) + + assert gooddata_layouts_dir_exists is False + assert zip_exists + + zip_exists = os.path.isfile( + Path( + tmpdir, + "services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts.zip", + ) + ) + gooddata_layouts_dir_exists = os.path.isdir( + Path( + tmpdir, + "services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts", + ) + ) + + assert gooddata_layouts_dir_exists is False + assert zip_exists + + zip_exists = os.path.isfile( + Path( + tmpdir, + "services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts.zip", + ) + ) + gooddata_layouts_dir_exists = os.path.isdir( + Path( + tmpdir, + "services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts", + ) + ) + + assert gooddata_layouts_dir_exists is False + assert zip_exists + + +def test_store_user_data_filters(backup_manager): + user_data_filters = { + "userDataFilters": [ + { + "id": "datafilter2", + "maql": '{label/campaign_channels.category} = "1"', + "title": "Status filter", + "user": { + "id": "5c867a8a-12af-45bf-8d85-c7d16bedebd1", + "type": "user", + }, + }, + { + "id": "datafilter4", + "maql": '{label/campaign_channels.category} = "1"', + "title": "Status filter", + "user": { + "id": "5c867a8a-12af-45bf-8d85-c7d16bedebd1", + "type": "user", + }, + }, + ] + } + user_data_filter_folderlocation = Path( + "tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/user_data_filters" + ) + backup_manager.store_user_data_filters( + user_data_filters, + Path( + "tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5" + ), + "wsid1", + ) + user_data_filter_folder = os.path.isdir( + Path(user_data_filter_folderlocation) + ) + user_data_filter2 = os.path.isfile( + Path(f"{user_data_filter_folderlocation}/datafilter2.yaml") + ) + user_data_filter4 = os.path.isfile( + Path(f"{user_data_filter_folderlocation}/datafilter4.yaml") + ) + assert user_data_filter_folder + assert user_data_filter2 + assert user_data_filter4 + + count = 0 + for path in os.listdir(user_data_filter_folderlocation): + if os.path.isfile(os.path.join(user_data_filter_folderlocation, path)): + count += 1 + + assert count == 2 + + shutil.rmtree( + "tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/user_data_filters" + ) + + +def test_local_storage_export(backup_manager): + with tempfile.TemporaryDirectory() as tmpdir: + org_store_location = Path(tmpdir + "/services") + shutil.copytree( + Path("tests/data/backup/test_exports/services/"), org_store_location + ) + local_storage = backup_manager.get_storage(LOCAL_CONFIG) + + local_storage.export( + folder=tmpdir, + org_id="services", + export_folder="tests/data/local_export", + ) + + local_export_folder_exist = os.path.isdir( + Path( + "tests/data/local_export/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model" + ) + ) + local_export_folder2_exist = os.path.isdir( + Path( + "tests/data/local_export/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/ldm" + ) + ) + + local_export_folder3_exist = os.path.isdir( + Path( + "tests/data/local_export/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/user_data_filters" + ) + ) + + local_export_file_exist = os.path.isfile( + Path( + "tests/data/local_export/services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid2/analytics_model/analytical_dashboards/id.yaml" + ) + ) + assert local_export_folder_exist + assert local_export_folder2_exist + assert local_export_folder3_exist + assert local_export_file_exist + shutil.rmtree("tests/data/local_export") + + +def test_file_upload(backup_manager, s3, s3_bucket, mock_boto_session): + backup_manager.storage.export("tests/data/backup/test_exports", "services") + s3.Object( + S3_BUCKET, + "some/s3/backup/path/org_id/services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid2/analytics_model/filter_contexts/id.yaml", + ).load() + + +def test_split_to_batches(backup_manager): + workspaces = ["ws1", "ws2", "ws3", "ws4", "ws5"] + batch_size = 2 + expected_batches = [ + BackupBatch(["ws1", "ws2"]), + BackupBatch(["ws3", "ws4"]), + BackupBatch(["ws5"]), + ] + + result = backup_manager.split_to_batches(workspaces, batch_size) + + for i, batch in enumerate(result): + assert isinstance(batch, BackupBatch) + assert batch.list_of_ids == expected_batches[i].list_of_ids + + +@mock.patch( + "gooddata_pipelines.backup_and_restore.backup_manager.BackupManager.get_workspace_export" +) +@mock.patch( + "gooddata_pipelines.backup_and_restore.backup_manager.BackupManager.archive_gooddata_layouts_to_zip" +) +def test_process_batch_success( + archive_gooddata_layouts_to_zip_mock, + get_workspace_export_mock, + backup_manager, +): + # Mock the storage's export method + backup_manager.storage = mock.Mock() + batch = BackupBatch(["ws1", "ws2"]) + + backup_manager.process_batch( + batch=batch, + stop_event=threading.Event(), + retry_count=0, + ) + + get_workspace_export_mock.assert_called_once() + archive_gooddata_layouts_to_zip_mock.assert_called_once() + backup_manager.storage.export.assert_called_once() + + +@mock.patch( + "gooddata_pipelines.backup_and_restore.backup_manager.BackupManager.get_workspace_export" +) +@mock.patch( + "gooddata_pipelines.backup_and_restore.backup_manager.BackupManager.archive_gooddata_layouts_to_zip" +) +def test_process_batch_retries_on_exception( + archive_gooddata_layouts_to_zip_mock, + get_workspace_export_mock, + backup_manager, + capsys, +): + backup_manager.storage = mock.Mock() + batch = BackupBatch(["ws1"]) + + # Raise exception on first call, succeed on second + call_count = {"count": 0} + + def fail_once(*args, **kwargs): + if call_count["count"] == 0: + call_count["count"] += 1 + raise Exception("fail") + return None + + get_workspace_export_mock.side_effect = fail_once + + backup_manager.process_batch( + batch=batch, + stop_event=threading.Event(), + ) + + assert get_workspace_export_mock.call_count == 2 + captured = capsys.readouterr() + assert captured.out.startswith( + "Exception encountered while processing a batch. Retrying" + ) + backup_manager.storage.export.assert_called_once() + + +@mock.patch( + "gooddata_pipelines.backup_and_restore.backup_manager.BackupManager.get_workspace_export" +) +@mock.patch( + "gooddata_pipelines.backup_and_restore.backup_manager.BackupManager.archive_gooddata_layouts_to_zip" +) +def test_process_batch_raises_after_max_retries( + archive_gooddata_layouts_to_zip_mock, + get_workspace_export_mock, + backup_manager, + capsys, +): + backup_manager.storage = mock.Mock() + batch = BackupBatch(["ws1"]) + get_workspace_export_mock.side_effect = Exception("fail") + + with pytest.raises(Exception) as exc_info: + backup_manager.process_batch( + batch=batch, + stop_event=threading.Event(), + retry_count=BackupSettings.MAX_RETRIES, + ) + assert str(exc_info.value) == "fail" + captured = capsys.readouterr() + assert captured.out.startswith("Batch failed:") diff --git a/gooddata-pipelines/tests/backup_and_restore/test_backup_input_processor.py b/gooddata-pipelines/tests/backup_and_restore/test_backup_input_processor.py new file mode 100644 index 0000000..a3b5304 --- /dev/null +++ b/gooddata-pipelines/tests/backup_and_restore/test_backup_input_processor.py @@ -0,0 +1,200 @@ +# (C) 2025 GoodData Corporation + +import os +import tempfile + +import pytest + +from gooddata_pipelines.backup_and_restore.backup_input_processor import ( + BackupInputProcessor, +) +from gooddata_pipelines.backup_and_restore.models.input_type import InputType +from gooddata_pipelines.backup_and_restore.models.workspace_response import ( + Hierarchy, + Links, + Meta, + Page, + Workspace, + WorkspaceResponse, +) + + +@pytest.fixture +def backup_input_processor(mock_gooddata_api, mock_logger): + processor = BackupInputProcessor(mock_gooddata_api, page_size=2) + processor.hierarchy_endpoint = ( + "/fake/hierarchy?filter=parent.id=={parent_id}" + ) + processor.all_workspaces_endpoint = "/fake/all" + processor.logger.subscribe(mock_logger) + return processor + + +def test_process_data_extracts_children_and_subparents(backup_input_processor): + ws1 = Workspace(id="ws1", meta=Meta(hierarchy=Hierarchy(children_count=2))) + ws2 = Workspace(id="ws2", meta=Meta(hierarchy=Hierarchy(children_count=0))) + ws3 = Workspace(id="ws3", meta=None) + + result = backup_input_processor.process_data([ws1, ws2, ws3]) + assert result.workspace_ids == ["ws1", "ws2", "ws3"] + assert result.sub_parents == ["ws1"] + + +def test_log_paging_progress_logs_info(backup_input_processor, capsys): + response = WorkspaceResponse( + data=[], + meta=Meta( + page=Page(size=5, total_elements=25, number=1, total_pages=5), + hierarchy=None, + ), + links=Links(self="self", next="next"), + ) + + backup_input_processor.log_paging_progress(response) + captured = capsys.readouterr() + assert "Fetched page: 2 of 5" in captured.out + + +def test_log_paging_progress_no_page(backup_input_processor, capsys): + response = WorkspaceResponse( + data=[], + meta=Meta(page=None, hierarchy=None), + links=Links(self="self", next="next"), + ) + + backup_input_processor.log_paging_progress(response) + captured = capsys.readouterr() + assert captured.out == "" + + +def test_paginate_calls_fetch_page_and_process_data( + backup_input_processor, monkeypatch +): + ws1 = Workspace(id="ws1", meta=Meta(hierarchy=Hierarchy(children_count=1))) + ws2 = Workspace(id="ws2", meta=Meta(hierarchy=Hierarchy(children_count=0))) + links1 = Links(self="self", next="next_url") + links2 = Links(self="self", next=None) + resp1 = WorkspaceResponse( + data=[ws1], meta=Meta(hierarchy=None, page=None), links=links1 + ) + resp2 = WorkspaceResponse( + data=[ws2], meta=Meta(hierarchy=None, page=None), links=links2 + ) + + fetch_page_calls = [] + + def fetch_page_side_effect(url): + fetch_page_calls.append(url) + return resp1 if len(fetch_page_calls) == 1 else resp2 + + backup_input_processor.fetch_page = fetch_page_side_effect + + process_data_calls = [] + + def process_data_side_effect(data): + process_data_calls.append(data) + if len(process_data_calls) == 1: + return BackupInputProcessor._ProcessDataOutput(["ws1"], ["ws1"]) + else: + return BackupInputProcessor._ProcessDataOutput(["ws2"], []) + + monkeypatch.setattr( + BackupInputProcessor, + "process_data", + staticmethod(process_data_side_effect), + ) + monkeypatch.setattr( + BackupInputProcessor, + "log_paging_progress", + staticmethod(lambda resp: None), + ) + + result = backup_input_processor._paginate("first_url") + assert len(result) == 2 + assert result[0].workspace_ids == ["ws1"] + assert result[1].workspace_ids == ["ws2"] + assert len(fetch_page_calls) == 2 + assert len(process_data_calls) == 2 + + +def test_get_hierarchy_recurses(backup_input_processor): + def fake_paginate(url): + if "p1" in url: + return [BackupInputProcessor._ProcessDataOutput(["c1"], ["c1"])] + if "c1" in url: + return [BackupInputProcessor._ProcessDataOutput(["c2"], [])] + return [] + + backup_input_processor._paginate = fake_paginate + result = backup_input_processor.get_hierarchy("p1") + assert set(result) == {"c1", "c2"} + + +def test_get_workspaces_to_backup_empty_org( + backup_input_processor, monkeypatch, capsys +): + """Test that the function returns an empty list if the organization contains no workspaces.""" + monkeypatch.setattr( + backup_input_processor, + "_paginate", + lambda _: [], + ) + backup_input_processor.get_ids_to_backup( + InputType.ORGANIZATION, + "some-csv-file.csv", + ) + captured = capsys.readouterr() + assert "No workspaces found in the organization." in captured.out + + +def test_read_csv_input_empty_file(backup_input_processor) -> None: + """Test with an empty CSV file.""" + with tempfile.NamedTemporaryFile() as temp_csv: + path_to_csv = temp_csv.name + with pytest.raises( + ValueError, match="No content found in the CSV file." + ): + backup_input_processor.csv_reader.read_backup_csv(path_to_csv) + + +def test_read_csv_input_only_header(backup_input_processor) -> None: + """Test with a CSV file that contains only the header.""" + with tempfile.NamedTemporaryFile() as temp_csv: + temp_csv.write(b"header1\n") + temp_csv.flush() + temp_csv.seek(0) + path_to_csv = temp_csv.name + with pytest.raises( + ValueError, match="No workspaces found in the CSV file." + ): + backup_input_processor.csv_reader.read_backup_csv(path_to_csv) + + +def test_read_csv_input_valid(backup_input_processor) -> None: + """Test with a valid CSV file.""" + with tempfile.NamedTemporaryFile(delete=False) as temp_csv: + temp_csv.write(b"header1\n") + temp_csv.write(b"workspace1\n") + temp_csv.write(b"workspace2\n") + temp_csv.flush() + temp_csv.seek(0) + path_to_csv = temp_csv.name + result = backup_input_processor.csv_reader.read_backup_csv(path_to_csv) + assert result == ["workspace1", "workspace2"] + os.remove(path_to_csv) + + +def test_read_csv_input_too_many_columns(backup_input_processor) -> None: + """Test with a CSV file that contains too many columns.""" + with tempfile.NamedTemporaryFile(delete=False) as temp_csv: + temp_csv.write(b"header1,header2\n") + temp_csv.write(b"workspace1,extra_column\n") + temp_csv.flush() + temp_csv.seek(0) + path_to_csv = temp_csv.name + with pytest.raises( + ValueError, + match="Input file contains more than one column. Please check the input and try again.", + ): + backup_input_processor.csv_reader.read_backup_csv(path_to_csv) + os.remove(path_to_csv) diff --git a/gooddata-pipelines/tests/conftest.py b/gooddata-pipelines/tests/conftest.py new file mode 100644 index 0000000..712cae1 --- /dev/null +++ b/gooddata-pipelines/tests/conftest.py @@ -0,0 +1,55 @@ +# (C) 2025 GoodData Corporation +import os +from typing import Generator + +import boto3 +import pytest + +from gooddata_pipelines.api import GoodDataAPI + + +@pytest.fixture(scope="session", autouse=True) +def aws_credentials() -> Generator[None, None, None]: + """ + Set dummy AWS credentials for the entire test session. + This is an autouse fixture, so it runs automatically. + """ + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "us-east-1" + yield + + +@pytest.fixture +def mock_boto_session(mocker): + """ + Mocks boto3.Session to prevent it from using a real AWS profile. + It will return a default Session object, which then uses the + dummy credentials set by the conftest.py fixture. + """ + # We patch boto3.Session and make it return a new, default session object. + # This new object will not have the `profile_name` and will fall back + # to using the environment variables we set in conftest. + mocker.patch("boto3.Session", return_value=boto3.Session()) + + +@pytest.fixture +def mock_gooddata_api(): + return GoodDataAPI("domain", "token") + + +@pytest.fixture +def mock_logger(): + class MockLogger: + def info(self, msg: str) -> None: + print(msg) + + def warning(self, msg: str) -> None: + print(msg) + + def error(self, msg: str) -> None: + print(msg) + + return MockLogger() diff --git a/gooddata-pipelines/tests/data/__init__.py b/gooddata-pipelines/tests/data/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/tests/data/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/tests/data/backup/__init__.py b/gooddata-pipelines/tests/data/backup/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/tests/data/backup/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model/analytical_dashboard_extensions/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model/analytical_dashboard_extensions/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model/analytical_dashboards/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model/analytical_dashboards/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model/dashboard_plugins/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model/dashboard_plugins/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model/filter_contexts/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model/filter_contexts/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model/metrics/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model/metrics/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model/visualization_objects/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid1/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid1/analytics_model/visualization_objects/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid2/analytics_model/analytical_dashboard_extensions/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid2/analytics_model/analytical_dashboard_extensions/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid2/analytics_model/dashboard_plugins/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid2/analytics_model/dashboard_plugins/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid2/analytics_model/metrics/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid2/analytics_model/metrics/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid2/ldm/datasets/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid2/ldm/datasets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid2/ldm/date_instances/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid2/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid2/ldm/date_instances/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/analytics_model/analytical_dashboard_extensions/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/analytics_model/analytical_dashboard_extensions/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/analytics_model/analytical_dashboards/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/analytics_model/analytical_dashboards/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/analytics_model/dashboard_plugins/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/analytics_model/dashboard_plugins/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/analytics_model/filter_contexts/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/analytics_model/filter_contexts/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/analytics_model/metrics/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/analytics_model/metrics/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/analytics_model/visualization_objects/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/analytics_model/visualization_objects/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/ldm/datasets/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/ldm/datasets/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/ldm/date_instances/.gitkeep b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/ldm/date_instances/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/user_data_filters/.gitignore b/gooddata-pipelines/tests/data/backup/test_exports/services/wsid3/20230713-132759-1_3_1_dev5/gooddata_layouts/services/workspaces/wsid3/user_data_filters/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/gooddata-pipelines/tests/data/mock_responses.py b/gooddata-pipelines/tests/data/mock_responses.py new file mode 100644 index 0000000..0630d1d --- /dev/null +++ b/gooddata-pipelines/tests/data/mock_responses.py @@ -0,0 +1,104 @@ +# (C) 2025 GoodData Corporation + +from typing import Any + +WDF_VALID_PAYLOAD = { + "data": [ + { + "id": "expected_wdf_setting_id", + "type": "workspaceDataFilterSetting", + "attributes": { + "title": "", + "filterValues": ["expected", "wdf", "values"], + }, + "relationships": { + "workspaceDataFilter": { + "data": { + "id": "expected_wdf_id", + "type": "workspaceDataFilter", + }, + }, + }, + "links": {"self": "https://some.uri.com"}, + "meta": { + "origin": { + "originType": "origin_type", + "originId": "origin_id", + } + }, + } + ] +} + +WDF_ACTUAL_WDF_SETTINGS: list[dict[str, Any]] = [ + { + "id": "expected_wdf_setting_id", + "type": "workspaceDataFilterSetting", + "attributes": { + "title": "", + "filterValues": ["expected", "wdf", "values"], + }, + "relationships": { + "workspaceDataFilter": { + "data": { + "id": "expected_wdf_id", + "type": "workspaceDataFilter", + }, + }, + }, + "links": {"self": "https://some.uri.com"}, + "meta": { + "origin": { + "originType": "origin_type", + "originId": "workspace_id", + } + }, + }, + { + "id": "expected_wdf_setting_id_2", + "type": "workspaceDataFilterSetting", + "attributes": { + "title": "", + "filterValues": ["expected", "wdf", "values"], + }, + "relationships": { + "workspaceDataFilter": { + "data": { + "id": "expected_wdf_id_2", + "type": "workspaceDataFilter", + }, + }, + }, + "links": {"self": "https://some.uri.com"}, + "meta": { + "origin": { + "originType": "origin_type", + "originId": "workspace_id", + } + }, + }, + # expected_wdf_id_3 is missing + { + "id": "expected_wdf_setting_id_4", + "type": "workspaceDataFilterSetting", + "attributes": { + "title": "", + "filterValues": ["expected", "wdf", "values"], + }, + "relationships": { + "workspaceDataFilter": { + "data": { + "id": "expected_wdf_id_4", + "type": "workspaceDataFilter", + }, + }, + }, + "links": {"self": "https://some.uri.com"}, + "meta": { + "origin": { + "originType": "origin_type", + "originId": "workspace_id", + } + }, + }, +] diff --git a/gooddata-pipelines/tests/panther/__init__.py b/gooddata-pipelines/tests/panther/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/tests/panther/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/tests/panther/test_api_wrapper.py b/gooddata-pipelines/tests/panther/test_api_wrapper.py new file mode 100644 index 0000000..40832f5 --- /dev/null +++ b/gooddata-pipelines/tests/panther/test_api_wrapper.py @@ -0,0 +1,37 @@ +# (C) 2025 GoodData Corporation + +import pytest + +from gooddata_pipelines.api.gooddata_api import ( + API_VERSION, + APIMethods, +) + + +@pytest.mark.parametrize( + "domain, expected_base_url", + [ + ("example.com", f"https://example.com/api/{API_VERSION}"), + ( + "https://example.com", + f"https://example.com/api/{API_VERSION}", + ), + ( + "http://example.com", + f"https://example.com/api/{API_VERSION}", + ), + ("example.com/", f"https://example.com/api/{API_VERSION}"), + ( + "https://example.com/", + f"https://example.com/api/{API_VERSION}", + ), + ( + "http://example.com/", + f"https://example.com/api/{API_VERSION}", + ), + ], +) +def test_get_base_url(domain, expected_base_url): + """Test the get_base_url method with various domain inputs.""" + result = APIMethods._get_base_url(domain) + assert result == expected_base_url diff --git a/gooddata-pipelines/tests/panther/test_sdk_wrapper.py b/gooddata-pipelines/tests/panther/test_sdk_wrapper.py new file mode 100644 index 0000000..2187d34 --- /dev/null +++ b/gooddata-pipelines/tests/panther/test_sdk_wrapper.py @@ -0,0 +1,130 @@ +# (C) 2025 GoodData Corporation + +import pytest +from gooddata_api_client import ApiException # type: ignore[import] +from gooddata_sdk.catalog.workspace.entity_model.workspace import ( + CatalogWorkspace, +) + +from gooddata_pipelines.api.exceptions import GoodDataApiException +from gooddata_pipelines.api.utils import raise_with_context + +GOODDATA_WRAPPER_OBJECT_PATH = ( + "gooddata_pipelines.api.gooddata_api_wrapper.GoodDataAPI.list_workspaces" +) + + +def test_raise_with_context_reraises_panther_exception(): + @raise_with_context() + def func(): + raise GoodDataApiException("Caught a GoodDataApiException") + + with pytest.raises(GoodDataApiException) as exc: + func() + assert "GoodDataApiException: Caught a GoodDataApiException" in str( + exc.value + ) + + +def test_raise_with_context_wraps_generic_exception(): + @raise_with_context(api_endpoint="some.function.name") + def func(): + raise ValueError("fail") + + with pytest.raises(GoodDataApiException) as exc: + func() + assert "ValueError: Some error: fail" in str(exc.value) + assert exc.value.api_endpoint == "some.function.name" + + +def test_raise_with_context_wraps_apiexception_and_sets_error_template(): + @raise_with_context(api_endpoint="endpoint") + def func(): + raise ApiException(404, "Not Found") + + with pytest.raises(GoodDataApiException) as exc: + func() + assert "ApiException: 404 Not Found" in str(exc.value) + assert exc.value.api_endpoint == "endpoint" + assert exc.value.http_status == "404 Not Found" + + +def test_raise_with_context_passes_method_kwargs(): + @raise_with_context() + def func(**kwargs): + raise RuntimeError("fail") + + with pytest.raises(GoodDataApiException) as exc: + func(http_method="bar") + assert exc.value.http_method == "bar" + + +def test_get_panther_children_workspaces_empty_response( + mock_gooddata_api, mocker +) -> None: + parent_ids: set[str] = {"parent_id_1", "parent_id_2"} + + mocker.patch( + GOODDATA_WRAPPER_OBJECT_PATH, + return_value=[], + ) + + panther_children = mock_gooddata_api.get_panther_children_workspaces( + parent_ids + ) + + assert panther_children == [] + + +def test_get_panther_children_full_match(mock_gooddata_api, mocker) -> None: + parent_ids: set[str] = {"parent_id_1", "parent_id_2"} + + mocker.patch( + GOODDATA_WRAPPER_OBJECT_PATH, + return_value=[ + CatalogWorkspace( + workspace_id="workspace_id1", + name="workspace_title1", + parent_id="parent_id_1", + ), + CatalogWorkspace( + workspace_id="workspace_id2", + name="workspace_title2", + parent_id="parent_id_2", + ), + ], + ) + + panther_children = mock_gooddata_api.get_panther_children_workspaces( + parent_ids + ) + + assert len(panther_children) == 2 + assert panther_children[0].workspace_id == "workspace_id1" + assert panther_children[1].workspace_id == "workspace_id2" + + +def test_get_panther_children_no_match(mock_gooddata_api, mocker) -> None: + parent_ids: set[str] = {"parent_id_3", "parent_id_4"} + + mocker.patch( + GOODDATA_WRAPPER_OBJECT_PATH, + return_value=[ + CatalogWorkspace( + workspace_id="workspace_id1", + name="workspace_title1", + parent_id="parent_id_1", + ), + CatalogWorkspace( + workspace_id="workspace_id2", + name="workspace_title2", + parent_id="parent_id_2", + ), + ], + ) + + panther_children = mock_gooddata_api.get_panther_children_workspaces( + parent_ids + ) + + assert len(panther_children) == 0 diff --git a/gooddata-pipelines/tests/provisioning/__init__.py b/gooddata-pipelines/tests/provisioning/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/tests/provisioning/entities/__init__.py b/gooddata-pipelines/tests/provisioning/entities/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/entities/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/tests/provisioning/entities/users/__init__.py b/gooddata-pipelines/tests/provisioning/entities/users/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/entities/users/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/tests/provisioning/entities/users/test_permissions.py b/gooddata-pipelines/tests/provisioning/entities/users/test_permissions.py new file mode 100644 index 0000000..0752a09 --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/entities/users/test_permissions.py @@ -0,0 +1,389 @@ +# (C) 2025 GoodData Corporation + +from gooddata_api_client.exceptions import ( # type: ignore[import] + NotFoundException, +) +from gooddata_sdk.catalog.identifier import CatalogAssigneeIdentifier +from gooddata_sdk.catalog.permission.declarative_model.permission import ( + CatalogDeclarativeSingleWorkspacePermission, + CatalogDeclarativeWorkspacePermissions, +) + +from gooddata_pipelines.provisioning.entities.users.models.permissions import ( + PermissionDeclaration, + PermissionIncrementalLoad, + PermissionType, +) + +TEST_CSV_PATH = "tests/data/permission_mgmt/input.csv" + +USER_1 = CatalogAssigneeIdentifier(id="user_1", type="user") +USER_2 = CatalogAssigneeIdentifier(id="user_2", type="user") +USER_3 = CatalogAssigneeIdentifier(id="user_3", type="user") +UG_1 = CatalogAssigneeIdentifier(id="ug_1", type="userGroup") +UG_2 = CatalogAssigneeIdentifier(id="ug_2", type="userGroup") +UG_3 = CatalogAssigneeIdentifier(id="ug_3", type="userGroup") + +UPSTREAM_PERMISSIONS = [ + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=USER_1 + ), + CatalogDeclarativeSingleWorkspacePermission(name="VIEW", assignee=USER_1), + CatalogDeclarativeSingleWorkspacePermission(name="MANAGE", assignee=USER_1), + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=USER_2 + ), + CatalogDeclarativeSingleWorkspacePermission(name="VIEW", assignee=USER_2), + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=USER_3 + ), + CatalogDeclarativeSingleWorkspacePermission(name="ANALYZE", assignee=UG_1), + CatalogDeclarativeSingleWorkspacePermission(name="VIEW", assignee=UG_1), + CatalogDeclarativeSingleWorkspacePermission(name="MANAGE", assignee=UG_1), + CatalogDeclarativeSingleWorkspacePermission(name="ANALYZE", assignee=UG_2), + CatalogDeclarativeSingleWorkspacePermission(name="VIEW", assignee=UG_2), + CatalogDeclarativeSingleWorkspacePermission(name="ANALYZE", assignee=UG_3), +] + +WS_PERMISSION_DECLARATION = PermissionDeclaration( + users={ + "user_1": {"ANALYZE": True, "VIEW": True, "MANAGE": True}, + "user_2": {"ANALYZE": True, "VIEW": True}, + "user_3": {"ANALYZE": True}, + }, + user_groups={ + "ug_1": {"ANALYZE": True, "VIEW": True, "MANAGE": True}, + "ug_2": {"ANALYZE": True, "VIEW": True}, + "ug_3": {"ANALYZE": True}, + }, +) + +UPSTREAM_WS_PERMISSION = CatalogDeclarativeWorkspacePermissions( + permissions=UPSTREAM_PERMISSIONS +) + +UPSTREAM_WS_PERMISSIONS = { + "ws_id_1": UPSTREAM_WS_PERMISSION, + "ws_id_2": UPSTREAM_WS_PERMISSION, +} + +EXPECTED_WS1_PERMISSIONS = CatalogDeclarativeWorkspacePermissions( + permissions=[ + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=USER_1 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="VIEW", assignee=USER_1 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=USER_2 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="MANAGE", assignee=USER_2 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=USER_3 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=UG_1 + ), + CatalogDeclarativeSingleWorkspacePermission(name="VIEW", assignee=UG_1), + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=UG_2 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="MANAGE", assignee=UG_2 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=UG_3 + ), + ] +) + +EXPECTED_WS2_PERMISSIONS = CatalogDeclarativeWorkspacePermissions( + permissions=[ + CatalogDeclarativeSingleWorkspacePermission( + name="MANAGE", assignee=USER_1 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="MANAGE", assignee=USER_3 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="MANAGE", assignee=UG_1 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="MANAGE", assignee=UG_3 + ), + ] +) + + +def test_declaration_from_populated_sdk_api_obj(): + declaration = PermissionDeclaration.from_sdk_api(UPSTREAM_WS_PERMISSION) + assert declaration == WS_PERMISSION_DECLARATION + + +def test_declaration_from_empty_sdk_api_obj(): + api_obj = CatalogDeclarativeWorkspacePermissions(permissions=[]) + declaration = PermissionDeclaration.from_sdk_api(api_obj) + assert len(declaration.users) == 0 + assert len(declaration.user_groups) == 0 + + +def test_declaration_to_populated_sdk_api_obj(): + api_obj = PermissionDeclaration.to_sdk_api(WS_PERMISSION_DECLARATION) + assert api_obj == UPSTREAM_WS_PERMISSION + + +def test_declaration_with_inactive_to_sdk_api_obj(): + users = { + "user_1": {"ANALYZE": True, "VIEW": False}, + "user_2": {"ANALYZE": True}, + } + ugs = { + "ug_1": {"ANALYZE": True, "VIEW": False}, + "ug_2": {"ANALYZE": True}, + } + declaration = PermissionDeclaration(users, ugs) + api_obj = declaration.to_sdk_api() + expected = CatalogDeclarativeWorkspacePermissions( + permissions=[ + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=USER_1 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=USER_2 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=UG_1 + ), + CatalogDeclarativeSingleWorkspacePermission( + name="ANALYZE", assignee=UG_2 + ), + ] + ) + assert api_obj == expected + + +def test_declaration_with_only_inactive_to_sdk_api_obj(): + users = { + "user_1": {"ANALYZE": False, "VIEW": False}, + "user_2": {"ANALYZE": False}, + } + ugs = { + "ug_1": {"ANALYZE": False, "VIEW": False}, + "ug_2": {"ANALYZE": False}, + } + declaration = PermissionDeclaration(users, ugs) + api_obj = declaration.to_sdk_api() + expected = CatalogDeclarativeWorkspacePermissions(permissions=[]) + assert api_obj == expected + + +# Declarations are explicitly defined anew here to avoid dict mutations +# in subsequent calls and to avoid dict deepcopy overhead. + + +def test_add_new_active_user_perm(): + declaration = PermissionDeclaration( + {"user_1": {"ANALYZE": True, "VIEW": False}}, + {"ug_1": {"VIEW": True, "ANALYZE": False}}, + ) + permission = PermissionIncrementalLoad( + "MANAGE", "", "user_1", PermissionType.user, True + ) + declaration.add_permission(permission) + assert declaration.users == { + "user_1": {"ANALYZE": True, "VIEW": False, "MANAGE": True} + } + assert declaration.user_groups == {"ug_1": {"VIEW": True, "ANALYZE": False}} + + +def test_add_new_inactive_user_perm(): + declaration = PermissionDeclaration( + {"user_1": {"ANALYZE": True, "VIEW": False}}, + {"ug_1": {"VIEW": True, "ANALYZE": False}}, + ) + permission = PermissionIncrementalLoad( + "MANAGE", "", "user_1", PermissionType.user, False + ) + declaration.add_permission(permission) + assert declaration.users == { + "user_1": {"ANALYZE": True, "VIEW": False, "MANAGE": False} + } + assert declaration.user_groups == {"ug_1": {"VIEW": True, "ANALYZE": False}} + + +def test_overwrite_inactive_user_perm(): + declaration = PermissionDeclaration( + {"user_1": {"ANALYZE": True, "VIEW": False}}, + {"ug_1": {"VIEW": True, "ANALYZE": False}}, + ) + permission = PermissionIncrementalLoad( + "VIEW", "", "user_1", PermissionType.user, True + ) + declaration.add_permission(permission) + assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": True}} + assert declaration.user_groups == {"ug_1": {"VIEW": True, "ANALYZE": False}} + + +def test_overwrite_active_user_perm(): + declaration = PermissionDeclaration( + {"user_1": {"ANALYZE": True, "VIEW": False}}, + {"ug_1": {"VIEW": True, "ANALYZE": False}}, + ) + permission = PermissionIncrementalLoad( + "ANALYZE", "", "user_1", PermissionType.user, False + ) + declaration.add_permission(permission) + assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} + assert declaration.user_groups == {"ug_1": {"VIEW": True, "ANALYZE": False}} + + +def test_add_new_user_perm(): + declaration = PermissionDeclaration( + {"user_1": {"ANALYZE": True, "VIEW": False}}, + {"ug_1": {"VIEW": True, "ANALYZE": False}}, + ) + permission = PermissionIncrementalLoad( + "VIEW", "", "user_2", PermissionType.user, True + ) + declaration.add_permission(permission) + assert declaration.users == { + "user_1": {"ANALYZE": True, "VIEW": False}, + "user_2": {"VIEW": True}, + } + assert declaration.user_groups == {"ug_1": {"VIEW": True, "ANALYZE": False}} + + +def test_modify_one_of_user_perms(): + declaration = PermissionDeclaration( + {"user_1": {"ANALYZE": True, "VIEW": False}, "user_2": {"VIEW": True}}, + {"ug_1": {"VIEW": True, "ANALYZE": False}}, + ) + permission = PermissionIncrementalLoad( + "MANAGE", "", "user_1", PermissionType.user, True + ) + declaration.add_permission(permission) + assert declaration.users == { + "user_1": {"ANALYZE": True, "VIEW": False, "MANAGE": True}, + "user_2": {"VIEW": True}, + } + assert declaration.user_groups == {"ug_1": {"VIEW": True, "ANALYZE": False}} + + +# Add userGroup permission + + +def test_add_new_active_ug_perm(): + declaration = PermissionDeclaration( + {"user_1": {"ANALYZE": True, "VIEW": False}}, + {"ug_1": {"VIEW": True, "ANALYZE": False}}, + ) + permission = PermissionIncrementalLoad( + "MANAGE", "", "ug_1", PermissionType.user_group, True + ) + declaration.add_permission(permission) + assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} + assert declaration.user_groups == { + "ug_1": {"VIEW": True, "ANALYZE": False, "MANAGE": True} + } + + +def test_add_new_inactive_ug_perm(): + declaration = PermissionDeclaration( + {"user_1": {"ANALYZE": True, "VIEW": False}}, + {"ug_1": {"VIEW": True, "ANALYZE": False}}, + ) + permission = PermissionIncrementalLoad( + "MANAGE", "", "ug_1", PermissionType.user_group, False + ) + declaration.add_permission(permission) + assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} + assert declaration.user_groups == { + "ug_1": {"VIEW": True, "ANALYZE": False, "MANAGE": False} + } + + +def test_overwrite_inactive_ug_perm(): + declaration = PermissionDeclaration( + {"user_1": {"ANALYZE": True, "VIEW": False}}, + {"ug_1": {"VIEW": True, "ANALYZE": False}}, + ) + permission = PermissionIncrementalLoad( + "ANALYZE", "", "ug_1", PermissionType.user_group, True + ) + declaration.add_permission(permission) + assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} + assert declaration.user_groups == {"ug_1": {"VIEW": True, "ANALYZE": True}} + + +def test_overwrite_active_ug_perm(): + declaration = PermissionDeclaration( + {"user_1": {"ANALYZE": True, "VIEW": False}}, + {"ug_1": {"VIEW": True, "ANALYZE": False}}, + ) + permission = PermissionIncrementalLoad( + "VIEW", "", "ug_1", PermissionType.user_group, False + ) + declaration.add_permission(permission) + assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} + assert declaration.user_groups == {"ug_1": {"VIEW": True, "ANALYZE": False}} + + +def test_add_new_ug_perm(): + declaration = PermissionDeclaration( + {"user_1": {"ANALYZE": True, "VIEW": False}}, + {"ug_1": {"VIEW": True, "ANALYZE": False}}, + ) + permission = PermissionIncrementalLoad( + "VIEW", "", "ug_2", PermissionType.user_group, True + ) + declaration.add_permission(permission) + assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} + assert declaration.user_groups == { + "ug_1": {"VIEW": True, "ANALYZE": False}, + "ug_2": {"VIEW": True}, + } + + +def test_modify_one_of_ug_perms(): + declaration = PermissionDeclaration( + {"user_1": {"ANALYZE": True, "VIEW": False}}, + {"ug_1": {"VIEW": True, "ANALYZE": False}, "ug_2": {"VIEW": True}}, + ) + permission = PermissionIncrementalLoad( + "MANAGE", "", "ug_1", PermissionType.user_group, True + ) + declaration.add_permission(permission) + assert declaration.users == {"user_1": {"ANALYZE": True, "VIEW": False}} + assert declaration.user_groups == { + "ug_1": {"VIEW": True, "ANALYZE": False, "MANAGE": True}, + "ug_2": {"VIEW": True}, + } + + +def test_upsert(): + owner = PermissionDeclaration( + {"user_1": {"ANALYZE": True}, "user_2": {"VIEW": True}}, + {"ug_1": {"ANALYZE": True}, "ug_2": {"VIEW": True}}, + ) + other = PermissionDeclaration( + {"user_1": {"MANAGE": True, "VIEW": False}}, + {"ug_2": {"MANAGE": True, "VIEW": False}}, + ) + owner.upsert(other) + assert owner.users == { + "user_1": {"MANAGE": True, "VIEW": False}, + "user_2": {"VIEW": True}, + } + assert owner.user_groups == { + "ug_1": {"ANALYZE": True}, + "ug_2": {"MANAGE": True, "VIEW": False}, + } + + +def mock_upstream_perms(ws_id: str) -> CatalogDeclarativeWorkspacePermissions: + if ws_id not in UPSTREAM_WS_PERMISSIONS: + raise NotFoundException(404) + return UPSTREAM_WS_PERMISSIONS[ws_id] diff --git a/gooddata-pipelines/tests/provisioning/entities/users/test_user_groups.py b/gooddata-pipelines/tests/provisioning/entities/users/test_user_groups.py new file mode 100644 index 0000000..d23d8f1 --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/entities/users/test_user_groups.py @@ -0,0 +1,119 @@ +# (C) 2025 GoodData Corporation + +from dataclasses import dataclass +from unittest import mock + +import pytest +from gooddata_sdk.catalog.user.entity_model.user_group import CatalogUserGroup + +from gooddata_pipelines.provisioning.entities.users.models.user_groups import ( + UserGroupIncrementalLoad, +) + +TEST_CSV_PATH = "tests/data/user_group_mgmt/input.csv" + + +@dataclass +class MockUserGroup: + id: str + name: str + parent_ids: list[str] + + def to_sdk(self): + return CatalogUserGroup.init( + user_group_id=self.id, + user_group_name=self.name, + user_group_parent_ids=self.parent_ids, + ) + + +def test_from_csv_row_standard(): + input = [ + { + "user_group_id": "ug_1", + "user_group_name": "Admins", + "parent_user_groups": "ug_2|ug_3", + "is_active": "True", + } + ] + result = UserGroupIncrementalLoad.from_list_of_dicts(input, "|") + expected = [ + UserGroupIncrementalLoad( + user_group_id="ug_1", + user_group_name="Admins", + parent_user_groups=["ug_2", "ug_3"], + is_active=True, + ) + ] + assert result == expected, "Standard row should be parsed correctly" + + +def test_from_csv_row_no_parent_groups(): + input = [ + { + "user_group_id": "ug_2", + "user_group_name": "Developers", + "parent_user_groups": "", + "is_active": "True", + } + ] + result = UserGroupIncrementalLoad.from_list_of_dicts(input, "|") + expected = [ + UserGroupIncrementalLoad( + user_group_id="ug_2", + user_group_name="Developers", + parent_user_groups=[], + is_active=True, + ) + ] + assert result == expected, ( + "Row without parent user groups should be parsed correctly" + ) + + +def test_from_csv_row_fallback_name(): + input = [ + { + "user_group_id": "ug_3", + "user_group_name": "", + "parent_user_groups": "", + "is_active": "False", + } + ] + result = UserGroupIncrementalLoad.from_list_of_dicts(input, "|") + expected = [ + UserGroupIncrementalLoad( + user_group_id="ug_3", + user_group_name="ug_3", + parent_user_groups=[], + is_active=False, + ) + ] + assert result == expected, ( + "Row with empty name should fallback to user group ID" + ) + + +def test_from_csv_row_invalid_is_active(): + input = [ + { + "user_group_id": "ug_4", + "user_group_name": "Testers", + "parent_user_groups": "ug_1", + "is_active": "not_a_boolean", + } + ] + with pytest.raises(ValueError): + UserGroupIncrementalLoad.from_list_of_dicts(input, "|") + + +def prepare_sdk(): + def mock_list_user_groups(): + return [ + MockUserGroup("ug_1", "Admins", []).to_sdk(), + MockUserGroup("ug_4", "TemporaryAccess", ["ug_2"]).to_sdk(), + ] + + sdk = mock.Mock() + sdk.catalog_user.list_user_groups = mock_list_user_groups + return sdk diff --git a/gooddata-pipelines/tests/provisioning/entities/users/test_users.py b/gooddata-pipelines/tests/provisioning/entities/users/test_users.py new file mode 100644 index 0000000..003bfd5 --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/entities/users/test_users.py @@ -0,0 +1,203 @@ +# (C) 2025 GoodData Corporation + +from dataclasses import dataclass +from typing import Any, Optional +from unittest import mock + +from gooddata_api_client.exceptions import ( # type: ignore[import] + NotFoundException, +) +from gooddata_sdk.catalog.user.entity_model.user import CatalogUser +from gooddata_sdk.catalog.user.entity_model.user_group import CatalogUserGroup + +from gooddata_pipelines.provisioning.entities.users.models.users import ( + UserIncrementalLoad, +) + + +@dataclass +class MockUser: + id: str + firstname: Optional[str] + lastname: Optional[str] + email: Optional[str] + authenticationId: Optional[str] + user_groups: list[str] + + def to_sdk(self): + return CatalogUser.init( + user_id=self.id, + firstname=self.firstname, + lastname=self.lastname, + email=self.email, + authentication_id=self.authenticationId, + user_group_ids=self.user_groups, + ) + + def to_json(self): + attrs = {} + if self.authenticationId: + attrs["authenticationId"] = self.authenticationId + if self.firstname: + attrs["firstname"] = self.firstname + if self.lastname: + attrs["lastname"] = self.lastname + if self.email: + attrs["email"] = self.email + + data = { + "id": self.id, + "type": "user", + "attributes": attrs, + } + + if not self.user_groups: + return data + + relsdata = [ + {"id": group, "type": "userGroup"} for group in self.user_groups + ] + if relsdata: + data["relationships"] = {"userGroups": {"data": relsdata}} + return data + + +def test_user_obj_from_sdk(): + user_input = MockUser( + "some.user", "some", "user", "some@email.com", "auth", ["ug"] + ) + excepted = UserIncrementalLoad( + user_id="some.user", + firstname="some", + lastname="user", + email="some@email.com", + auth_id="auth", + user_groups=["ug"], + is_active=True, + ) + user = UserIncrementalLoad.from_sdk_obj(user_input.to_sdk()) + assert excepted == user + + +def test_user_obj_from_sdk_no_ugs(): + user_input = MockUser( + "some.user", "some", "user", "some@email.com", "auth", [] + ) + excepted = UserIncrementalLoad( + user_id="some.user", + firstname="some", + lastname="user", + email="some@email.com", + auth_id="auth", + user_groups=[], + is_active=True, + ) + user = UserIncrementalLoad.from_sdk_obj(user_input.to_sdk()) + assert excepted == user + + +def test_user_obj_to_sdk(): + user_input = MockUser( + "some.user", "some", "user", "some@email.com", "auth", ["ug"] + ) + user = UserIncrementalLoad( + user_id="some.user", + firstname="some", + lastname="user", + email="some@email.com", + auth_id="auth", + user_groups=["ug"], + is_active=True, + ) + excepted = user_input.to_sdk() + assert excepted == user.to_sdk_obj() + + +def test_user_obj_to_sdk_no_ugs(): + user_input = MockUser( + "some.user", "some", "user", "some@email.com", "auth", [] + ) + user = UserIncrementalLoad( + user_id="some.user", + firstname="some", + lastname="user", + email="some@email.com", + auth_id="auth", + user_groups=[], + is_active=True, + ) + excepted = user_input.to_sdk() + assert excepted == user.to_sdk_obj() + + +class MockResponse: + def __init__( + self, status_code, json_response: dict[str, Any] = {}, text: str = "" + ): + self.status_code = status_code + self.json_response = json_response + self.text = text + + def json(self): + return self.json_response + + +UPSTREAM_USERS = { + "jozef.mrkva": MockUser( + "jozef.mrkva", "jozef", "mrkva", "jozef.mrkva@test.com", "auth_id_1", [] + ), + "kristian.kalerab": MockUser( + "kristian.kalerab", + "kristian", + "kalerab", + "kristian.kalerab@test.com", + "auth_id_5", + [], + ), + "richard.cvikla": MockUser( + "richard.cvikla", "richard", "cvikla", None, "auth_id_6", [] + ), + "adam.avokado": MockUser("adam.avokado", None, None, None, "auth_id_7", []), +} + +UPSTREAM_UG_ID = "ug_1" +EXPECTED_NEW_UG_OBJ = CatalogUserGroup.init("ug_2", "ug_2") +EXPECTED_GET_IDS = { + "jozef.mrkva", + "kristian.kalerab", + "peter.pertzlen", + "zoltan.zeler", +} +EXPECTED_CREATE_OR_UPDATE_IDS = { + "peter.pertzlen", + "zoltan.zeler", + "kristian.kalerab", +} + + +def prepare_sdk(): + def mock_get_user(user_id): + if user_id not in UPSTREAM_USERS: + raise NotFoundException + return UPSTREAM_USERS[user_id].to_sdk() + + def mock_get_user_group(ug_id): + if ug_id != UPSTREAM_UG_ID: + raise NotFoundException + return + + sdk = mock.Mock() + sdk.catalog_user.get_user.side_effect = mock_get_user + sdk.catalog_user.get_user_group.side_effect = mock_get_user_group + return sdk + + +""" +jozef - No change; user exists +bartolomej - no change; user doesnt exist +peter - create (2 ugs); 1 ug exists, 1 doesnt +zoltan - create (1 ug); ug exists +kristian - update +richard - delete (diff fields than in upstream) +adam - delete (same fields as in upstream) +""" diff --git a/gooddata-pipelines/tests/provisioning/entities/workspaces/__init__.py b/gooddata-pipelines/tests/provisioning/entities/workspaces/__init__.py new file mode 100644 index 0000000..37d863d --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/entities/workspaces/__init__.py @@ -0,0 +1 @@ +# (C) 2025 GoodData Corporation diff --git a/gooddata-pipelines/tests/provisioning/entities/workspaces/test_provisioning.py b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_provisioning.py new file mode 100644 index 0000000..45183f8 --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_provisioning.py @@ -0,0 +1,53 @@ +# (C) 2025 GoodData Corporation + +from gooddata_pipelines.provisioning.provisioning import Provisioning + +MOCK_PROVISIONER: Provisioning = Provisioning.create("host", "token") + + +def test_create_groups_base_case() -> None: + provisioner: Provisioning = MOCK_PROVISIONER + test_source_ids = {"source_id_1", "source_id_2", "source_id_3"} + test_panther_ids = {"source_id_2", "source_id_3", "source_id_4"} + + id_groups = provisioner._create_groups(test_source_ids, test_panther_ids) + + assert id_groups.ids_in_both_systems == {"source_id_2", "source_id_3"} + assert id_groups.ids_to_delete == {"source_id_4"} + assert id_groups.ids_to_create == {"source_id_1"} + + +def test_create_groups_empty_sets() -> None: + provisioner: Provisioning = MOCK_PROVISIONER + test_source_ids: set[str] = set() + test_panther_ids: set[str] = set() + + id_groups = provisioner._create_groups(test_source_ids, test_panther_ids) + + assert id_groups.ids_in_both_systems == set() + assert id_groups.ids_to_delete == set() + assert id_groups.ids_to_create == set() + + +def test_create_groups_no_overlap() -> None: + provisioner: Provisioning = MOCK_PROVISIONER + test_source_ids = {"source_id_1", "source_id_2"} + test_panther_ids = {"source_id_3", "source_id_4"} + + id_groups = provisioner._create_groups(test_source_ids, test_panther_ids) + + assert id_groups.ids_in_both_systems == set() + assert id_groups.ids_to_delete == {"source_id_3", "source_id_4"} + assert id_groups.ids_to_create == {"source_id_1", "source_id_2"} + + +def test_create_groups_full_overlap() -> None: + provisioner: Provisioning = MOCK_PROVISIONER + test_source_ids = {"source_id_1", "source_id_2"} + test_panther_ids = {"source_id_1", "source_id_2"} + + id_groups = provisioner._create_groups(test_source_ids, test_panther_ids) + + assert id_groups.ids_in_both_systems == {"source_id_1", "source_id_2"} + assert id_groups.ids_to_delete == set() + assert id_groups.ids_to_create == set() diff --git a/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace.py b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace.py new file mode 100644 index 0000000..81f73b6 --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace.py @@ -0,0 +1,143 @@ +# (C) 2025 GoodData Corporation + +from gooddata_sdk.catalog.workspace.entity_model.workspace import ( + CatalogWorkspace, +) + +from gooddata_pipelines.provisioning import WorkspaceProvisioner +from gooddata_pipelines.provisioning.entities.workspaces.models import ( + WorkspaceFullLoad, +) + +MOCK_WORKSPACE_PROVISIONER: WorkspaceProvisioner = WorkspaceProvisioner.create( + "host", "token" +) + + +def test_find_workspaces_to_update_same_ids_and_names() -> None: + provisioner: WorkspaceProvisioner = MOCK_WORKSPACE_PROVISIONER + ids_in_both_systems = {"workspace_id1", "workspace_id2"} + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="client_id1", + workspace_id="workspace_id1", + workspace_name="workspace_title1", + ), + WorkspaceFullLoad( + parent_id="client_id2", + workspace_id="workspace_id2", + workspace_name="workspace_title2", + ), + ] + panther_group: list[CatalogWorkspace] = [ + CatalogWorkspace( + workspace_id="workspace_id1", + name="workspace_title1", + parent_id="parent_id", + ), + CatalogWorkspace( + workspace_id="workspace_id2", + name="workspace_title2", + parent_id="parent_id", + ), + ] + + workspaces_to_update = provisioner._find_workspaces_to_update( + source_group, panther_group, ids_in_both_systems + ) + + assert workspaces_to_update == set() + + +def test_find_workspaces_to_update_different_ids() -> None: + provisioner: WorkspaceProvisioner = MOCK_WORKSPACE_PROVISIONER + ids_in_both_systems = {"workspace_id1", "workspace_id2"} + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="client_id1", + workspace_id="workspace_id1", + workspace_name="workspace_title1", + ), + WorkspaceFullLoad( + parent_id="client_id2", + workspace_id="workspace_id2", + workspace_name="workspace_title2", + ), + ] + panther_group: list[CatalogWorkspace] = [ + CatalogWorkspace( + workspace_id="workspace_id1", + name="workspace_title1", + parent_id="parent_id", + ), + CatalogWorkspace( + workspace_id="workspace_id2", + name="workspace_title2", + parent_id="parent_id", + ), + ] + + workspaces_to_update = provisioner._find_workspaces_to_update( + source_group, panther_group, ids_in_both_systems + ) + + assert workspaces_to_update == set() + + +def test_find_workspaces_to_update_same_ids_different_names() -> None: + provisioner: WorkspaceProvisioner = MOCK_WORKSPACE_PROVISIONER + ids_in_both_systems: set[str] = {"workspace_id1", "workspace_id2"} + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="client_id1", + workspace_id="workspace_id1", + workspace_name="workspace_title1", + ), + WorkspaceFullLoad( + parent_id="client_id2", + workspace_id="workspace_id2", + workspace_name="workspace_title2", + ), + ] + panther_group: list[CatalogWorkspace] = [ + CatalogWorkspace( + workspace_id="workspace_id1", + name="old_workspace_title1", + parent_id="parent_id", + ), + CatalogWorkspace( + workspace_id="workspace_id2", + name="old_workspace_title2", + parent_id="parent_id", + ), + ] + + workspaces_to_update = provisioner._find_workspaces_to_update( + source_group, panther_group, ids_in_both_systems + ) + + assert workspaces_to_update == {"workspace_id1", "workspace_id2"} + + +def test_find_workspaces_to_update_no_panther() -> None: + provisioner: WorkspaceProvisioner = MOCK_WORKSPACE_PROVISIONER + ids_in_both_systems: set[str] = set() + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="client_id1", + workspace_id="workspace_id1", + workspace_name="workspace_title1", + ), + WorkspaceFullLoad( + parent_id="client_id2", + workspace_id="workspace_id2", + workspace_name="workspace_title2", + ), + ] + panther_group: list[CatalogWorkspace] = [] + + workspaces_to_update = provisioner._find_workspaces_to_update( + source_group, panther_group, ids_in_both_systems + ) + + assert workspaces_to_update == set() diff --git a/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace_data_filters.py b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace_data_filters.py new file mode 100644 index 0000000..31f2d73 --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace_data_filters.py @@ -0,0 +1,193 @@ +# (C) 2025 GoodData Corporation + +import json +from typing import Any + +import pytest +from pydantic import ValidationError +from requests import Response + +from gooddata_pipelines.provisioning.entities.workspaces.workspace_data_filters import ( + WDFSetting, + WorkspaceDataFilterManager, +) +from gooddata_pipelines.provisioning.entities.workspaces.workspace_data_parser import ( + WorkspaceDataMaps, +) +from gooddata_pipelines.provisioning.utils.context_objects import ( + WorkspaceContext, +) +from gooddata_pipelines.provisioning.utils.exceptions import WorkspaceException +from tests.data.mock_responses import ( + WDF_ACTUAL_WDF_SETTINGS, + WDF_VALID_PAYLOAD, +) + + +@pytest.fixture +def wdf_manager(mock_gooddata_api): + return WorkspaceDataFilterManager(mock_gooddata_api, WorkspaceDataMaps()) + + +def test_create_wdf_setting_dict(wdf_manager) -> None: + """Test construction of the WDF setting dictionary.""" + wdf_setting_id: str = "expected_wdf_setting_id" + wdf_id: str = "expected_wdf_id" + wdf_values: list[str] = ["expected", "wdf", "values"] + + expected_setting: dict[str, Any] = { + "data": { + "attributes": {"filterValues": wdf_values}, + "id": wdf_setting_id, + "relationships": { + "workspaceDataFilter": { + "data": {"id": wdf_id, "type": "workspaceDataFilter"} + } + }, + "type": "workspaceDataFilterSetting", + } + } + + result_setting: dict[str, Any] = wdf_manager._create_wdf_setting_dict( + wdf_setting_id, wdf_id, wdf_values + ) + + assert result_setting == expected_setting, ( + f"Expected {expected_setting}, but got {result_setting}" + ) + + +def test_get_wdf_settings_for_workspace_valid_payload( + wdf_manager, mock_gooddata_api, mocker +) -> None: + """Test processing of a valid response""" + workspace_id: str = "expected_workspace_id" + mock_response: Response = Response() + mock_response.status_code = 200 + + payload = WDF_VALID_PAYLOAD + + mock_response._content = json.dumps(payload).encode("utf-8") + + mocker.patch.object( + mock_gooddata_api, + "get_workspace_data_filter_settings", + return_value=mock_response, + ) + + result = wdf_manager._get_wdf_settings_for_workspace(workspace_id) + + assert isinstance(result[0], WDFSetting), ( + f"Expected WDFSetting instance, but got {type(result)}" + ) + + +def test_get_wdf_settings_for_workspace_invalid_payload( + wdf_manager, mock_gooddata_api, mocker +) -> None: + """Test with an invalid payload -> will raise ValidationError / ValueError""" + workspace_id: str = "expected_workspace_id" + mock_response: Response = Response() + mock_response.status_code = 200 + + payload = { + "data": [ + { + "id": "expected_wdf_setting_id", + "type": "workspaceDataFilterSetting", + "attributes": { + "missing": "data", + }, + } + ] + } + + mock_response._content = json.dumps(payload).encode("utf-8") + + mocker.patch.object( + mock_gooddata_api, + "get_workspace_data_filter_settings", + return_value=mock_response, + ) + + with pytest.raises(ValidationError): + wdf_manager._get_wdf_settings_for_workspace(workspace_id) + + +def test_get_actual_wdf_setting_id_and_values(wdf_manager) -> None: + """Test getting the actual WDF setting ID and values.""" + data: dict[str, Any] = WDF_VALID_PAYLOAD["data"][0] + actual_wdf_settings: list[WDFSetting] = [WDFSetting(**data)] + wdf_id: str = "expected_wdf_id" + + actual_wdf_setting_id, actual_wdf_values = ( + wdf_manager._get_actual_wdf_setting_id_and_values( + actual_wdf_settings, wdf_id + ) + ) + + assert actual_wdf_setting_id == "expected_wdf_setting_id", ( + f"Expected 'expected_wdf_setting_id', but got {actual_wdf_setting_id}" + ) + + assert actual_wdf_values == ["expected", "wdf", "values"], ( + f"Expected ['expected', 'wdf', 'values'], but got {actual_wdf_values}" + ) + + +def test_get_actual_wdf_setting_id_and_values_no_actuals(wdf_manager) -> None: + """Should raise ValueError if no actuals are found""" + actual_wdf_settings: list[WDFSetting] = [] + wdf_id: str = "expected_wdf_id" + + with pytest.raises(WorkspaceException): + actual_wdf_setting_id, actual_wdf_values = ( + wdf_manager._get_actual_wdf_setting_id_and_values( + actual_wdf_settings, wdf_id + ) + ) + + +def test_get_actual_wdf_setting_id_and_values_no_match(wdf_manager) -> None: + """Should raise ValueError if no match is found""" + data: dict[str, Any] = WDF_VALID_PAYLOAD["data"][0] + actual_wdf_settings: list[WDFSetting] = [WDFSetting(**data)] + wdf_id: str = "non_existent_wdf_id" + + with pytest.raises(WorkspaceException): + actual_wdf_setting_id, actual_wdf_values = ( + wdf_manager._get_actual_wdf_setting_id_and_values( + actual_wdf_settings, wdf_id + ) + ) + + +def test_compare_wdf_settings(wdf_manager, mocker) -> None: + """Test the comparison of WDF settings.""" + workspace_context: WorkspaceContext = WorkspaceContext( + workspace_id="workspace_id", workspace_name="workspace_name" + ) + source_wdf_config: dict[str, list[str]] = { + "expected_wdf_id": ["expected", "wdf", "values"], + "expected_wdf_id_2": ["unexpected", "wdf", "values"], + "expected_wdf_id_3": ["expected", "wdf", "values"], + } + actual_wdf_settings: list[WDFSetting] = [ + WDFSetting(**setting) for setting in WDF_ACTUAL_WDF_SETTINGS + ] + + mock_put = mocker.patch.object(wdf_manager, "_put_wdf_setting") + mock_post = mocker.patch.object(wdf_manager, "_post_wdf_setting") + mock_delete = mocker.patch.object( + wdf_manager, "_delete_redundant_wdf_setting" + ) + + wdf_manager._compare_wdf_settings( + workspace_context, + source_wdf_config, + actual_wdf_settings, + ) + + assert mock_put.call_count == 1, "Expected one PUT call" + assert mock_post.call_count == 1, "Expected one POST call" + assert mock_delete.call_count == 1, "Expected one DELETE call" diff --git a/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace_data_parser.py b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace_data_parser.py new file mode 100644 index 0000000..acf25fe --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace_data_parser.py @@ -0,0 +1,290 @@ +# (C) 2025 GoodData Corporation + +from gooddata_sdk.catalog.workspace.entity_model.workspace import ( + CatalogWorkspace, +) + +from gooddata_pipelines.provisioning.entities.workspaces.models import ( + WorkspaceFullLoad, +) +from gooddata_pipelines.provisioning.entities.workspaces.workspace_data_parser import ( + WorkspaceDataParser, +) + +parser = WorkspaceDataParser() + + +def test_get_id_to_name_map_no_overlap() -> None: + """No overlap between source and Panther groups.""" + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="some_parent", + workspace_id="1", + workspace_name="Source Workspace 1", + ), + WorkspaceFullLoad( + parent_id="some_parent", + workspace_id="2", + workspace_name="Source Workspace 2", + ), + ] + panther_group: list[CatalogWorkspace] = [ + CatalogWorkspace(workspace_id="3", name="Panther Workspace 1"), + CatalogWorkspace(workspace_id="4", name="Panther Workspace 2"), + ] + + expected_result = { + "3": "Panther Workspace 1", + "4": "Panther Workspace 2", + "1": "Source Workspace 1", + "2": "Source Workspace 2", + } + + result = parser._get_id_to_name_map(source_group, panther_group) + assert result == expected_result + + +def test_get_id_to_name_map_with_overlap() -> None: + """Overlaping groups -> source group name takse precedence.""" + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="some_parent", + workspace_id="1", + workspace_name="Source Workspace 1", + ), + WorkspaceFullLoad( + parent_id="some_parent", + workspace_id="2", + workspace_name="Source Workspace 2", + ), + ] + panther_group: list[CatalogWorkspace] = [ + CatalogWorkspace(workspace_id="1", name="Panther Workspace 1"), + CatalogWorkspace(workspace_id="4", name="Panther Workspace 2"), + ] + + expected_result = { + "1": "Source Workspace 1", + "4": "Panther Workspace 2", + "2": "Source Workspace 2", + } + + result = parser._get_id_to_name_map(source_group, panther_group) + assert result == expected_result + + +def test_get_id_to_name_map_empty() -> None: + """Empty source and Panther groups -> will return empty dict.""" + source_group: list[WorkspaceFullLoad] = [] + panther_group: list[CatalogWorkspace] = [] + + expected_result: dict[str, str] = {} + + result = parser._get_id_to_name_map(source_group, panther_group) + assert result == expected_result + + +def test_get_child_to_parent_map() -> None: + """Maps child ID to parent ID.""" + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + workspace_id="child_1", + parent_id="parent_1", + workspace_name="Child Workspace 1", + ), + WorkspaceFullLoad( + workspace_id="child_2", + parent_id="parent_2", + workspace_name="Child Workspace 2", + ), + WorkspaceFullLoad( + workspace_id="child_3", + parent_id="parent_1", + workspace_name="Child Workspace 3", + ), + ] + + expected_result = { + "child_1": "parent_1", + "child_2": "parent_2", + "child_3": "parent_1", + } + + result = parser._get_child_to_parent_map(source_group) + assert result == expected_result + + +def test_get_child_to_parent_map_empty() -> None: + """Empty source group -> will return empty dict.""" + source_group: list[WorkspaceFullLoad] = [] + + expected_result: dict[str, str] = {} + + result = parser._get_child_to_parent_map(source_group) + assert result == expected_result + + +def test_get_child_to_parent_map_with_duplicates() -> None: + """Child ID will be unique in the resulting dict""" + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + workspace_id="child_1", + parent_id="parent_1", + workspace_name="Child Workspace 1", + ), + WorkspaceFullLoad( + workspace_id="child_2", + parent_id="parent_2", + workspace_name="Child Workspace 2", + ), + WorkspaceFullLoad( + workspace_id="child_3", + parent_id="parent_1", + workspace_name="Child Workspace 3", + ), + WorkspaceFullLoad( + workspace_id="child_1", + parent_id="parent_1", + workspace_name="Child Workspace 1", + ), + WorkspaceFullLoad( + workspace_id="child_2", + parent_id="parent_2", + workspace_name="Child Workspace 2", + ), + WorkspaceFullLoad( + workspace_id="child_3", + parent_id="parent_1", + workspace_name="Child Workspace 3", + ), + ] + + expected_result: dict[str, str] = { + "child_1": "parent_1", + "child_2": "parent_2", + "child_3": "parent_1", + } + + result = parser._get_child_to_parent_map(source_group) + assert result == expected_result + + +def test_get_child_to_wdfs_map() -> None: + """Mapping child ID to WDF ID and WDF values.""" + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="parent_1", + workspace_id="child_1", + workspace_name="Child Workspace 1", + workspace_data_filter_id="wdf_1", + workspace_data_filter_values=["value_1", "value_2"], + ), + WorkspaceFullLoad( + parent_id="parent_2", + workspace_id="child_2", + workspace_name="Child Workspace 2", + workspace_data_filter_id="wdf_2", + workspace_data_filter_values=["value_3", "value_4"], + ), + WorkspaceFullLoad( + parent_id="parent_1", + workspace_id="child_3", + workspace_name="Child Workspace 3", + workspace_data_filter_id=None, + workspace_data_filter_values=None, + ), + ] + + expected_result = { + "child_1": {"wdf_1": ["value_1", "value_2"]}, + "child_2": {"wdf_2": ["value_3", "value_4"]}, + } + + result = parser._get_child_to_wdfs_map(source_group) + assert result == expected_result + + +def test_get_child_to_wdfs_map_integers() -> None: + """Is capable of handling int values (in case source column is int).""" + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + workspace_id="child_1", + parent_id="parent_1", + workspace_name="Child Workspace 1", + workspace_data_filter_id="wdf_1", + workspace_data_filter_values=[1], # type: ignore + ), + WorkspaceFullLoad( + workspace_id="child_2", + parent_id="parent_2", + workspace_name="Child Workspace 2", + workspace_data_filter_id="wdf_2", + workspace_data_filter_values=["value_3", "value_4"], + ), + WorkspaceFullLoad( + workspace_id="child_3", + parent_id="parent_1", + workspace_name="Child Workspace 3", + workspace_data_filter_id=None, + workspace_data_filter_values=None, + ), + ] + + expected_result = { + "child_1": {"wdf_1": ["1"]}, + "child_2": {"wdf_2": ["value_3", "value_4"]}, + } + + result = parser._get_child_to_wdfs_map(source_group) + assert result == expected_result + + +def test_get_child_to_wdfs_map_multiple_wdfs() -> None: + """Handles multiple WDFs on a child workspace.""" + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + workspace_id="child_1", + parent_id="parent_1", + workspace_name="Child Workspace 1", + workspace_data_filter_id="wdf_1", + workspace_data_filter_values=[1], # type: ignore + ), + WorkspaceFullLoad( + workspace_id="child_1", + parent_id="parent_1", + workspace_name="Child Workspace 1", + workspace_data_filter_id="wdf_2", + workspace_data_filter_values=["value_3", "value_4"], + ), + WorkspaceFullLoad( + workspace_id="child_2", + parent_id="parent_2", + workspace_name="Child Workspace 2", + workspace_data_filter_id="wdf_2", + workspace_data_filter_values=["value_3", "value_4"], + ), + WorkspaceFullLoad( + workspace_id="child_3", + parent_id="parent_1", + workspace_name="Child Workspace 3", + workspace_data_filter_id=None, + workspace_data_filter_values=None, + ), + ] + + expected_result = { + "child_1": {"wdf_1": ["1"], "wdf_2": ["value_3", "value_4"]}, + "child_2": {"wdf_2": ["value_3", "value_4"]}, + } + + result = parser._get_child_to_wdfs_map(source_group) + assert result == expected_result + + +def test_get_child_to_wdfs_map_empty() -> None: + source_group: list[WorkspaceFullLoad] = [] + + expected_result: dict[str, dict[str, list[str]]] = {} + + result = parser._get_child_to_wdfs_map(source_group) + assert result == expected_result diff --git a/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace_data_validator.py b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace_data_validator.py new file mode 100644 index 0000000..a82b004 --- /dev/null +++ b/gooddata-pipelines/tests/provisioning/entities/workspaces/test_workspace_data_validator.py @@ -0,0 +1,210 @@ +# (C) 2025 GoodData Corporation + +import logging + +import pytest + +from gooddata_pipelines.logger.logger import LogObserver +from gooddata_pipelines.provisioning.entities.workspaces.models import ( + WorkspaceFullLoad, +) +from gooddata_pipelines.provisioning.entities.workspaces.workspace_data_validator import ( + WorkspaceDataValidator, +) +from gooddata_pipelines.provisioning.utils.exceptions import ( + WorkspaceException, +) + +# Set up logging for the test +logger = logging.getLogger("test_logger") + +observer = LogObserver() +observer.subscribe(logger) + + +@pytest.fixture +def validator(mock_gooddata_api): + return WorkspaceDataValidator(mock_gooddata_api) + + +def test_check_basic_integrity_on_valid_data(validator) -> None: + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="parent_id_1", + workspace_id="workspace_id_1", + workspace_name="workspace_name_1", + workspace_data_filter_id="wdf_id_1", + workspace_data_filter_values=["wdf_values_1"], + ), + WorkspaceFullLoad( + parent_id="parent_id_2", + workspace_id="workspace_id_2", + workspace_name="workspace_name_2", + workspace_data_filter_id=None, + workspace_data_filter_values=None, + ), + WorkspaceFullLoad( + parent_id="parent_id_3", + workspace_id="workspace_id_3", + workspace_name="workspace_name_3", + workspace_data_filter_id="wdf_id_1", + workspace_data_filter_values=["wdf_values_1"], + ), + WorkspaceFullLoad( + parent_id="parent_id_3", + workspace_id="workspace_id_3", + workspace_name="workspace_name_3", + workspace_data_filter_id="wdf_id_2", + workspace_data_filter_values=["wdf_values_1", "wdf_values_2"], + ), + ] + + parent_workspaces, parent_wdf_map = validator._check_basic_integrity( + source_group + ) + + assert parent_workspaces == {"parent_id_1", "parent_id_2", "parent_id_3"} + assert parent_wdf_map == { + "parent_id_1": ["wdf_id_1"], + "parent_id_3": ["wdf_id_1", "wdf_id_2"], + } + + +def test_check_basic_integrity_missing_parent_id(validator) -> None: + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="", + workspace_id="workspace_id_1", + workspace_name="workspace_name_1", + workspace_data_filter_id=None, + workspace_data_filter_values=None, + ), + ] + + try: + validator._check_basic_integrity(source_group) + except Exception as e: + assert isinstance(e, WorkspaceException) + assert e.error_message == "Parent ID is not defined in source data." + assert e.workspace_id == "workspace_id_1" + + +def test_check_basic_integrity_missing_workspace_id(validator) -> None: + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="parent_id_1", + workspace_id="", + workspace_name="workspace_name_1", + workspace_data_filter_id=None, + workspace_data_filter_values=None, + ), + ] + + try: + validator._check_basic_integrity(source_group) + except Exception as e: + assert isinstance(e, WorkspaceException) + assert ( + e.error_message + == "Workspace ID is not defined for parent parent_id_1" + ) + + +def test_check_basic_integrity_missing_wdf_id(validator) -> None: + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="parent_id_1", + workspace_id="workspace_id_1", + workspace_name="workspace_name_1", + workspace_data_filter_id=None, + workspace_data_filter_values=["some_values"], + ), + ] + + try: + validator._check_basic_integrity(source_group) + except Exception as e: + assert isinstance(e, WorkspaceException) + assert ( + e.error_message + == "WDF values are provided but WDF ID is not defined." + ) + assert e.workspace_name == "workspace_name_1" + assert e.workspace_id == "workspace_id_1" + assert e.wdf_values == "some_values" + + +def test_check_basic_integrity_missing_wdf_values(validator) -> None: + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="parent_id_1", + workspace_id="workspace_id_1", + workspace_name="workspace_name_1", + workspace_data_filter_id="wdf_id_1", + workspace_data_filter_values=None, + ), + ] + + try: + validator._check_basic_integrity(source_group) + except Exception as e: + assert isinstance(e, WorkspaceException) + assert ( + e.error_message + == "WDF ID is defined but no WDF values are provided" + ) + assert e.workspace_id == "workspace_id_1" + assert e.wdf_id == "wdf_id_1" + assert e.workspace_name == "workspace_name_1" + assert e.wdf_values is None + + +def test_check_basic_integrity_missing_both_ids(validator) -> None: + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="", + workspace_id="", + workspace_name="", + workspace_data_filter_id=None, + workspace_data_filter_values=None, + ), + ] + + try: + validator._check_basic_integrity(source_group) + except Exception as e: + assert isinstance(e, WorkspaceException) + assert e.error_message == ( + "Parent ID and workspace ID are not defined for at least one row. Please check the source data." + ) + + +def test_check_basic_integrity_duplicates_warning(validator, caplog) -> None: + source_group: list[WorkspaceFullLoad] = [ + WorkspaceFullLoad( + parent_id="parent_id_1", + workspace_id="workspace_id_1", + workspace_name="workspace_name_1", + workspace_data_filter_id="wdf_id_1", + workspace_data_filter_values=["wdf_values_1"], + ), + WorkspaceFullLoad( + parent_id="parent_id_1", + workspace_id="workspace_id_1", + workspace_name="workspace_name_1", + workspace_data_filter_id="wdf_id_1", + workspace_data_filter_values=["wdf_values_1"], + ), + ] + with caplog.at_level("WARNING"): + validator._check_basic_integrity(source_group) + + warning_log_found = any( + "Duplicate combinations of parent_id, workspace_id, wdf_id exist in the source data." + in message + for message in caplog.messages + ) + + assert warning_log_found, ( + "Warning log for duplicate combinations not found in logs." + ) diff --git a/gooddata-pipelines/tox.ini b/gooddata-pipelines/tox.ini new file mode 100644 index 0000000..9a63c33 --- /dev/null +++ b/gooddata-pipelines/tox.ini @@ -0,0 +1,20 @@ +# (C) 2025 GoodData Corporation +[tox] +envlist = py3{10,11,12,13} + +[testenv] +deps = + pytest + pytest-mock + poetry +commands = poetry run pytest + +[testenv:mypy] +basepython = python3.13 +deps = + poetry + mypy +skip_install = true +commands = + poetry install + mypy gooddata_pipelines