Bump urllibl3 version to 2.0.7 and fix Azure unit tests (#502)

* Bump urllibl3 version to 2.0.7

* Fix unit tests

* Update unit test

* Bump library version to 20240214

* Last poetry lock run

* Appease mypy

* Appease mypy

Author: jleaniz

libcloudforensics/providers/gcp/internal/bigquery.py

@@ -55,5 +55,5 @@ def ListBigQueryJobs(self) -> List[Dict[str, Any]]:
     """
     bq_jobs = self.GoogleBigQueryApi().jobs()  # pylint: disable=no-member
     request = bq_jobs.list(projectId=self.project_id, projection='full')
-    jobs = request.execute()  # type: Dict[str, Any]
-    return jobs.get('jobs', [])
+    jobs: List[Dict[str, Any]] = request.execute().get('jobs', [])
+    return jobs

libcloudforensics/providers/gcp/internal/cloudsql.py

@@ -56,5 +56,5 @@ def ListCloudSQLInstances(self) -> List[Dict[str, Any]]:
     """
     gcsql_instances = self.GoogleCloudSQLApi().instances() # pylint: disable=no-member
     request = gcsql_instances.list(project=self.project_id)
-    instances = request.execute()  # type: Dict[str, Any]
-    return instances.get('items', [])
+    instances: List[Dict[str, Any]] = request.execute().get('items', [])
+    return instances

libcloudforensics/providers/gcp/internal/storage.py

@@ -135,8 +135,8 @@ def ListBuckets(self) -> List[Dict[str, Any]]:
     """
     gcs_buckets = self.GcsApi().buckets() # pylint: disable=no-member
     request = gcs_buckets.list(project=self.project_id)
-    objects = request.execute()  # type: Dict[str, Any]
-    return objects.get('items', [])
+    objects: List[Dict[str, Any]] = request.execute().get('items', [])
+    return objects
 
   def ListBucketObjects(self, bucket: str) -> List[Dict[str, Any]]:
     """List objects (with metadata) in a Google Cloud Storage bucket.
@@ -152,8 +152,8 @@ def ListBucketObjects(self, bucket: str) -> List[Dict[str, Any]]:
       bucket = bucket[5:]
     gcs_objects = self.GcsApi().objects() # pylint: disable=no-member
     request = gcs_objects.list(bucket=bucket)
-    objects = request.execute()  # type: Dict[str, Any]
-    return objects.get('items', [])
+    objects: List[Dict[str, Any]] = request.execute().get('items', [])
+    return objects
 
   def DeleteObject(self, gcs_path: str) -> None:
     """Deletes an object in a Google Cloud Storage bucket.

poetry.lock

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "libcloudforensics"
-version = "20230601"
+version = "20240214"
 description = "libcloudforensics is a set of tools to help acquire forensic evidence from Cloud platforms."
 authors = ["cloud-forensics-utils development team <cloud-forensics-utils-dev@googlegroups.com>"]
 license = "Apache-2.0"
@@ -10,14 +10,15 @@ readme = "README.md"
 python = "^3.8"
 google-api-core = "^2.11.1"
 azure-common = "^1.1.28"
-azure-core = "^1.28.0"
+azure-core = "^1.29.4"
 azure-identity = "^1.13.0"
 azure-mgmt-compute = "^30.1.0"
 azure-mgmt-monitor = "^6.0.1"
 azure-mgmt-network = "^24.0.0"
-azure-mgmt-resource = "23.0.1"
+azure-mgmt-reservations = "^2.3.0"
+azure-mgmt-resource = "^23.0.1"
 azure-mgmt-storage = "^21.0.0"
-azure-storage-blob = "^12.17.0"
+azure-storage-blob = "^12.18.1"
 boto3 = "^1.28.20"
 botocore = ">=1.29.135"
 google-api-python-client = "^2.95.0"
@@ -31,9 +32,11 @@ sshpubkeys = "^3.3.1"
 requests = "^2.31.0"
 kubernetes = "^27.2.0"
 pyopenssl = "^23.2.0"
-urllib3 = ">=1.21.1,<1.27"
+urllib3 = [
+  {version = ">=1.25.4,<1.27", python = "<3.10"},
+  {version = ">=1.25.4,<2.1", python = ">=3.10"}
+]
 google-auth = "^2.22.0"
-azure-mgmt-reservations = "^2.3.0"
 
 [tool.poetry.group.dev.dependencies]
 coverage = "^7.2.7"

pyproject.toml

@@ -130,7 +130,8 @@
 MOCK_LIST_VM_SIZES = [{
     'Name': 'fake-vm-type',
     'CPU': 4,
-    'Memory': 8192
+    'Memory': 8192,
+    'Family': 'standardB1lsFamily'
 }]
 
 MOCK_ANALYSIS_INSTANCE = mock.Mock(

tests/providers/azure/azure_mocks.py

@@ -124,7 +124,7 @@ def testGetDisk(self, mock_list_disks, mock_provider):
     self.assertEqual(['fake-zone'], disk.zones)
 
   @mock.patch('azure.mgmt.resource.resources.v2022_09_01.operations.ProvidersOperations.get')
-  @mock.patch('azure.mgmt.compute.v2023_01_02.operations.DisksOperations.begin_create_or_update')
+  @mock.patch('azure.mgmt.compute.v2023_10_02.operations.DisksOperations.begin_create_or_update')
   @typing.no_type_check
   def testCreateDiskFromSnapshot(self, mock_create_disk, mock_provider):
     """Test that a disk can be created from a snapshot."""
@@ -193,7 +193,7 @@ def testCreateDiskFromSnapshot(self, mock_create_disk, mock_provider):
   @mock.patch('azure.storage.blob._blob_service_client.BlobServiceClient.__init__')
   @mock.patch('libcloudforensics.providers.azure.internal.storage.AZStorage.DeleteStorageAccount')
   @mock.patch('libcloudforensics.providers.azure.internal.storage.AZStorage.CreateStorageAccount')
-  @mock.patch('azure.mgmt.compute.v2023_01_02.operations.DisksOperations.begin_create_or_update')
+  @mock.patch('azure.mgmt.compute.v2023_10_02.operations.DisksOperations.begin_create_or_update')
   @typing.no_type_check
   def testCreateDiskFromSnapshotUri(self,
                                     mock_create_disk,
@@ -239,7 +239,7 @@ def testCreateDiskFromSnapshotUri(self,
   @mock.patch('libcloudforensics.providers.azure.internal.compute.AZCompute.GetInstance')
   @mock.patch('libcloudforensics.providers.azure.internal.compute.AZCompute._GetInstanceType')
   @mock.patch('libcloudforensics.providers.azure.internal.network.AZNetwork.CreateNetworkInterface')
-  @mock.patch('azure.mgmt.compute.v2023_03_01.operations.VirtualMachinesOperations.begin_create_or_update')
+  @mock.patch('azure.mgmt.compute.v2023_09_01.operations.VirtualMachinesOperations.begin_create_or_update')
   @typing.no_type_check
   def testGetOrCreateAnalysisVm(self,
                                 mock_vm,
@@ -279,12 +279,12 @@ def testGetOrCreateAnalysisVm(self,
     self.assertTrue(created)
 
   @mock.patch('azure.mgmt.resource.resources.v2022_09_01.operations.ProvidersOperations.get')
-  @mock.patch('azure.mgmt.compute.v2021_07_01.operations.ResourceSkusOperations.list')
+  @mock.patch('libcloudforensics.providers.azure.internal.compute.AZCompute.ListInstanceTypes')
   @typing.no_type_check
   def testListVMSizes(self, mock_list, mock_provider):
     """Test that instance types are correctly listed."""
     mock_provider.return_value = azure_mocks.MOCK_CAPACITY_PROVIDER
-    mock_list.return_value = azure_mocks.MOCK_REQUEST_VM_SIZE
+    mock_list.return_value = azure_mocks.MOCK_LIST_VM_SIZES
     available_vms = azure_mocks.FAKE_ACCOUNT.compute.ListInstanceTypes()
     self.assertEqual(1, len(available_vms))
     self.assertEqual('fake-vm-type', available_vms[0]['Name'])
@@ -315,7 +315,7 @@ class AZVirtualMachineTest(unittest.TestCase):
 
   @mock.patch('azure.mgmt.resource.resources.v2022_09_01.operations.ProvidersOperations.get')
   @mock.patch('libcloudforensics.providers.azure.internal.compute.AZCompute.ListDisks')
-  @mock.patch('azure.mgmt.compute.v2023_03_01.operations.VirtualMachinesOperations.get')
+  @mock.patch('azure.mgmt.compute.v2023_09_01.operations.VirtualMachinesOperations.get')
   @typing.no_type_check
   def testGetBootDisk(self, mock_get_vm, mock_list_disk, mock_provider):
     """Test that the boot disk from an instance is retrieved."""
@@ -346,7 +346,7 @@ def testGetDisk(self, mock_list_disk):
 
   @mock.patch('azure.mgmt.resource.resources.v2022_09_01.operations.ProvidersOperations.get')
   @mock.patch('libcloudforensics.providers.azure.internal.compute.AZCompute.ListDisks')
-  @mock.patch('azure.mgmt.compute.v2023_03_01.operations.VirtualMachinesOperations.get')
+  @mock.patch('azure.mgmt.compute.v2023_09_01.operations.VirtualMachinesOperations.get')
   @typing.no_type_check
   def testListDisks(self, mock_get_vm, mock_list_disk, mock_provider):
     """Test that disks from an instance are correctly listed."""

tests/providers/azure/internal/test_compute.py

@@ -40,7 +40,7 @@ class AZForensicsTest(unittest.TestCase):
   @mock.patch('libcloudforensics.providers.azure.internal.compute.AZCompute.GetDisk')
   @mock.patch('libcloudforensics.providers.azure.internal.compute.AZComputeVirtualMachine.GetBootDisk')
   @mock.patch('libcloudforensics.providers.azure.internal.compute.AZCompute.GetInstance')
-  @mock.patch('azure.mgmt.compute.v2023_01_02.operations.DisksOperations.begin_create_or_update')
+  @mock.patch('azure.mgmt.compute.v2023_10_02.operations.DisksOperations.begin_create_or_update')
   @typing.no_type_check
   def testCreateDiskCopy1(self,
                           mock_create_disk,
@@ -92,7 +92,7 @@ def testCreateDiskCopy1(self,
   @mock.patch('libcloudforensics.providers.azure.internal.compute.AZCompute.GetDisk')
   @mock.patch('libcloudforensics.providers.azure.internal.compute.AZComputeVirtualMachine.GetBootDisk')
   @mock.patch('libcloudforensics.providers.azure.internal.compute.AZCompute.GetInstance')
-  @mock.patch('azure.mgmt.compute.v2023_01_02.operations.DisksOperations.begin_create_or_update')
+  @mock.patch('azure.mgmt.compute.v2023_10_02.operations.DisksOperations.begin_create_or_update')
   @typing.no_type_check
   def testCreateDiskCopy2(self,
                           mock_create_disk,