Azure sdk update (cloud-custodian#6544)

Author: Kirill Logachev

tools/c7n_azure/c7n_azure/__init__.py

@@ -9,3 +9,7 @@
 logging.getLogger("msrest").setLevel(logging.ERROR)
 logging.getLogger("keyring").setLevel(logging.WARNING)
 logging.getLogger("azure.storage.common.storageclient").setLevel(logging.WARNING)
+
+# This logger is spamming INFO with a bunch of requests data
+logging.getLogger('azure.identity').setLevel(logging.WARNING)
+logging.getLogger('azure.core.pipeline.policies.http_logging_policy').setLevel(logging.WARNING)

tools/c7n_azure/c7n_azure/actions/delete.py

@@ -64,6 +64,7 @@ def _process_resource(self, resource):
         if is_resource_group(resource):
             self.client.resource_groups.delete(resource['name'])
         else:
-            self.client.resources.delete_by_id(resource['id'],
-                                               self.session.resource_api_version(resource['id']))
+            self.client.resources.begin_delete_by_id(
+                resource['id'],
+                self.session.resource_api_version(resource['id']))
         return "deleted"

tools/c7n_azure/c7n_azure/actions/logic_app.py

@@ -1,7 +1,7 @@
 # Copyright The Cloud Custodian Authors.
 # SPDX-License-Identifier: Apache-2.0
 
-from azure.mgmt.logic import logic_management_client
+from azure.mgmt.logic import LogicManagementClient
 
 from c7n.actions.webhook import Webhook
 from c7n.utils import type_schema
@@ -59,9 +59,9 @@ def process(self, resources, event=None):
     def get_callback_url(self, resource_group, workflow_name):
         """ Gets the logic app invoke trigger with secrets using RBAC """
 
-        # type: logic_management_client.LogicManagementClient
+        # type: LogicManagementClient
         client = self.manager.get_client(
-            'azure.mgmt.logic.logic_management_client.LogicManagementClient')
+            'azure.mgmt.logic.LogicManagementClient')
 
         callback = client.workflow_triggers.list_callback_url(
             resource_group,

tools/c7n_azure/c7n_azure/azure_events.py

@@ -139,5 +139,6 @@ def create(destination, name, subscription_id, session=None, event_filter=None):
         scope = '/subscriptions/%s' % subscription_id
 
         client = s.client('azure.mgmt.eventgrid.EventGridManagementClient')
-        event_subscription = client.event_subscriptions.create_or_update(scope, name, event_info)
+        event_subscription = \
+            client.event_subscriptions.begin_create_or_update(scope, name, event_info)
         return event_subscription.result()

tools/c7n_azure/c7n_azure/container_host/host.py

@@ -159,15 +159,15 @@ def update_policies(self):
 
             client.get_blob_to_path(container, blob.name, policy_path)
             self.load_policy(policy_path, policies_copy)
-            self.blob_cache.update({blob.name: blob.properties.content_settings.content_md5})
+            self.blob_cache.update({blob.name: blob.content_settings.content_md5})
 
         # Assign our copy back over the original
         self.policies = policies_copy
 
     def _get_new_blobs(self, blobs):
         new_blobs = []
         for blob in blobs:
-            md5_hash = blob.properties.content_settings.content_md5
+            md5_hash = blob.content_settings.content_md5
             if not md5_hash:
                 blob, md5_hash = self._try_create_md5_content_hash(blob)
             if blob and md5_hash and md5_hash != self.blob_cache.get(blob.name):
@@ -191,7 +191,7 @@ def _try_create_md5_content_hash(self, blob):
             # Re-fetch the blob with the new hash
             hashed_blob = client.get_blob_properties(container, blob.name)
 
-            return hashed_blob, hashed_blob.properties.content_settings.content_md5
+            return hashed_blob, hashed_blob.content_settings.content_md5
         except AzureHttpError as e:
             log.warning("Failed to apply a md5 content hash to policy {}. "
                         "This policy will be skipped.".format(blob.name))

tools/c7n_azure/c7n_azure/filters.py

@@ -12,12 +12,11 @@
                                               QueryDataset, QueryDefinition,
                                               QueryFilter, QueryGrouping,
                                               QueryTimePeriod, TimeframeType)
-from azure.mgmt.policyinsights import PolicyInsightsClient
 from c7n_azure.tags import TagHelper
 from c7n_azure.utils import (IpRangeHelper, Math, ResourceIdParser,
                              StringUtils, ThreadHelper, now, utcnow, is_resource_group)
 from dateutil.parser import parse
-from msrest.exceptions import HttpOperationError
+from azure.core.exceptions import HttpResponseError
 
 from c7n.filters import Filter, FilterValidationError, ValueFilter
 from c7n.filters.core import PolicyValidationError
@@ -186,7 +185,7 @@ def get_metric_data(self, resource):
                 aggregation=self.aggregation,
                 filter=self.get_filter(resource)
             )
-        except HttpOperationError:
+        except HttpResponseError:
             self.log.exception("Could not get metric: %s on %s" % (
                 self.metric, resource['id']))
             return None
@@ -484,7 +483,7 @@ def process(self, resources, event=None):
                               d.name in self.definitions]
 
         # Find non-compliant resources
-        client = PolicyInsightsClient(s.get_credentials())
+        client = s.client('azure.mgmt.policyinsights.PolicyInsightsClient')
         query = client.policy_states.list_query_results_for_subscription(
             policy_states_resource='latest', subscription_id=s.subscription_id).value
         non_compliant = [f.resource_id.lower() for f in query
@@ -821,13 +820,11 @@ class CostFilter(ValueFilter):
 
         - ``WeekToDate``
         - ``MonthToDate``
-        - ``YearToDate``
 
       - All days in the previous calendar period:
 
-        - ``TheLastWeek``
         - ``TheLastMonth``
-        - ``TheLastYear``
+        - ``TheLastBillingMonth``
 
     :examples:
 
@@ -925,7 +922,7 @@ def _query_costs(self):
 
         client = manager.get_client('azure.mgmt.costmanagement.CostManagementClient')
 
-        aggregation = {'totalCost': QueryAggregation(name='PreTaxCost')}
+        aggregation = {'totalCost': QueryAggregation(name='PreTaxCost', function='Sum')}
 
         grouping = [QueryGrouping(type='Dimension',
                                   name='ResourceGroupName' if is_resource_group else 'ResourceId')]
@@ -950,22 +947,24 @@ def _query_costs(self):
             timeframe = 'Custom'
             time_period = QueryTimePeriod(from_property=start_time, to=end_time)
 
-        definition = QueryDefinition(timeframe=timeframe, time_period=time_period, dataset=dataset)
+        definition = QueryDefinition(type='ActualCost',
+                                     timeframe=timeframe,
+                                     time_period=time_period,
+                                     dataset=dataset)
 
         subscription_id = manager.get_session().get_subscription_id()
 
         scope = '/subscriptions/' + subscription_id
 
-        query = client.query.usage_by_scope(scope, definition)
+        query = client.query.usage(scope, definition)
 
         if hasattr(query, '_derserializer'):
             original = query._derserializer._deserialize
             query._derserializer._deserialize = lambda target, data: \
                 original(target, self.fix_wrap_rest_response(data))
 
-        result_list = list(query)[0]
-        result_list = [{result_list.columns[i].name: v for i, v in enumerate(row)}
-                       for row in result_list.rows]
+        result_list = [{query.columns[i].name: v for i, v in enumerate(row)}
+                       for row in query.rows]
 
         for r in result_list:
             if 'ResourceGroupName' in r:

tools/c7n_azure/c7n_azure/function_package.py

@@ -78,11 +78,13 @@ def _add_functions_required_files(
                 auth_contents = s.get_functions_auth_string(target_sub_id)
             elif identity['type'] == AUTH_TYPE_MSI:
                 auth_contents = json.dumps({
-                    'use_msi': True, 'subscription_id': target_sub_id})
+                    'use_msi': True, 'subscription_id': target_sub_id,
+                    'tenant_id': s.get_tenant_id()})
             elif identity['type'] == AUTH_TYPE_UAI:
                 auth_contents = json.dumps({
                     'use_msi': True, 'subscription_id': target_sub_id,
-                    'client_id': identity['client_id']})
+                    'client_id': identity['client_id'],
+                    'tenant_id': s.get_tenant_id()})
 
             self.pkg.add_contents(dest=name + '/auth.json', contents=auth_contents)
             self.pkg.add_file(self.function_path,

tools/c7n_azure/c7n_azure/functionapp_utils.py

@@ -29,7 +29,7 @@ def get_storage_account_connection_string(id):
         rg_name = ResourceIdParser.get_resource_group(id)
         name = ResourceIdParser.get_resource_name(id)
         client = local_session(Session).client('azure.mgmt.storage.StorageManagementClient')
-        obj = client.storage_accounts.list_keys(rg_name, name)
+        obj = client.storage_accounts.list_keys(rg_name, name, expand=None)
 
         connection_string = 'DefaultEndpointsProtocol={};AccountName={};AccountKey={}'.format(
             'https',
@@ -102,7 +102,7 @@ def publish_functions_package(cls, function_params, package):
 
         cls.log.info('Publishing Function application')
 
-        publish_creds = web_client.web_apps.list_publishing_credentials(
+        publish_creds = web_client.web_apps.begin_list_publishing_credentials(
             function_params.function_app['resource_group_name'],
             function_params.function_app['name']).result()
 

tools/c7n_azure/c7n_azure/handler.py

@@ -8,6 +8,7 @@
 
 from azure.common import AzureHttpError
 from msrestazure.azure_exceptions import CloudError
+from azure.core.exceptions import AzureError
 
 from c7n.utils import reset_session_cache
 from c7n.config import Config
@@ -53,7 +54,7 @@ def run(event, context, subscription_id=None):
         for p in policies:
             try:
                 p.push(event, context)
-            except (CloudError, AzureHttpError) as error:
+            except (CloudError, AzureHttpError, AzureError) as error:
                 log.error("Unable to process policy: %s :: %s" % (p.name, error))
 
     reset_session_cache()

tools/c7n_azure/c7n_azure/provisioning/app_insights.py

@@ -1,6 +1,6 @@
 # Copyright The Cloud Custodian Authors.
 # SPDX-License-Identifier: Apache-2.0
-from msrestazure.azure_exceptions import CloudError
+from azure.core.exceptions import ResourceNotFoundError
 
 from c7n_azure.provisioning.deployment_unit import DeploymentUnit
 from c7n_azure.provisioning.resource_group import ResourceGroupUnit
@@ -16,7 +16,7 @@ def __init__(self):
     def _get(self, params):
         try:
             return self.client.components.get(params['resource_group_name'], params['name'])
-        except CloudError:
+        except ResourceNotFoundError:
             return None
 
     def _provision(self, params):