OPS-15474 Automate APM alerts creation (#191)

* Automate APM alerts creation

* Add override possibility for NewRelic APM

efopen/newrelic_executor.py

@@ -17,6 +17,7 @@ def __init__(self, context, clients):
     self.ec2_conditions = self.config.get('ec2_alert_conditions', {})
     self.ecs_conditions = self.config.get('ecs_alert_conditions', {})
     self.local_alert_nrql_conditions = self.config.get('alert_nrql_conditions', {})
+    self.local_alert_apm_conditions = self.config.get('apm_metric_alert_conditions', {})
     self.admin_token = self.config.get('admin_token', "")
     self.all_notification_channels = self.config.get('env_notification_map', {})
     self.opsgenie_api_key = self.config["opsgenie_api_key"]
@@ -58,6 +59,8 @@ def populate_alert_policy_values(self, policy, service_type):
       policy.config_conditions = deepcopy(self.ecs_conditions)
     policy.remote_alert_nrql_conditions = self.newrelic.get_policy_alert_nrql_conditions(policy.id)
     policy.local_alert_nrql_conditions = deepcopy(self.local_alert_nrql_conditions)
+    policy.remote_alert_apm_conditions = self.newrelic.get_policy_alert_apm_conditions(policy.id)
+    policy.local_alert_apm_conditions = deepcopy(self.local_alert_apm_conditions)
 
   def delete_conditions_not_matching_config_values(self, policy):
     # Remove conditions with values that differ from config
@@ -201,6 +204,34 @@ def update_alert_nrql_condition_if_different(self, local_alert_nrql_condition, p
           logger.debug("Local: {}\nRemote: {}".format(local_alert_nrql_condition, remote_alert_nrql_condition))
           self.newrelic.put_policy_alert_nrql_condition(remote_alert_nrql_condition["id"], local_alert_nrql_condition)
 
+  def update_alert_apm_condition_if_different(self, local_alert_apm_condition, policy):
+    for remote_alert_apm_condition in policy.remote_alert_apm_conditions:
+      if remote_alert_apm_condition["name"] == local_alert_apm_condition["name"]:
+        # Add fields that exist in the remote alert condition object but not in the local alert condition object.
+        # This is done so that we can test equality.
+        local_alert_apm_condition['id'] = remote_alert_apm_condition['id']
+        local_alert_apm_condition['type'] = remote_alert_apm_condition['type']
+        local_alert_apm_condition['entities'] = remote_alert_apm_condition['entities']
+
+        if local_alert_apm_condition != remote_alert_apm_condition:
+          logger.info("Local alert apm condition differs from remote alert apm condition for {}-{}. Updating remote.".format(policy.env,policy.service))
+          logger.debug("Local: {}\nRemote: {}".format(local_alert_apm_condition, remote_alert_apm_condition))
+          self.newrelic.put_policy_alert_apm_condition(remote_alert_apm_condition["id"], local_alert_apm_condition)
+
+  def override_alert_apm_condition_values(self, policy, service_alert_overrides):
+    # Update policy.config_conditions with overrides from service_registry
+    for condition_name, override_obj in service_alert_overrides.items():
+      if condition_name in policy.local_alert_apm_conditions.keys():
+        for override_key, override_value in override_obj.items():
+          if isinstance(override_value, dict):
+            for inner_key, inner_val in override_value.items():
+              policy.local_alert_apm_conditions[condition_name][override_key][inner_key] = inner_val
+          else:
+            policy.local_alert_apm_conditions[condition_name][override_key] = override_value
+    logger.debug("Policy {} APM alert condition values:\n{}".format(policy.name, policy.local_alert_apm_conditions))
+
+    return policy
+
   def update_application_services_policies(self):
     for service_name, service_config in self.context.service_registry.iter_services(service_group="application_services"):
       service_environments = service_config['environments']
@@ -238,6 +269,22 @@ def update_application_services_policies(self):
           else:
             self.update_alert_nrql_condition_if_different(condition_value, policy)
 
+        # APM alert conditions
+        policy = self.override_alert_apm_condition_values(policy, service_alert_overrides)
+        remote_alert_apm_condition_names = [remote_alert_apm_condition['name'] for remote_alert_apm_condition in policy.remote_alert_apm_conditions]
+        for condition_name, condition_value in policy.local_alert_apm_conditions.items():
+          if condition_name not in remote_alert_apm_condition_names:
+            applications = self.newrelic.get_applications(application_name=policy.name)
+
+            if not len(applications):
+              logger.info('No applications hosted for this policy. Skip creating any APM alert')
+              continue
+
+            condition_value['entities'] = [applications[0]['id']]
+            self.newrelic.create_alert_apm_condition(policy.id, condition_value)
+          else:
+            self.update_alert_apm_condition_if_different(condition_value, policy)
+
   def run(self):
     if self.context.env in self.all_notification_channels.keys():
       if self.config['token_kms_encrypted']:

efopen/newrelic_interface.py

@@ -18,6 +18,8 @@ def __init__(self, env, service):
     self.remote_conditions = None
     self.local_alert_nrql_conditions = None
     self.remote_alert_nrql_conditions = None
+    self.local_alert_apm_conditions = None
+    self.remote_alert_apm_conditions = None
     self.symbols = None
     self.set_name()
     self.set_symbols()
@@ -206,6 +208,39 @@ def put_policy_alert_nrql_condition(self, condition_id, condition):
     put_condition.raise_for_status()
     return
 
+
+  def create_alert_apm_condition(self, policy_id, condition):
+    add_condition = requests.post(
+      url='https://api.newrelic.com/v2/alerts_conditions/policies/{}.json'.format(policy_id),
+      headers=self.auth_header,
+      data=json.dumps({"condition": condition})
+    )
+    add_condition.raise_for_status()
+    return
+
+  def get_policy_alert_apm_conditions(self, policy_id):
+    return self.iterative_get(
+      endpoint_url='https://api.newrelic.com/v2/alerts_conditions.json',
+      response_key='conditions',
+      params={'policy_id': policy_id}
+    )
+
+  def put_policy_alert_apm_condition(self, condition_id, condition):
+    put_condition = requests.put(
+      url='https://api.newrelic.com/v2/alerts_conditions/{}.json'.format(condition_id),
+      headers=self.auth_header,
+      data=json.dumps({"condition": condition})
+    )
+    put_condition.raise_for_status()
+    return
+
+  def get_applications(self, application_name=''):
+    return self.iterative_get(
+      endpoint_url='https://api.newrelic.com/v2/applications.json',
+      response_key='applications',
+      params={'filter[name]': application_name}
+    )
+
   def create_alert_channel(self, name, channel_type, configuration):
     create_channel = requests.post(
       url='https://api.newrelic.com/v2/alerts_channels.json',