[Cortex XDR - IR] Fix incidents not pulled (demisto#35696)

* init

* minor changes

* remove ctf

* Revert "remove ctf"

This reverts commit 28d4585.

revert

* revert

* more tests

* UTs complete

* RN

* Bump pack from version CortexXDR to 6.1.60.

* build fixes

---------

Co-authored-by: Content Bot <[email protected]>

Packs/CortexXDR/Integrations/CortexXDRIR/CortexXDRIR.py

@@ -1059,14 +1059,17 @@ def update_related_alerts(client: Client, args: dict):
         return_results(update_alerts_in_xdr_command(client, args_for_command))
 
 
-def fetch_incidents(client, first_fetch_time, integration_instance, exclude_artifacts: bool, last_run: dict = None,
+def fetch_incidents(client, first_fetch_time, integration_instance, exclude_artifacts: bool, last_run: dict,
                     max_fetch: int = 10, statuses: List = [], starred: Optional[bool] = None,
                     starred_incidents_fetch_window: str = None):
     global ALERTS_LIMIT_PER_INCIDENTS
     # Get the last fetch time, if exists
-    last_fetch = last_run.get('time') if isinstance(last_run, dict) else None
-    incidents_from_previous_run = last_run.get('incidents_from_previous_run', []) if isinstance(last_run,
-                                                                                                dict) else []
+    last_fetch = last_run.get('time')
+    incidents_from_previous_run = last_run.get('incidents_from_previous_run', [])
+
+    incidents_at_last_timestamp = set(last_run.get('incidents_at_last_timestamp', []))
+    new_incidents_at_last_timestamp = []
+
     # Handle first time fetch, fetch incidents retroactively
     if last_fetch is None:
         last_fetch, _ = parse_date_range(first_fetch_time, to_timestamp=True)
@@ -1077,7 +1080,7 @@ def fetch_incidents(client, first_fetch_time, integration_instance, exclude_arti
     incidents = []
     if incidents_from_previous_run:
         raw_incidents = incidents_from_previous_run
-        ALERTS_LIMIT_PER_INCIDENTS = last_run.get('alerts_limit_per_incident', -1) if isinstance(last_run, dict) else -1
+        ALERTS_LIMIT_PER_INCIDENTS = last_run.get('alerts_limit_per_incident', -1)
     else:
         if statuses:
             raw_incidents = []
@@ -1109,29 +1112,34 @@ def fetch_incidents(client, first_fetch_time, integration_instance, exclude_arti
         for raw_incident in raw_incidents:
             incident_data: dict[str, Any] = sort_incident_data(raw_incident) if raw_incident.get('incident') else raw_incident
             incident_id = incident_data.get('incident_id')
+            if incident_id in incidents_at_last_timestamp:  # remove duplicates
+                demisto.debug(f'incident {incident_id!r} is a duplicate, skipping. {incident_data=}')
+                continue
             alert_count = arg_to_number(incident_data.get('alert_count')) or 0
             if alert_count > ALERTS_LIMIT_PER_INCIDENTS:
                 demisto.debug(f'for incident:{incident_id} using the old call since alert_count:{alert_count} >" \
                               "limit:{ALERTS_LIMIT_PER_INCIDENTS}')
                 raw_incident_ = client.get_incident_extra_data(incident_id=incident_id)
                 incident_data = sort_incident_data(raw_incident_)
             sort_all_list_incident_fields(incident_data)
-            incident_data['mirror_direction'] = MIRROR_DIRECTION.get(demisto.params().get('mirror_direction', 'None'),
-                                                                     None)
+            incident_data['mirror_direction'] = MIRROR_DIRECTION.get(demisto.params().get('mirror_direction', 'None'))
             incident_data['mirror_instance'] = integration_instance
             incident_data['last_mirrored_in'] = int(datetime.now().timestamp() * 1000)
             description = incident_data.get('description')
             occurred = timestamp_to_datestring(incident_data['creation_time'], TIME_FORMAT + 'Z')
-            incident: Dict[str, Any] = {
+            incident: dict[str, Any] = {
                 'name': f'XDR Incident {incident_id} - {description}',
                 'occurred': occurred,
                 'rawJSON': json.dumps(incident_data),
             }
             if demisto.params().get('sync_owners') and incident_data.get('assigned_user_mail'):
-                incident['owner'] = demisto.findUser(email=incident_data.get('assigned_user_mail')).get('username')
+                incident['owner'] = demisto.findUser(email=incident_data['assigned_user_mail']).get('username')
             # Update last run and add incident if the incident is newer than last fetch
             if incident_data.get('creation_time', 0) > last_fetch:
                 last_fetch = incident_data['creation_time']
+                new_incidents_at_last_timestamp = [incident_id]
+            elif incident_data.get('creation_time') == last_fetch:
+                new_incidents_at_last_timestamp.append(incident_id)
             incidents.append(incident)
             non_created_incidents.remove(raw_incident)
 
@@ -1152,7 +1160,8 @@ def fetch_incidents(client, first_fetch_time, integration_instance, exclude_arti
     else:
         next_run['incidents_from_previous_run'] = []
 
-    next_run['time'] = last_fetch + 1
+    next_run['time'] = last_fetch
+    next_run['incidents_at_last_timestamp'] = new_incidents_at_last_timestamp
 
     return next_run, incidents
 
@@ -1331,7 +1340,7 @@ def main():  # pragma: no cover
                                                   first_fetch_time=first_fetch_time,
                                                   integration_instance=integration_instance,
                                                   exclude_artifacts=exclude_artifacts,
-                                                  last_run=demisto.getLastRun().get('next_run'),
+                                                  last_run=demisto.getLastRun().get('next_run') or {},
                                                   max_fetch=max_fetch,
                                                   statuses=statuses,
                                                   starred=starred,

Packs/CortexXDR/Integrations/CortexXDRIR/CortexXDRIR_test.py

@@ -72,7 +72,7 @@ def test_fetch_incidents(requests_mock, mocker):
         modified_raw_incident.get('alerts')[0].get('host_ip').split(',')
     mocker.patch("CortexXDRIR.ALERTS_LIMIT_PER_INCIDENTS", new=50)
     mocker.patch.object(Client, 'save_modified_incidents_to_integration_context')
-    next_run, incidents = fetch_incidents(client, '3 month', 'MyInstance', exclude_artifacts=False)
+    next_run, incidents = fetch_incidents(client, '3 month', 'MyInstance', exclude_artifacts=False, last_run={})
     sort_all_list_incident_fields(modified_raw_incident)
     assert len(incidents) == 1
     assert incidents[0]['name'] == "XDR Incident 1 - desc1"
@@ -104,7 +104,8 @@ def test_fetch_incidents_filtered_by_status(requests_mock, mocker):
     mocker.patch.object(Client, 'save_modified_incidents_to_integration_context')
     statuses_to_fetch = ['under_investigation', 'new']
 
-    next_run, incidents = fetch_incidents(client, '3 month', 'MyInstance', exclude_artifacts=False, statuses=statuses_to_fetch)
+    next_run, incidents = fetch_incidents(
+        client, '3 month', 'MyInstance', exclude_artifacts=False, statuses=statuses_to_fetch, last_run={})
 
     assert len(incidents) == 2
     assert incidents[0]['name'] == "XDR Incident 1 - 'Local Analysis Malware' generated by XDR Agent detected on host AAAAAA " \
@@ -143,7 +144,7 @@ def test_fetch_incidents_with_rate_limit_error(requests_mock, mocker):
     client = Client(
         base_url=f'{XDR_URL}/public_api/v1', verify=False, timeout=120, proxy=False)
     with pytest.raises(Exception) as e:
-        next_run, incidents = fetch_incidents(client, '3 month', 'MyInstance', exclude_artifacts=False)
+        next_run, incidents = fetch_incidents(client, '3 month', 'MyInstance', exclude_artifacts=False, last_run={})
     assert str(e.value) == 'Rate limit exceeded'
 
 
@@ -213,7 +214,7 @@ def test_fetch_only_starred_incidents(self, mocker):
         client = Client(
             base_url=f'{XDR_URL}/public_api/v1', verify=False, timeout=120, proxy=False)
         next_run, incidents = fetch_incidents(client, '3 month', 'MyInstance', exclude_artifacts=False,
-                                              last_run=last_run_obj.get('next_run'),
+                                              last_run=last_run_obj.get('next_run') or {},
                                               starred=True,
                                               starred_incidents_fetch_window='3 days')
         assert len(incidents) == 2
@@ -742,7 +743,7 @@ def test_fetch_incidents_extra_data(requests_mock, mocker):
     mocker.patch.object(Client, 'save_modified_incidents_to_integration_context')
     mocker.patch.object(Client, 'save_modified_incidents_to_integration_context')
     mocker.patch("CortexXDRIR.ALERTS_LIMIT_PER_INCIDENTS", new=2)
-    next_run, incidents = fetch_incidents(client, '3 month', 'MyInstance', exclude_artifacts=False)
+    next_run, incidents = fetch_incidents(client, '3 month', 'MyInstance', exclude_artifacts=False, last_run={})
     assert len(incidents) == 2
     assert incidents[0]['name'] == 'XDR Incident 1 - desc1'
     assert json.loads(incidents[0]['rawJSON']).get('incident_id') == '1'
@@ -1580,3 +1581,72 @@ def test_get_xsoar_close_reasons(mocker):
     }
     mocker.patch.object(demisto, 'internalHttpRequest', return_value=mock_response)
     assert get_xsoar_close_reasons() == list(XSOAR_RESOLVED_STATUS_TO_XDR.keys()) + ['CustomReason1', 'CustomReason 2', 'Foo']
+
+
+@freeze_time('1970-01-01 00:00:00.100')
+def test_fetch_incidents_dedup():
+    """
+    Unit test to verify that incidents that occur in the same instant are not not missed or duplicated.
+
+    Given:
+        - Two incidents occur in the same instant.
+    When:
+        - Fetching incidents.
+    Then:
+        - Assert no incidents are missed or duplicated.
+    """
+    from CortexXDRIR import fetch_incidents
+
+    last_run = {}
+
+    class MockClient:
+
+        incidents = load_test_data('./test_data/get_incidents_list_dedup.json')
+
+        def save_modified_incidents_to_integration_context(self): ...
+
+        def get_multiple_incidents_extra_data(self, gte_creation_time_milliseconds=0, limit=100, **_):
+            return [
+                inc for inc in self.incidents
+                if inc['creation_time'] >= gte_creation_time_milliseconds
+            ][:limit]
+
+    mock_client = MockClient()
+
+    last_run, result_1 = fetch_incidents(
+        client=mock_client,
+        first_fetch_time='3 days',
+        integration_instance={},
+        exclude_artifacts=True,
+        last_run=last_run,
+        max_fetch=2,
+    )
+
+    assert 'XDR Incident 1' in result_1[0]['name']
+    assert 'XDR Incident 2' in result_1[1]['name']
+    assert last_run['incidents_at_last_timestamp'] == ['2']
+
+    last_run, result_2 = fetch_incidents(
+        client=mock_client,
+        first_fetch_time='3 days',
+        integration_instance={},
+        exclude_artifacts=True,
+        last_run=last_run,
+        max_fetch=3,
+    )
+
+    assert 'XDR Incident 3' in result_2[0]['name']
+    assert 'XDR Incident 4' in result_2[1]['name']
+    assert last_run['incidents_at_last_timestamp'] == ['4']
+
+    # run empty test and assert last_run['incidents_at_last_timestamp'] is empty
+    last_run, result_2 = fetch_incidents(
+        client=mock_client,
+        first_fetch_time='3 days',
+        integration_instance={},
+        exclude_artifacts=True,
+        last_run=last_run,
+        max_fetch=0,
+    )
+
+    assert last_run['incidents_at_last_timestamp'] == []

Packs/CortexXDR/Integrations/CortexXDRIR/test_data/get_incidents_list_dedup.json

@@ -0,0 +1,94 @@
+[
+    {
+        "incident_id": "1",
+        "creation_time": 100000000,
+        "modification_time": 1575813875168,
+        "detection_time": null,
+        "status": "under_investigation",
+        "severity": "medium",
+        "description": "'Local Analysis Malware' generated by XDR Agent detected on host AAAAA involving user Administrator",
+        "assigned_user_mail": null,
+        "assigned_user_pretty_name": null,
+        "alert_count": 1,
+        "low_severity_alert_count": 0,
+        "med_severity_alert_count": 1,
+        "high_severity_alert_count": 0,
+        "user_count": 1,
+        "host_count": 1,
+        "notes": null,
+        "resolve_comment": null,
+        "manual_severity": null,
+        "manual_description": null,
+        "xdr_url": "https://demisto.hello.com/incident-view/1",
+        "starred": false
+    },
+    {
+        "incident_id": "2",
+        "creation_time": 100000001,
+        "modification_time": 1575813875168,
+        "detection_time": null,
+        "status": "new",
+        "severity": "high",
+        "description": "'Local Analysis Malware' generated by XDR Agent detected on host BBBBB involving user Administrator",
+        "assigned_user_mail": null,
+        "assigned_user_pretty_name": null,
+        "alert_count": 1,
+        "low_severity_alert_count": 0,
+        "med_severity_alert_count": 1,
+        "high_severity_alert_count": 0,
+        "user_count": 1,
+        "host_count": 1,
+        "notes": null,
+        "resolve_comment": null,
+        "manual_severity": null,
+        "manual_description": null,
+        "xdr_url": "https://demisto.hello.com/incident-view/2",
+        "starred": false
+    },
+    {
+        "incident_id": "3",
+        "creation_time": 100000001,
+        "modification_time": 1575813875168,
+        "detection_time": null,
+        "status": "under_investigation",
+        "severity": "medium",
+        "description": "'Local Analysis Malware' generated by XDR Agent detected on host AAAAA involving user Administrator",
+        "assigned_user_mail": null,
+        "assigned_user_pretty_name": null,
+        "alert_count": 1,
+        "low_severity_alert_count": 0,
+        "med_severity_alert_count": 1,
+        "high_severity_alert_count": 0,
+        "user_count": 1,
+        "host_count": 1,
+        "notes": null,
+        "resolve_comment": null,
+        "manual_severity": null,
+        "manual_description": null,
+        "xdr_url": "https://demisto.hello.com/incident-view/1",
+        "starred": false
+    },
+    {
+        "incident_id": "4",
+        "creation_time": 100000002,
+        "modification_time": 1575813875168,
+        "detection_time": null,
+        "status": "new",
+        "severity": "high",
+        "description": "'Local Analysis Malware' generated by XDR Agent detected on host BBBBB involving user Administrator",
+        "assigned_user_mail": null,
+        "assigned_user_pretty_name": null,
+        "alert_count": 1,
+        "low_severity_alert_count": 0,
+        "med_severity_alert_count": 1,
+        "high_severity_alert_count": 0,
+        "user_count": 1,
+        "host_count": 1,
+        "notes": null,
+        "resolve_comment": null,
+        "manual_severity": null,
+        "manual_description": null,
+        "xdr_url": "https://demisto.hello.com/incident-view/2",
+        "starred": false
+    }
+]

Packs/CortexXDR/ReleaseNotes/6_1_60.md

@@ -0,0 +1,6 @@
+
+#### Integrations
+
+##### Palo Alto Networks Cortex XDR - Investigation and Response
+
+- Fixed an issue in which multiple incidents created in the same instant would not all be fetched. 

Packs/CortexXDR/pack_metadata.json

@@ -2,7 +2,7 @@
     "name": "Cortex XDR by Palo Alto Networks",
     "description": "Automates Cortex XDR incident response, and includes custom Cortex XDR incident views and layouts to aid analyst investigations.",
     "support": "xsoar",
-    "currentVersion": "6.1.59",
+    "currentVersion": "6.1.60",
     "author": "Cortex XSOAR",
     "url": "https://www.paloaltonetworks.com/cortex",
     "email": "",