From a9df147de10183d7abf531e71b2f8cced7244a0d Mon Sep 17 00:00:00 2001 From: "gapanovichal@gmail.com" Date: Mon, 8 Apr 2024 21:22:34 -0700 Subject: [PATCH 01/10] Clean Automative --- rss_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rss_data.py b/rss_data.py index 7c2cd80..619914b 100644 --- a/rss_data.py +++ b/rss_data.py @@ -89,7 +89,6 @@ "apple", "arin", "aruba", - "autoptimize", "aws", "azure", "beyond trust", From 997dfed54ac0dec15759087e3183f12ac93ef8a6 Mon Sep 17 00:00:00 2001 From: "gapanovichal@gmail.com" Date: Tue, 9 Apr 2024 16:20:01 -0700 Subject: [PATCH 02/10] Clean filter lis for Pharma --- rss_data.py | 97 ++++++----------------------------------------------- 1 file changed, 10 insertions(+), 87 deletions(-) diff --git a/rss_data.py b/rss_data.py index 619914b..263d427 100644 --- a/rss_data.py +++ b/rss_data.py @@ -77,101 +77,24 @@ } keywords = { - "last_modified": "2024-03-18", + "last_modified": "2024-04-09", "ignored": [ "hiring" ], "static_keywords": [ - "adobe", - "airmagnet survey ", - "amazon", - "android", - "apple", - "arin", - "aruba", + "365", "aws", - "azure", - "beyond trust", - "bomgar", - "centos", - "centrify", - "checkmk", - "chrome", - "cisco", - "code42", - "connectwise", - "crash plan", - "debian", - "defender", - "dell", - "designcad", - "digicert", - "docusign", + "ms365 ", + "one drive", + "sharepoint ", + "defender ", "duo", - "fortianalyzer", - "forticloud", - "fortigate", - "fortimanager", - "fortinet", - "fortiswitch", - "google", - "global protect", - "hp", - "hubspot", - "idaptive", - "infoblox", - "ingram micro", - "ipad", - "iphone", - "iris", - "it glue", - "ivanti", - "jamf", - "juniper ", - "jwt", - "knowbe4", - "lenovo", - "linux", - "lucid", - "macbook", - "macos", - "meraki", + "lastpass", + "unifi", + "intune", "microsoft", - "mozilla firefox", - "defender", - "office", - "nessus", - "netapp", - "notepad++", - "office365", - "okta", - "palo alto", - "parallels", - "paycor", - "pluralsight", - "pulse secure", - "putty", - "pycharm", - "rapidfire", - "ricoh", - "rubrick", - "salesforce", - "secret server", - "sentinel", - "slack", - "sonicwall", - "supermicro", - "synnex", - "ubuntu", - "vcenter", - "veeam", - "vmware", - "vsphere", - "webex", - "wiline", "windows", "windows 10", - "windows 11", - "zoom" + "windows 11" ] } From 74ffc9fe68bb194210106d53b89d6976231e6271 Mon Sep 17 00:00:00 2001 From: "gapanovichal@gmail.com" Date: Tue, 9 Apr 2024 16:52:34 -0700 Subject: [PATCH 03/10] Fixed syphon_pharma_prod in branches --- .github/workflows/azure-functions-app-python.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/azure-functions-app-python.yml b/.github/workflows/azure-functions-app-python.yml index e046931..8dc624f 100644 --- a/.github/workflows/azure-functions-app-python.yml +++ b/.github/workflows/azure-functions-app-python.yml @@ -20,7 +20,7 @@ name: Deploy Python project to Azure Function App on: push: - branches: ["main", "development"] + branches: ["main", "development", "syphon_pharma_prod"] env: AZURE_FUNCTIONAPP_NAME: 'RSS-syphon' # set this to your function app name on Azure From 4ab24b59617039212d973f0ec4244bceb05defd3 Mon Sep 17 00:00:00 2001 From: "gapanovichal@gmail.com" Date: Thu, 11 Apr 2024 14:41:16 -0700 Subject: [PATCH 04/10] Slacker to pharma --- slacker/slacker.py | 65 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 45 insertions(+), 20 deletions(-) diff --git a/slacker/slacker.py b/slacker/slacker.py index 606e200..10b5ec6 100644 --- a/slacker/slacker.py +++ b/slacker/slacker.py @@ -22,7 +22,10 @@ def init_slack_client(slack_token): return WebClient(token=slack_token) -def read_channel(client, channel_id, rss_type): +def read_channel(client, channel_id, rss_type, pages_to_read: str) -> dict: + + + """ Reads channel conversations and returns matching content @@ -52,24 +55,33 @@ def read_channel(client, channel_id, rss_type): } try: - # Call the conversations.history method using the WebClient - # The conversations.history returns 99 messages by default - # Results are paginated, see: https://api.slack.com/method/conversations.history$pagination - # TODO handle paginating multiple pages + # Convert pages_to_read to an integer if it's not already + + conversation_history = [] result = client.conversations_history(channel=channel_id) - conversation_history = result["messages"] - - # Initialize dict and lists for storing links/md5s + conversation_history.extend(result.get("messages", [])) + + # Initialize next_cursor safely + next_cursor = result.get("response_metadata", {}).get("next_cursor") + + while next_cursor and pages_to_read > 0: # Check if next_cursor is not empty and pages_to_read > 0 + result = client.conversations_history(channel=channel_id, cursor=next_cursor) + conversation_history.extend(result.get("messages", [])) + pages_to_read -= 1 + # Safely update next_cursor + next_cursor = result.get("response_metadata", {}).get("next_cursor") + + # Process extracted messages to find links and MD5 hashes re_link = [] - link_regex = r"(?:link\:.+?)(https?:\/\/(?:www\.)?[-a-zA-Z-1-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b(?:[-a-zA-Z0-9()@:%_\+.~#?&\/=]*))" - re_results = re.findall(link_regex, str(conversation_history), re.IGNORECASE) + link_regex = re.compile(r"(?:link\:.+?)(https?:\/\/(?:www\.)?[-a-zA-Z-1-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b(?:[-a-zA-Z0-9()@:%_\+.~#?&\/=]*))", re.IGNORECASE) + re_results = link_regex.findall(str(conversation_history)) for re_result in re_results: if re_result not in re_link: re_link.append(re_result) re_md5 = [] - md5_regex = r"(?:md5:\s)([a-f0-9]{32})" - re_results = re.findall(md5_regex, str(conversation_history), re.IGNORECASE) + md5_regex = re.compile(r"(?:md5:\s)([a-f0-9]{32})", re.IGNORECASE) + re_results = md5_regex.findall(str(conversation_history)) for re_result in re_results: if re_result not in re_md5: re_md5.append(re_result) @@ -77,19 +89,19 @@ def read_channel(client, channel_id, rss_type): already_fixed_list = [] already_seen_list = [] - # Save timestamp if cve - if rss_type == "cve": - cve_regex = r"(CVE-20[0-9]{2}-\d+)" + # Save data if job type is CVE + if rss_type.lower() == "cve": + cve_regex = re.compile(r"(CVE-20[0-9]{2}-\d+)", re.IGNORECASE) for dialog in conversation_history: if "reactions" in dialog: - if list(filter(lambda item: item['name'] == 'white_check_mark', dialog["reactions"])): - cve_dialog_results = re.findall(cve_regex, str(dialog), re.IGNORECASE) + if any(item['name'] == 'white_check_mark' for item in dialog.get("reactions", [])): + cve_dialog_results = cve_regex.findall(str(dialog)) for dialog_result in cve_dialog_results: if dialog_result not in already_fixed_list: already_fixed_list.append(dialog_result) - cve_convo_results = re.findall(cve_regex, str(conversation_history), re.IGNORECASE) + cve_convo_results = cve_regex.findall(str(conversation_history)) for convo_result in cve_convo_results: if convo_result not in already_seen_list: already_seen_list.append(convo_result) @@ -102,8 +114,13 @@ def read_channel(client, channel_id, rss_type): } except SlackApiError as e: - msg = f"Error creating conversation: {e}" + msg = f"Error fetching conversation data: {e.response.get('error', 'Unknown error')}" logger.error(msg) + return re_dict + except KeyError as e: + # This catches missing keys in the result dictionary + logger.error(f"Key error: {e} - likely missing in the API response.") + return re_dict return re_dict @@ -240,6 +257,7 @@ def send_message(job_type, message_params, matched, errors, check_stale_keywords :param errors: List of feeds that have an error :param check_stale_keywords: None or date """ + # Check if module is enabled and bail out if not if str(message_params["slack_enabled"]).lower() == "false": logger.debug("Debug: Slack not enabled.") @@ -248,13 +266,20 @@ def send_message(job_type, message_params, matched, errors, check_stale_keywords slack_token = message_params["slack_token"] slack_channel = message_params["channels"] + try: + pages_to_read = int(message_params["pages_to_read"]) # Convert pages_to_read to integer + except ValueError: + logger.error(f"pages_to_read should be an integer, got: {message_params['pages_to_read']}") + return # Exit the function if conversion fails + + # Check if slack_token is set if slack_token: # Init Slack Client slack_client = init_slack_client(slack_token) # Pull RSS that was found already in channel - rss_found = read_channel(slack_client, slack_channel[job_type], job_type) + rss_found = read_channel(slack_client, slack_channel[job_type], job_type, pages_to_read) # Build the message that will be sent message_body = build_results_message(matched, rss_found, job_type) From 45f598973e44b3e6402a906f8410aab30f5aa351 Mon Sep 17 00:00:00 2001 From: "gapanovichal@gmail.com" Date: Thu, 11 Apr 2024 14:57:57 -0700 Subject: [PATCH 05/10] List updated --- rss_data.py | 84 +++++------------------------------------------------ 1 file changed, 8 insertions(+), 76 deletions(-) diff --git a/rss_data.py b/rss_data.py index 26dbac8..0f8aadf 100644 --- a/rss_data.py +++ b/rss_data.py @@ -92,85 +92,17 @@ "hiring" ], "static_keywords": [ - "amazon", - "android", - "apple", - "arin", - "aruba", + "365", "aws", - "azure", - "beyond trust", - "bomgar", - "centos", - "centrify", - "checkmk", - "chrome", - "cisco", - "code42", - "connectwise", - "defender", - "dell", - "designcad", - "digicert", - "docusign", + "ms365 ", + "one drive", + "sharepoint ", + "defender ", "duo", - "fortianalyzer", - "forticloud", - "fortigate", - "fortimanager", - "fortinet", - "fortiswitch", - "google", - "global protect", - "hp", - "hubspot", - "idaptive", - "infoblox", - "ingram micro", - "ipad", - "iphone", - "iris", - "it glue", - "ivanti", - "jamf", - "juniper ", - "knowbe4", - "lenovo", - "linux", - "lucid", - "macbook", - "macos", - "meraki", + "lastpass", + "unifi", + "intune", "microsoft", - "mozilla firefox", - "defender", - "office", - "nessus", - "netapp", - "office365", - "okta", - "palo alto", - "parallels", - "paycor", - "pulse secure", - "pycharm", - "rapidfire", - "ricoh", - "rubrick", - "salesforce", - "secret server", - "sentinel", - "slack", - "sonicwall", - "supermicro", - "synnex", - "ubuntu", - "vcenter", - "veeam", - "vmware", - "vsphere", - "webex", - "wiline", "windows", "windows 10", "windows 11" From 8fb33cbb17479b39d49ec21a2bcd9050849f5982 Mon Sep 17 00:00:00 2001 From: "gapanovichal@gmail.com" Date: Fri, 12 Apr 2024 14:53:30 -0700 Subject: [PATCH 06/10] Clean aws out --- rss_data.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/rss_data.py b/rss_data.py index 0f8aadf..25a9633 100644 --- a/rss_data.py +++ b/rss_data.py @@ -80,8 +80,6 @@ {"name": "seclists-full", "url": "https://seclists.org/rss/fulldisclosure.rss"}, {"name": "seclists-oss", "url": "https://seclists.org/rss/oss-sec.rss"}, {"name": "inthewild", "url": "https://raw.githubusercontent.com/gmatuz/inthewilddb/master/rss.xml"}, - {"name": "tenable", "url": "https://www.tenable.com/cve/feeds?sort=newest"}, - {"name": "tenable-updated", "url": "https://www.tenable.com/cve/feeds?sort=updated"}, {"name": "center-for-internet-security", "url": "https://www.cisecurity.org/feed/advisories"} ] } @@ -93,7 +91,6 @@ ], "static_keywords": [ "365", - "aws", "ms365 ", "one drive", "sharepoint ", From ef700cbc4cfae431dc2b6ceb9fe7a7a6b141b2de Mon Sep 17 00:00:00 2001 From: "gapanovichal@gmail.com" Date: Fri, 12 Apr 2024 15:01:59 -0700 Subject: [PATCH 07/10] Linux Audit cleaned out --- rss_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/rss_data.py b/rss_data.py index 25a9633..fadfba9 100644 --- a/rss_data.py +++ b/rss_data.py @@ -51,7 +51,6 @@ {"name": "Max Justicz", "url": "https://justi.cz/feed.xml"}, {"name": "Blog of Osanda", "url": "https://osandamalith.com/feed/"}, {"name": "The Exploit Laboratory", "url": "https://blog.exploitlab.net/feeds/posts/default"}, - {"name": "Linux Audit", "url": "https://linux-audit.com/feed/"}, {"name": "The Human Machine Interface", "url": "https://h0mbre.github.io/feed.xml"}, {"name": "Trail of Bits Blog", "url": "https://blog.trailofbits.com/feed/"}, {"name": "Exodus Intelligence", "url": "https://blog.exodusintel.com/feed/"}, From 411e3ef84b477ab3b5bf95f69358448a011b564c Mon Sep 17 00:00:00 2001 From: "gapanovichal@gmail.com" Date: Wed, 17 Apr 2024 11:11:51 -0700 Subject: [PATCH 08/10] ZDI Remocal --- rss_data.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/rss_data.py b/rss_data.py index fadfba9..fd23da9 100644 --- a/rss_data.py +++ b/rss_data.py @@ -72,8 +72,6 @@ {"name": "Windows Internals Blog", "url": "https://windows-internals.com/feed/"}, {"name": "nist-analyzed", "url": "https://nvd.nist.gov/feeds/xml/cve/misc/nvd-rss-analyzed.xml"}, {"name": "nist-upcoming", "url": "https://nvd.nist.gov/feeds/xml/cve/misc/nvd-rss.xml"}, - {"name": "zdi-upcoming", "url": "https://www.zerodayinitiative.com/rss/upcoming/"}, - {"name": "zdi-analyzed", "url": "https://www.zerodayinitiative.com/rss/published/"}, {"name": "vulners", "url": "https://vulners.com/rss.xml"}, {"name": "seclists-bugtraq", "url": "https://seclists.org/rss/bugtraq.rss"}, {"name": "seclists-full", "url": "https://seclists.org/rss/fulldisclosure.rss"}, From 644fec486fa4bb9211d6000a9b72577339015891 Mon Sep 17 00:00:00 2001 From: "gapanovichal@gmail.com" Date: Thu, 18 Apr 2024 14:00:30 -0700 Subject: [PATCH 09/10] Win was swiped out --- rss_data.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/rss_data.py b/rss_data.py index fd23da9..b7da979 100644 --- a/rss_data.py +++ b/rss_data.py @@ -96,8 +96,6 @@ "lastpass", "unifi", "intune", - "microsoft", - "windows", "windows 10", "windows 11" ] From 8569cd76e82cfec03779c2a306603856cb82564c Mon Sep 17 00:00:00 2001 From: "gapanovichal@gmail.com" Date: Fri, 7 Jun 2024 17:49:24 -0700 Subject: [PATCH 10/10] Revert "Update pagination function" This reverts commit 2dc8392c077ba3dfada973240c852907b22cd5a1. --- rss_config.py | 3 +-- rss_data.py | 26 ++++++++++---------------- slacker/slacker.py | 17 +++++++---------- 3 files changed, 18 insertions(+), 28 deletions(-) diff --git a/rss_config.py b/rss_config.py index aabef9e..fe248e4 100644 --- a/rss_config.py +++ b/rss_config.py @@ -15,8 +15,7 @@ "channels": { "cve": os.getenv("SLACK_CHANNEL_CVE"), "news": os.getenv("SLACK_CHANNEL_NEWS"), - "error": os.getenv("SLACK_CHANNEL_ERRORS"), - "pages_to_read": os.getenv("SLACK_PAGES_TO_READ", 5) + "error": os.getenv("SLACK_CHANNEL_ERRORS") } } diff --git a/rss_data.py b/rss_data.py index fc7588a..643cc6b 100644 --- a/rss_data.py +++ b/rss_data.py @@ -27,8 +27,7 @@ {"name": "rapid7", "url": "https://blog.rapid7.com/rss/"}, {"name": "checkpoint", "url": "https://research.checkpoint.com/feed/"}, {"name": "isc.sans.edu", "url": "https://isc.sans.edu/rssfeed_full.xml"}, - {"name": "msrc.microsoft", "url": "https://msrc.microsoft.com/blog/feed"}, - + {"name": "msrc.microsoft", "url": "https://msrc.microsoft.com/blog/feed"} ], "cve": [ {"name": "esecurityplanet", "url": "https://www.esecurityplanet.com/feed/"}, @@ -70,28 +69,18 @@ {"name": "Corelan Team", "url": "https://www.corelan.be/index.php/feed/"}, {"name": "NCC Group Research", "url": "https://research.nccgroup.com/feed/"}, {"name": "Alexander Popov", "url": "https://a13xp0p0v.github.io/feed.xml"}, - {"name": "Windows Internals Blog", "url": "https://windows-internals.com/feed/"}, - {"name": "nist-analyzed", "url": "https://nvd.nist.gov/feeds/xml/cve/misc/nvd-rss-analyzed.xml"}, - {"name": "nist-upcoming", "url": "https://nvd.nist.gov/feeds/xml/cve/misc/nvd-rss.xml"}, - {"name": "zdi-upcoming", "url": "https://www.zerodayinitiative.com/rss/upcoming/"}, - {"name": "zdi-analyzed", "url": "https://www.zerodayinitiative.com/rss/published/"}, - {"name": "vulners", "url": "https://vulners.com/rss.xml"}, - {"name": "seclists-bugtraq", "url": "https://seclists.org/rss/bugtraq.rss"}, - {"name": "seclists-full", "url": "https://seclists.org/rss/fulldisclosure.rss"}, - {"name": "seclists-oss", "url": "https://seclists.org/rss/oss-sec.rss"}, - {"name": "inthewild", "url": "https://raw.githubusercontent.com/gmatuz/inthewilddb/master/rss.xml"}, - {"name": "tenable", "url": "https://www.tenable.com/cve/feeds?sort=newest"}, - {"name": "tenable-updated", "url": "https://www.tenable.com/cve/feeds?sort=updated"}, - {"name": "center-for-internet-security", "url": "https://www.cisecurity.org/feed/advisories"} + {"name": "Windows Internals Blog", "url": "https://windows-internals.com/feed/"} ] } keywords = { - "last_modified": "2024-04-10", + "last_modified": "2024-03-18", "ignored": [ "hiring" ], "static_keywords": [ + "adobe", + "airmagnet survey ", "amazon", "android", "apple", @@ -108,6 +97,7 @@ "cisco", "code42", "connectwise", + "debian", "defender", "dell", "designcad", @@ -134,6 +124,7 @@ "ivanti", "jamf", "juniper ", + "jwt", "knowbe4", "lenovo", "linux", @@ -147,12 +138,15 @@ "office", "nessus", "netapp", + "notepad++", "office365", "okta", "palo alto", "parallels", "paycor", + "pluralsight", "pulse secure", + "putty", "pycharm", "rapidfire", "ricoh", diff --git a/slacker/slacker.py b/slacker/slacker.py index 7936d0d..606e200 100644 --- a/slacker/slacker.py +++ b/slacker/slacker.py @@ -22,7 +22,7 @@ def init_slack_client(slack_token): return WebClient(token=slack_token) -def read_channel(client, channel_id, rss_type, pages_to_read): +def read_channel(client, channel_id, rss_type): """ Reads channel conversations and returns matching content @@ -52,14 +52,12 @@ def read_channel(client, channel_id, rss_type, pages_to_read): } try: - conversation_history = [] + # Call the conversations.history method using the WebClient + # The conversations.history returns 99 messages by default + # Results are paginated, see: https://api.slack.com/method/conversations.history$pagination + # TODO handle paginating multiple pages result = client.conversations_history(channel=channel_id) - conversation_history.extend(result["messages"]) - - while result["response_metadata"]["next_cursor"] is not None and pages_to_read > 0: - result = client.conversations_history(channel=channel_id, cursor=result["response_metadata"]["next_cursor"]) - conversation_history.extend(result["messages"]) - pages_to_read = pages_to_read - 1 + conversation_history = result["messages"] # Initialize dict and lists for storing links/md5s re_link = [] @@ -249,7 +247,6 @@ def send_message(job_type, message_params, matched, errors, check_stale_keywords slack_token = message_params["slack_token"] slack_channel = message_params["channels"] - pages_to_read = message_params["pages_to_read"] # Check if slack_token is set if slack_token: @@ -257,7 +254,7 @@ def send_message(job_type, message_params, matched, errors, check_stale_keywords slack_client = init_slack_client(slack_token) # Pull RSS that was found already in channel - rss_found = read_channel(slack_client, slack_channel[job_type], job_type, pages_to_read) + rss_found = read_channel(slack_client, slack_channel[job_type], job_type) # Build the message that will be sent message_body = build_results_message(matched, rss_found, job_type)