diff --git a/example/ndpiReader.c b/example/ndpiReader.c index 182c82772f9..7d8573df6f2 100644 --- a/example/ndpiReader.c +++ b/example/ndpiReader.c @@ -96,6 +96,16 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0, num_bin_cluster u_int8_t verbose = 0, enable_flow_stats = 0; int stun_monitoring_pkts_to_process = -1; /* Default */ int stun_monitoring_flags = -1; /* Default */ + +struct cfg { + char *proto; + char *param; + char *value; +}; +#define MAX_NUM_CFGS 16 +static struct cfg cfgs[MAX_NUM_CFGS]; +static int num_cfgs = 0; + int nDPI_LogLevel = 0; char *_debug_protocols = NULL; char *_disabled_protocols = NULL; @@ -594,6 +604,7 @@ static void help(u_int long_help) { " --lru-cache-ttl=NAME:size | Specify the TTL [in seconds] for this LRU cache (0 to disable it). This flag can be used multiple times\n" " --stun-monitoring=: | Configure STUN monitoring: keep monitoring STUN session for more pkts looking for RTP\n" " | (0:0 to disable the feature); set the specified features in \n" + " --cfg=proto,param,value | Configure the specific attribute of this protocol\n" , human_readeable_string_len, min_pattern_len, max_pattern_len, max_num_packets_per_flow, max_packet_payload_dissection, @@ -649,6 +660,8 @@ static void help(u_int long_help) { #define OPTLONG_VALUE_STUN_MONITORING 2000 +#define OPTLONG_VALUE_CFG 3000 + static struct option longopts[] = { /* mandatory extcap options */ { "extcap-interfaces", no_argument, NULL, '0'}, @@ -694,6 +707,8 @@ static struct option longopts[] = { { "lru-cache-ttl", required_argument, NULL, OPTLONG_VALUE_LRU_CACHE_TTL}, { "stun-monitoring", required_argument, NULL, OPTLONG_VALUE_STUN_MONITORING}, + { "cfg", required_argument, NULL, OPTLONG_VALUE_CFG}, + {0, 0, 0, 0} }; @@ -950,6 +965,42 @@ static int parse_two_unsigned_integer(char *param, u_int32_t *num1, u_int32_t *n return -1; } +static int parse_three_strings(char *param, char **s1, char **s2, char **s3) +{ + char *saveptr, *tmp_str, *s1_str, *s2_str = NULL, *s3_str; + + tmp_str = ndpi_strdup(param); + if(tmp_str) { + if(param[0] == ',') { /* First parameter might be missing */ + s1_str = NULL; + s2_str = strtok_r(tmp_str, ",", &saveptr); + } else { + s1_str = strtok_r(tmp_str, ",", &saveptr); + if(s1_str) { + s2_str = strtok_r(NULL, ",", &saveptr); + } + } + if(s2_str) { + s3_str = strtok_r(NULL, ",", &saveptr); + if(s3_str) { + *s1 = ndpi_strdup(s1_str); + *s2 = ndpi_strdup(s2_str); + *s3 = ndpi_strdup(s3_str); + ndpi_free(tmp_str); + if(!s1 || !s2 || !s3) { + ndpi_free(s1); + ndpi_free(s2); + ndpi_free(s3); + return -1; + } + return 0; + } + } + } + ndpi_free(tmp_str); + return -1; +} + /* ********************************** */ /** @@ -968,6 +1019,7 @@ static void parseOptions(int argc, char **argv) { #endif int cache_idx, cache_size, cache_ttl; u_int32_t num_pkts, flags; + char *s1, *s2, *s3; #ifdef USE_DPDK { @@ -1288,7 +1340,20 @@ static void parseOptions(int argc, char **argv) { break; case 'z': - init_prefs |= ndpi_enable_ja3_plus; + if(num_cfgs < MAX_NUM_CFGS) { + cfgs[num_cfgs].proto = ndpi_strdup("tls"); + cfgs[num_cfgs].param = ndpi_strdup("ja3_plus.enable"); + cfgs[num_cfgs].value = ndpi_strdup("1"); + if(cfgs[num_cfgs].proto && + cfgs[num_cfgs].param && + cfgs[num_cfgs].value) { + num_cfgs++; + } else { + ndpi_free(cfgs[num_cfgs].proto); + ndpi_free(cfgs[num_cfgs].param); + ndpi_free(cfgs[num_cfgs].value); + } + } break; case OPTLONG_VALUE_LRU_CACHE_SIZE: @@ -1316,6 +1381,18 @@ static void parseOptions(int argc, char **argv) { stun_monitoring_flags = flags; break; + case OPTLONG_VALUE_CFG: + if(num_cfgs >= MAX_NUM_CFGS || + parse_three_strings(optarg, &s1, &s2, &s3) == -1) { + printf("Invalid parameter [%s] [num:%d/%d]\n", optarg, num_cfgs, MAX_NUM_CFGS); + exit(1); + } + cfgs[num_cfgs].proto = s1; + cfgs[num_cfgs].param = s2; + cfgs[num_cfgs].value = s3; + num_cfgs++; + break; + default: #ifdef DEBUG_TRACE if(trace) fprintf(trace, " #### Unknown option -%c: skipping it #### \n", opt); @@ -2660,7 +2737,7 @@ static void debug_printf(u_int32_t protocol, void *id_struct, static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { NDPI_PROTOCOL_BITMASK enabled_bitmask; struct ndpi_workflow_prefs prefs; - int i; + int i, rc; memset(&prefs, 0, sizeof(prefs)); prefs.decode_tunnels = decode_tunnels; @@ -2739,6 +2816,14 @@ static void setupDetection(u_int16_t thread_id, pcap_t * pcap_handle) { ndpi_set_protocol_aggressiveness(ndpi_thread_info[thread_id].workflow->ndpi_struct, i, aggressiveness[i]); } + for(i = 0; i < num_cfgs; i++) { + rc = ndpi_set_config(ndpi_thread_info[thread_id].workflow->ndpi_struct, + cfgs[i].proto, cfgs[i].param, cfgs[i].value); + if (rc != 0) + fprintf(stderr, "Error setting config [%s][%s][%s]: %d\n", + cfgs[i].proto, cfgs[i].param, cfgs[i].value, rc); + } + if(stun_monitoring_pkts_to_process != -1 && stun_monitoring_flags != -1) ndpi_set_monitoring_state(ndpi_thread_info[thread_id].workflow->ndpi_struct, NDPI_PROTOCOL_STUN, @@ -5658,6 +5743,12 @@ int main(int argc, char **argv) { ndpi_free(_debug_protocols); ndpi_free(_disabled_protocols); + for(i = 0; i < num_cfgs; i++) { + ndpi_free(cfgs[i].proto); + ndpi_free(cfgs[i].param); + ndpi_free(cfgs[i].value); + } + #ifdef DEBUG_TRACE if(trace) fclose(trace); #endif diff --git a/fuzz/fuzz_common_code.c b/fuzz/fuzz_common_code.c index bc4d3215540..b99390fae02 100644 --- a/fuzz/fuzz_common_code.c +++ b/fuzz/fuzz_common_code.c @@ -36,7 +36,7 @@ void fuzz_set_alloc_callbacks_and_seed(int seed) void fuzz_init_detection_module(struct ndpi_detection_module_struct **ndpi_info_mod) { - ndpi_init_prefs prefs = ndpi_enable_ja3_plus; + ndpi_init_prefs prefs = 0; NDPI_PROTOCOL_BITMASK all; NDPI_PROTOCOL_BITMASK debug_bitmask; diff --git a/fuzz/fuzz_config.cpp b/fuzz/fuzz_config.cpp index 53d5c2fd825..8f2e5c20988 100644 --- a/fuzz/fuzz_config.cpp +++ b/fuzz/fuzz_config.cpp @@ -37,6 +37,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { 2 + 1 + 4 + /* ndpi_set_detection_preferences() */ 1 + 3 + 1 + 3 + /* Monitoring */ 7 + /* Opportunistic tls */ + 2 * 21 + /* Cfgs */ 2 + /* Pid */ 2 + /* Category */ 1 + /* Tunnel */ @@ -101,9 +102,12 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { if(fuzzed_data.ConsumeBool()) ndpi_set_detection_preferences(ndpi_info_mod, ndpi_pref_enable_tls_block_dissection, 0 /* unused */); + /* TODO */ +#if 0 if(fuzzed_data.ConsumeBool()) ndpi_set_detection_preferences(ndpi_info_mod, ndpi_pref_max_packets_to_process, fuzzed_data.ConsumeIntegralInRange(0, (1 << 16))); +#endif ndpi_set_detection_preferences(ndpi_info_mod, static_cast(0xFF), 0xFF); /* Invalid preference */ @@ -138,6 +142,51 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { ndpi_get_protocol_aggressiveness(ndpi_info_mod, i); } + /* Cfgs */ + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "amazonaws", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "azure", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "cachefly", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "cloudflare", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "gambling", "domain_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "google", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "googlecloud", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "microsoft", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "mining", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "mullvad", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "protonvpn", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "tor", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "tls", "ja3_plus.enable", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "tls", "metadata.sha1_fingerprint.enable", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "whatsapp", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "zoom", "ip_list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, NULL, "asn_lists.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, NULL, "flow_risk.anonymous_subscriber.list.icloudprivaterelay.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, NULL, "flow_risk.anonymous_subscriber.list.protonvpn.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, NULL, "flow_risk.crawler_bot.list.load", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + /* Wrong */ + if(fuzzed_data.ConsumeBool()) + ndpi_set_config(ndpi_info_mod, "xxx", "xxx", std::to_string(fuzzed_data.ConsumeIntegralInRange(0,2)).c_str()); + ndpi_finalize_initialization(ndpi_info_mod); /* Random protocol configuration */ diff --git a/fuzz/fuzz_filecfg_protocols.c b/fuzz/fuzz_filecfg_protocols.c index 6731b7af91b..35a6bb1f34a 100644 --- a/fuzz/fuzz_filecfg_protocols.c +++ b/fuzz/fuzz_filecfg_protocols.c @@ -4,29 +4,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { struct ndpi_detection_module_struct *ndpi_struct; FILE *fd; - /* Try to be fast */ - ndpi_init_prefs prefs = ndpi_dont_load_tor_list | - ndpi_dont_load_azure_list | - ndpi_dont_load_whatsapp_list | - ndpi_dont_load_amazon_aws_list | - ndpi_dont_load_ethereum_list | - ndpi_dont_load_zoom_list | - ndpi_dont_load_cloudflare_list | - ndpi_dont_load_microsoft_list | - ndpi_dont_load_google_list | - ndpi_dont_load_google_cloud_list | - ndpi_dont_load_asn_lists | - ndpi_dont_init_risk_ptree | - ndpi_dont_load_cachefly_list | - ndpi_dont_load_protonvpn_list | - ndpi_dont_load_mullvad_list; NDPI_PROTOCOL_BITMASK all; NDPI_PROTOCOL_BITMASK debug_bitmask; /* To allow memory allocation failures */ fuzz_set_alloc_callbacks_and_seed(size); - ndpi_struct = ndpi_init_detection_module(prefs); + ndpi_struct = ndpi_init_detection_module(0); NDPI_BITMASK_SET_ALL(all); ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all); diff --git a/fuzz/fuzz_ndpi_reader.c b/fuzz/fuzz_ndpi_reader.c index fd7d185e15b..f6af8c4cbb0 100644 --- a/fuzz/fuzz_ndpi_reader.c +++ b/fuzz/fuzz_ndpi_reader.c @@ -18,7 +18,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0; u_int8_t enable_flow_stats = 1; u_int8_t human_readeable_string_len = 5; u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */; -ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic; +ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_tcp_ack_payload_heuristic; int enable_malloc_bins = 1; int malloc_size_stats = 0; int max_malloc_bins = 14; @@ -71,6 +71,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size) { ndpi_set_monitoring_state(workflow->ndpi_struct, NDPI_PROTOCOL_STUN, 10, NDPI_MONITORING_STUN_SUBCLASSIFIED); + ndpi_set_config(workflow->ndpi_struct, "tls", "ja3_plus.enable", "1"); + memset(workflow->stats.protocol_counter, 0, sizeof(workflow->stats.protocol_counter)); memset(workflow->stats.protocol_counter_bytes, 0, diff --git a/fuzz/fuzz_readerutils_parseprotolist.cpp b/fuzz/fuzz_readerutils_parseprotolist.cpp index 771719a8ff3..b41a83d09d1 100644 --- a/fuzz/fuzz_readerutils_parseprotolist.cpp +++ b/fuzz/fuzz_readerutils_parseprotolist.cpp @@ -13,7 +13,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0; u_int8_t enable_flow_stats = 0; u_int8_t human_readeable_string_len = 5; u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */; -ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic; +ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_tcp_ack_payload_heuristic; int enable_malloc_bins = 0; int malloc_size_stats = 0; int max_malloc_bins = 14; diff --git a/fuzz/fuzz_readerutils_workflow.cpp b/fuzz/fuzz_readerutils_workflow.cpp index f753560f271..a271fc2c7aa 100644 --- a/fuzz/fuzz_readerutils_workflow.cpp +++ b/fuzz/fuzz_readerutils_workflow.cpp @@ -15,7 +15,7 @@ u_int8_t enable_protocol_guess = 1, enable_payload_analyzer = 0; u_int8_t enable_flow_stats = 0; u_int8_t human_readeable_string_len = 5; u_int8_t max_num_udp_dissected_pkts = 16 /* 8 is enough for most protocols, Signal requires more */, max_num_tcp_dissected_pkts = 80 /* due to telnet */; -ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_ja3_plus | ndpi_enable_tcp_ack_payload_heuristic; +ndpi_init_prefs init_prefs = ndpi_track_flow_payload | ndpi_enable_tcp_ack_payload_heuristic; int enable_malloc_bins = 0; int malloc_size_stats = 0; int max_malloc_bins = 14; diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index 13e11f65125..a20ad327615 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -2154,6 +2154,15 @@ extern "C" { */ void *ndpi_get_user_data(struct ndpi_detection_module_struct *ndpi_str); + + int ndpi_set_config(struct ndpi_detection_module_struct *ndpi_str, + const char *proto, const char *param, const char *value); + char *ndpi_get_config(struct ndpi_detection_module_struct *ndpi_str, + const char *proto, const char *param, char *buf, int buf_len); + char *ndpi_dump_config(struct ndpi_detection_module_struct *ndpi_str, + FILE *fd); + + #ifdef __cplusplus } #endif diff --git a/src/include/ndpi_define.h.in b/src/include/ndpi_define.h.in index 4d61d7130e4..c0a0f550033 100644 --- a/src/include/ndpi_define.h.in +++ b/src/include/ndpi_define.h.in @@ -153,7 +153,6 @@ /* misc definitions */ #define NDPI_DEFAULT_MAX_TCP_RETRANSMISSION_WINDOW_SIZE 0x10000 -#define NDPI_DEFAULT_MAX_NUM_PKTS_PER_FLOW_TO_DISSECT 32 /* TODO: rebuild all memory areas to have a more aligned memory block here */ diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 919a42501ac..fd4a6640c6e 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1153,7 +1153,6 @@ typedef enum { typedef enum { ndpi_pref_direction_detect_disable = 0, - ndpi_pref_max_packets_to_process, ndpi_pref_enable_tls_block_dissection, /* nDPI considers only those blocks past the certificate exchange */ } ndpi_detection_preference; @@ -1247,11 +1246,39 @@ typedef struct { } nbpf_filter; #endif +struct ndpi_detection_module_config_struct { + /* IP lists */ + int ip_list_amazonaws_enabled; + int ip_list_azure_enabled; + int ip_list_cachefly_enabled; + int ip_list_cloudflare_enabled; + int ip_list_google_enabled; + int ip_list_googlecloud_enabled; + int ip_list_microsoft_enabled; + int ip_list_mining_enabled; + int ip_list_mullvad_enabled; + int ip_list_protonvpn_enabled; + int ip_list_tor_enabled; + int ip_list_whatsapp_enabled; + int ip_list_zoom_enabled; + + int asn_lists_enabled; + + int risk_anonymous_subscriber_list_icloudprivaterelay_enabled; + int risk_anonymous_subscriber_list_protonvpn_enabled; + int risk_crawler_bot_list_enabled; + + int max_packets_to_process; + + /* TLS */ + int sha1_fingerprint_enabled; + int ja3_plus_enabled; +}; + struct ndpi_detection_module_struct { NDPI_PROTOCOL_BITMASK detection_bitmask; u_int64_t current_ts; - u_int16_t max_packets_to_process; u_int16_t num_tls_blocks_to_follow; u_int8_t skip_tls_blocks_until_change_cipher:1, enable_ja3_plus:1, _notused:6; u_int8_t tls_certificate_expire_in_x_days; @@ -1326,6 +1353,8 @@ struct ndpi_detection_module_struct { u_int8_t ip_version_limit; + struct ndpi_detection_module_config_struct cfg; + /* NDPI_PROTOCOL_TINC */ struct cache *tinc_cache; @@ -1791,22 +1820,8 @@ typedef u_int32_t ndpi_init_prefs; typedef enum { ndpi_no_prefs = 0, - ndpi_dont_load_tor_list = (1 << 0), ndpi_dont_init_libgcrypt = (1 << 1), - ndpi_enable_ja3_plus = (1 << 2), - ndpi_dont_load_azure_list = (1 << 3), - ndpi_dont_load_whatsapp_list = (1 << 4), - ndpi_dont_load_amazon_aws_list = (1 << 5), - ndpi_dont_load_ethereum_list = (1 << 6), - ndpi_dont_load_zoom_list = (1 << 7), - ndpi_dont_load_cloudflare_list = (1 << 8), - ndpi_dont_load_microsoft_list = (1 << 9), - ndpi_dont_load_google_list = (1 << 10), - ndpi_dont_load_google_cloud_list = (1 << 11), - ndpi_dont_load_asn_lists = (1 << 12), - ndpi_dont_load_icloud_private_relay_list = (1 << 13), ndpi_dont_init_risk_ptree = (1 << 14), - ndpi_dont_load_cachefly_list = (1 << 15), ndpi_track_flow_payload = (1 << 16), /* In some networks, there are some anomalous TCP flows where the smallest ACK packets have some kind of zero padding. @@ -1819,15 +1834,11 @@ typedef enum { correct detection/classification. See #1946 for other details */ ndpi_enable_tcp_ack_payload_heuristic = (1 << 17), - ndpi_dont_load_crawlers_list = (1 << 18), - ndpi_dont_load_protonvpn_list = (1 << 19), /* Heuristic to detect fully encrypted sessions, i.e. flows where every bytes of the payload is encrypted in an attempt to “look like nothing”. This heuristic only analyzes the first packet of the flow. See: https://www.usenix.org/system/files/sec23fall-prepub-234-wu-mingshi.pdf */ - ndpi_disable_fully_encrypted_heuristic = (1 << 20), - ndpi_dont_load_protonvpn_exit_nodes_list = (1 << 21), - ndpi_dont_load_mullvad_list = (1 << 22), + ndpi_disable_fully_encrypted_heuristic = (1 << 21), } ndpi_prefs; typedef struct { diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 5eb9f38d6ca..4b8fc3207ee 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -24,6 +24,7 @@ #include #include #include +#include #ifdef __APPLE__ #include @@ -235,6 +236,8 @@ static int ndpi_callback_init(struct ndpi_detection_module_struct *ndpi_str); static void ndpi_enabled_callbacks_init(struct ndpi_detection_module_struct *ndpi_str, const NDPI_PROTOCOL_BITMASK *dbm, int count_only); +static void set_default_config(struct ndpi_detection_module_config_struct *cfg); + /* ****************************************** */ ndpi_custom_dga_predict_fctn ndpi_dga_function = NULL; @@ -1031,13 +1034,6 @@ int ndpi_set_detection_preferences(struct ndpi_detection_module_struct *ndpi_str ndpi_str->skip_tls_blocks_until_change_cipher = 1; break; - case ndpi_pref_max_packets_to_process: - if(value > 0xFFFF) { - return(-1); - } - ndpi_str->max_packets_to_process = value; - break; - default: return(-1); } @@ -2843,9 +2839,6 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs NDPI_BITMASK_RESET(ndpi_str->debug_bitmask); #endif /* NDPI_ENABLE_DEBUG_MESSAGES */ - if(prefs & ndpi_enable_ja3_plus) - ndpi_str->enable_ja3_plus = 1; - if(!(prefs & ndpi_dont_init_libgcrypt)) { if(!gcry_control (GCRYCTL_INITIALIZATION_FINISHED_P)) { const char *gcrypt_ver = gcry_check_version(NULL); @@ -2864,105 +2857,18 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs if((ndpi_str->protocols_ptree = ndpi_patricia_new(32 /* IPv4 */)) != NULL) { ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, host_protocol_list); - - if(!(prefs & ndpi_dont_load_cachefly_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_cachefly_protocol_list); - - if(!(prefs & ndpi_dont_load_tor_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_tor_protocol_list); - - if(!(prefs & ndpi_dont_load_azure_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_microsoft_azure_protocol_list); - - if(!(prefs & ndpi_dont_load_whatsapp_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_whatsapp_protocol_list); - - if(!(prefs & ndpi_dont_load_amazon_aws_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_amazon_aws_protocol_list); - - if(!(prefs & ndpi_dont_load_ethereum_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_mining_protocol_list); - - if(!(prefs & ndpi_dont_load_zoom_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_zoom_protocol_list); - - if(!(prefs & ndpi_dont_load_cloudflare_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_cloudflare_protocol_list); - - if(!(prefs & ndpi_dont_load_microsoft_list)) { - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_microsoft_365_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_ms_one_drive_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_ms_outlook_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_skype_teams_protocol_list); - } - - if(!(prefs & ndpi_dont_load_google_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_google_protocol_list); - - if(!(prefs & ndpi_dont_load_google_cloud_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_google_cloud_protocol_list); - - if(!(prefs & ndpi_dont_load_protonvpn_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_protonvpn_protocol_list); - - if(!(prefs & ndpi_dont_load_mullvad_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_mullvad_protocol_list); - - if(!(prefs & ndpi_dont_load_asn_lists)) { - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_telegram_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_apple_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_twitter_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_netflix_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_webex_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_teamviewer_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_facebook_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_tencent_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_opendns_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_dropbox_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_starcraft_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_ubuntuone_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_twitch_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_hotspot_shield_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_github_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_steam_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_bloomberg_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_citrix_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_edgecast_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_goto_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_riotgames_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_threema_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_alibaba_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_avast_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_discord_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_line_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_vk_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_yandex_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_yandex_cloud_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_disneyplus_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_hulu_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_epicgames_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_nvidia_protocol_list); - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_roblox_protocol_list); - } - - if(prefs & ndpi_track_flow_payload) - ndpi_str->max_payload_track_len = 1024; /* track up to X payload bytes */ } + if(prefs & ndpi_track_flow_payload) + ndpi_str->max_payload_track_len = 1024; /* track up to X payload bytes */ + ndpi_str->ip_risk_mask_ptree = ndpi_patricia_new(32 /* IPv4 */); if(!(prefs & ndpi_dont_init_risk_ptree)) { - if((ndpi_str->ip_risk_ptree = ndpi_patricia_new(32 /* IPv4 */)) != NULL) { - if(!(prefs & ndpi_dont_load_icloud_private_relay_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->ip_risk_ptree, ndpi_anonymous_subscriber_icloud_private_relay_protocol_list); - if(!(prefs & ndpi_dont_load_protonvpn_exit_nodes_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->ip_risk_ptree, ndpi_anonymous_subscriber_protonvpn_protocol_list); - if(!(prefs & ndpi_dont_load_crawlers_list)) - ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->ip_risk_ptree, ndpi_http_crawler_bot_protocol_list); - } + ndpi_str->ip_risk_ptree = ndpi_patricia_new(32 /* IPv4 */); } - ndpi_str->max_packets_to_process = NDPI_DEFAULT_MAX_NUM_PKTS_PER_FLOW_TO_DISSECT; + set_default_config(&ndpi_str->cfg); NDPI_BITMASK_SET_ALL(ndpi_str->detection_bitmask); ndpi_str->user_data = NULL; @@ -3173,6 +3079,102 @@ void ndpi_finalize_initialization(struct ndpi_detection_module_struct *ndpi_str) if(!ndpi_str) return; + /* IP lists */ + + if(ndpi_str->protocols_ptree) { + if(ndpi_str->cfg.ip_list_amazonaws_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_amazon_aws_protocol_list); + } + if(ndpi_str->cfg.ip_list_azure_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_microsoft_azure_protocol_list); + } + if(ndpi_str->cfg.ip_list_cachefly_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_cachefly_protocol_list); + } + if(ndpi_str->cfg.ip_list_cloudflare_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_cloudflare_protocol_list); + } + if(ndpi_str->cfg.ip_list_google_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_google_protocol_list); + } + if(ndpi_str->cfg.ip_list_googlecloud_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_google_cloud_protocol_list); + } + if(ndpi_str->cfg.ip_list_microsoft_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_microsoft_365_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_ms_one_drive_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_ms_outlook_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_skype_teams_protocol_list); + } + if(ndpi_str->cfg.ip_list_mining_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_mining_protocol_list); + } + if(ndpi_str->cfg.ip_list_protonvpn_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_protonvpn_protocol_list); + } + if(ndpi_str->cfg.ip_list_tor_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_tor_protocol_list); + } + if(ndpi_str->cfg.ip_list_whatsapp_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_whatsapp_protocol_list); + } + if(ndpi_str->cfg.ip_list_zoom_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_zoom_protocol_list); + } + if(ndpi_str->cfg.ip_list_mullvad_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_mullvad_protocol_list); + } + + if(ndpi_str->cfg.asn_lists_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_telegram_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_apple_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_twitter_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_netflix_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_webex_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_teamviewer_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_facebook_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_tencent_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_opendns_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_dropbox_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_starcraft_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_ubuntuone_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_twitch_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_hotspot_shield_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_github_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_steam_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_bloomberg_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_citrix_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_edgecast_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_goto_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_riotgames_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_threema_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_alibaba_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_avast_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_discord_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_line_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_vk_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_yandex_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_yandex_cloud_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_disneyplus_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_hulu_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_epicgames_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_nvidia_protocol_list); + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->protocols_ptree, ndpi_protocol_roblox_protocol_list); + } + } + + if(ndpi_str->ip_risk_ptree) { + if(ndpi_str->cfg.risk_anonymous_subscriber_list_icloudprivaterelay_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->ip_risk_ptree, ndpi_anonymous_subscriber_icloud_private_relay_protocol_list); + } + if(ndpi_str->cfg.risk_anonymous_subscriber_list_protonvpn_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->ip_risk_ptree, ndpi_anonymous_subscriber_protonvpn_protocol_list); + } + if(ndpi_str->cfg.risk_crawler_bot_list_enabled) { + ndpi_init_ptree_ipv4(ndpi_str, ndpi_str->ip_risk_ptree, ndpi_http_crawler_bot_protocol_list); + } + } + ndpi_add_domain_risk_exceptions(ndpi_str); if(ndpi_str->ookla_cache_num_entries > 0) { @@ -7262,7 +7264,7 @@ static ndpi_protocol ndpi_internal_detection_process_packet(struct ndpi_detectio return(ret); } - if(ndpi_str->max_packets_to_process > 0 && flow->num_processed_pkts >= ndpi_str->max_packets_to_process) { + if(ndpi_str->cfg.max_packets_to_process > 0 && flow->num_processed_pkts >= ndpi_str->cfg.max_packets_to_process) { flow->extra_packets_func = NULL; /* To allow ndpi_extra_dissection_possible() to fail */ flow->fail_with_unknown = 1; return(ret); /* Avoid spending too much time with this flow */ @@ -10272,3 +10274,198 @@ void *ndpi_get_user_data(struct ndpi_detection_module_struct *ndpi_str) return ndpi_str->user_data; return NULL; } + +/* ******************************************************************** */ + +static int _set_param_enable_disable(void *_variable, const char *value) +{ + int *variable = (int *)_variable; + + if(strcmp(value, "1") == 0 || + strcmp(value, "enable") == 0) { + *variable = 1; + return 0; + } + if(strcmp(value, "0") == 0 || + strcmp(value, "disable") == 0) { + *variable = 0; + return 0; + } + return -1; +} + +static int _set_param_int(void *_variable, const char *value) +{ + int *variable = (int *)_variable; + char *endptr; + long val; + + errno = 0; /* To distinguish success/failure after call */ + val = strtol(value, &endptr, 10); + + /* Check for various possible errors */ + if((errno == ERANGE && (val == LONG_MAX || val == LONG_MIN)) || + (errno != 0 && val == 0)) { + return -1; + } + if (endptr == value) { + /* No digits were found */ + return -1; + } + /* If we got here, strtol() successfully parsed a number */ + *variable = val; + return 0; +} + +/* It can be used for CFG_PARAM_ENABLE_DISABLE parameters, too */ +static char *_get_param_int(void *_variable, char *buf, int buf_len) +{ + int *variable = (int *)_variable; + + snprintf(buf, buf_len, "%d", *variable); + buf[buf_len - 1] = '\0'; + return buf; +} + + +typedef int (*cfg_fn)(void *variable, const char *value); + +enum cfg_param_type { + CFG_PARAM_ENABLE_DISABLE = 0, + CFG_PARAM_INT = 1, +}; + +#define __OFF(a) offsetof(struct ndpi_detection_module_config_struct, a) + +static const struct cfg_param { + char *proto; + char *param; + char *default_value; + enum cfg_param_type type; + int offset; +} cfg_params[] = { + /* Per-protocol parameters */ + + { "amazonaws", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_amazonaws_enabled) }, + { "azure", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_azure_enabled) }, + { "cachefly", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_cachefly_enabled) }, + { "cloudflare", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_cloudflare_enabled) }, + { "google", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_google_enabled) }, + { "googlecloud", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_googlecloud_enabled) }, + { "microsoft", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_microsoft_enabled) }, + { "mining", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_mining_enabled) }, + { "mullvad", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_mullvad_enabled) }, + { "protonvpn", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_protonvpn_enabled) }, + { "tor", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_tor_enabled) }, + { "tls", "ja3_plus.enable", "0", CFG_PARAM_ENABLE_DISABLE, __OFF(ja3_plus_enabled) }, + /* An example of metadata configuration (yes/no) */ + { "tls", "metadata.sha1_fingerprint.enable", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(sha1_fingerprint_enabled) }, + { "whatsapp", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_whatsapp_enabled) }, + { "zoom", "ip_list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(ip_list_zoom_enabled) }, + + /* Global parameters */ + + { NULL, "asn_lists.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(asn_lists_enabled) }, + { NULL, "flow_risk.anonymous_subscriber.list.icloudprivaterelay.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(risk_anonymous_subscriber_list_icloudprivaterelay_enabled) }, + { NULL, "flow_risk.anonymous_subscriber.list.protonvpn.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(risk_anonymous_subscriber_list_protonvpn_enabled) }, + { NULL, "flow_risk.crawler_bot.list.load", "1", CFG_PARAM_ENABLE_DISABLE, __OFF(risk_crawler_bot_list_enabled) }, + /* An example of integer configuration */ + { NULL, "packets_limit_per_flow", "32", CFG_PARAM_INT, __OFF(max_packets_to_process) }, + + { NULL, NULL, NULL, 0, -1 }, +}; + +#undef __OFF + +static void set_default_config(struct ndpi_detection_module_config_struct *cfg) +{ + const struct cfg_param *c; + + for(c = &cfg_params[0]; c && c->param; c++) { + switch(c->type) { + case CFG_PARAM_ENABLE_DISABLE: + _set_param_enable_disable((void *)((char *)cfg + c->offset), c->default_value); + break; + case CFG_PARAM_INT: + _set_param_int((void *)((char *)cfg + c->offset), c->default_value); + break; + } + } +} + +int ndpi_set_config(struct ndpi_detection_module_struct *ndpi_str, + const char *proto, const char *param, const char *value) +{ + const struct cfg_param *c; + + if(!ndpi_str || !param || !value) + return -2; + + NDPI_LOG_ERR(ndpi_str, "Set [%s][%s][%s]\n", proto, param, value); + + for(c = &cfg_params[0]; c && c->param; c++) { + if(((proto == NULL && c->proto == NULL) || + (proto && c->proto && strcmp(proto, c->proto) == 0)) && + strcmp(param, c->param) == 0) { + + switch(c->type) { + case CFG_PARAM_ENABLE_DISABLE: + return _set_param_enable_disable((void *)((char *)&ndpi_str->cfg + c->offset), value); + case CFG_PARAM_INT: + return _set_param_int((void *)((char *)&ndpi_str->cfg + c->offset), value); + } + } + } + return -3; +} + +char *ndpi_get_config(struct ndpi_detection_module_struct *ndpi_str, + const char *proto, const char *param, char *buf, int buf_len) +{ + const struct cfg_param *c; + + if(!ndpi_str || !param || !buf || buf_len <= 0) + return NULL; + + NDPI_LOG_ERR(ndpi_str, "Get [%s][%s]\n", proto, param); + + for(c = &cfg_params[0]; c && c->param; c++) { + if(((proto == NULL && c->proto == NULL) || + (proto && c->proto && strcmp(proto, c->proto) == 0)) && + strcmp(param, c->param) == 0) { + + switch(c->type) { + case CFG_PARAM_ENABLE_DISABLE: + case CFG_PARAM_INT: + return _get_param_int((void *)((char *)&ndpi_str->cfg + c->offset), buf, buf_len); + } + } + } + return NULL; +} + +char *ndpi_dump_config(struct ndpi_detection_module_struct *ndpi_str, + FILE *fd) +{ + const struct cfg_param *c; + char buf[64]; + + if(!ndpi_str || !fd) + return NULL; + + NDPI_LOG_ERR(ndpi_str, "Dump\n"); + + fprintf(fd, "Configuration:\n"); + + for(c = &cfg_params[0]; c && c->param; c++) { + switch(c->type) { + case CFG_PARAM_ENABLE_DISABLE: + case CFG_PARAM_INT: + fprintf(fd, " *) %s.%s: %s [%s]\n", + c->proto, c->param, + _get_param_int((void *)((char *)&ndpi_str->cfg + c->offset), buf, sizeof(buf)), + c->default_value); + } + } + return NULL; +} diff --git a/src/lib/protocols/tls.c b/src/lib/protocols/tls.c index 09e10858ee3..695bb5f9747 100644 --- a/src/lib/protocols/tls.c +++ b/src/lib/protocols/tls.c @@ -773,7 +773,6 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct, u_int32_t certificates_length, length = (packet->payload[1] << 16) + (packet->payload[2] << 8) + packet->payload[3]; u_int32_t certificates_offset = 7 + (is_dtls ? 8 : 0); u_int8_t num_certificates_found = 0; - SHA1_CTX srv_cert_fingerprint_ctx ; #ifdef DEBUG_TLS printf("[TLS] %s() [payload_packet_len=%u][direction: %u][%02X %02X %02X %02X %02X %02X...]\n", @@ -827,8 +826,6 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct, if(num_certificates_found++ == 0) /* Dissect only the first certificate that is the one we care */ { /* For SHA-1 we take into account only the first certificate and not all of them */ - SHA1Init(&srv_cert_fingerprint_ctx); - #ifdef DEBUG_CERTIFICATE_HASH { u_int32_t i; @@ -840,36 +837,42 @@ int processCertificate(struct ndpi_detection_module_struct *ndpi_struct, } #endif - SHA1Update(&srv_cert_fingerprint_ctx, - &packet->payload[certificates_offset], - certificate_len); + if(ndpi_struct->cfg.sha1_fingerprint_enabled) { + SHA1_CTX srv_cert_fingerprint_ctx ; - SHA1Final(flow->protos.tls_quic.sha1_certificate_fingerprint, &srv_cert_fingerprint_ctx); + SHA1Init(&srv_cert_fingerprint_ctx); - flow->protos.tls_quic.fingerprint_set = 1; + SHA1Update(&srv_cert_fingerprint_ctx, + &packet->payload[certificates_offset], + certificate_len); - uint8_t * sha1 = flow->protos.tls_quic.sha1_certificate_fingerprint; - const size_t sha1_siz = sizeof(flow->protos.tls_quic.sha1_certificate_fingerprint); - char sha1_str[20 /* sha1_siz */ * 2 + 1]; - static const char hexalnum[] = "0123456789ABCDEF"; - size_t i; - for (i = 0; i < sha1_siz; ++i) { - u_int8_t lower = (sha1[i] & 0x0F); - u_int8_t upper = (sha1[i] & 0xF0) >> 4; - sha1_str[i*2] = hexalnum[upper]; - sha1_str[i*2 + 1] = hexalnum[lower]; - } - sha1_str[sha1_siz * 2] = '\0'; + SHA1Final(flow->protos.tls_quic.sha1_certificate_fingerprint, &srv_cert_fingerprint_ctx); + + flow->protos.tls_quic.fingerprint_set = 1; + + uint8_t * sha1 = flow->protos.tls_quic.sha1_certificate_fingerprint; + const size_t sha1_siz = sizeof(flow->protos.tls_quic.sha1_certificate_fingerprint); + char sha1_str[20 /* sha1_siz */ * 2 + 1]; + static const char hexalnum[] = "0123456789ABCDEF"; + size_t i; + for (i = 0; i < sha1_siz; ++i) { + u_int8_t lower = (sha1[i] & 0x0F); + u_int8_t upper = (sha1[i] & 0xF0) >> 4; + sha1_str[i*2] = hexalnum[upper]; + sha1_str[i*2 + 1] = hexalnum[lower]; + } + sha1_str[sha1_siz * 2] = '\0'; #ifdef DEBUG_TLS - printf("[TLS] SHA-1: %s\n", sha1_str); + printf("[TLS] SHA-1: %s\n", sha1_str); #endif - if(ndpi_struct->malicious_sha1_hashmap != NULL) { - u_int16_t rc1 = ndpi_hash_find_entry(ndpi_struct->malicious_sha1_hashmap, sha1_str, sha1_siz * 2, NULL); + if(ndpi_struct->malicious_sha1_hashmap != NULL) { + u_int16_t rc1 = ndpi_hash_find_entry(ndpi_struct->malicious_sha1_hashmap, sha1_str, sha1_siz * 2, NULL); - if(rc1 == 0) - ndpi_set_risk(ndpi_struct, flow, NDPI_MALICIOUS_SHA1_CERTIFICATE, sha1_str); + if(rc1 == 0) + ndpi_set_risk(ndpi_struct, flow, NDPI_MALICIOUS_SHA1_CERTIFICATE, sha1_str); + } } processCertificateElements(ndpi_struct, flow, certificates_offset, certificate_len); diff --git a/tests/cfgs/disable_metadata/config.txt b/tests/cfgs/disable_metadata/config.txt new file mode 100644 index 00000000000..1e5e30bf838 --- /dev/null +++ b/tests/cfgs/disable_metadata/config.txt @@ -0,0 +1 @@ +--cfg=tls,metadata.sha1_fingerprint.enable,0 diff --git a/tests/cfgs/disable_metadata/pcap/tls_verylong_certificate.pcap b/tests/cfgs/disable_metadata/pcap/tls_verylong_certificate.pcap new file mode 120000 index 00000000000..2f722f28ed6 --- /dev/null +++ b/tests/cfgs/disable_metadata/pcap/tls_verylong_certificate.pcap @@ -0,0 +1 @@ +../../default/pcap/tls_verylong_certificate.pcap \ No newline at end of file diff --git a/tests/cfgs/disable_metadata/result/tls_verylong_certificate.pcap.out b/tests/cfgs/disable_metadata/result/tls_verylong_certificate.pcap.out new file mode 100644 index 00000000000..3c1daaac642 --- /dev/null +++ b/tests/cfgs/disable_metadata/result/tls_verylong_certificate.pcap.out @@ -0,0 +1,30 @@ +Guessed flow protos: 0 + +DPI Packets (TCP): 11 (11.00 pkts/flow) +Confidence DPI : 1 (flows) +Num dissector calls: 1 (1.00 diss/flow) +LRU cache ookla: 0/0/0 (insert/search/found) +LRU cache bittorrent: 0/0/0 (insert/search/found) +LRU cache zoom: 0/0/0 (insert/search/found) +LRU cache stun: 0/0/0 (insert/search/found) +LRU cache tls_cert: 0/0/0 (insert/search/found) +LRU cache mining: 0/0/0 (insert/search/found) +LRU cache msteams: 0/0/0 (insert/search/found) +LRU cache stun_zoom: 0/0/0 (insert/search/found) +Automa host: 1/1 (search/found) +Automa domain: 1/0 (search/found) +Automa tls cert: 0/0 (search/found) +Automa risk mask: 0/0 (search/found) +Automa common alpns: 1/1 (search/found) +Patricia risk mask: 0/0 (search/found) +Patricia risk: 0/0 (search/found) +Patricia protocols: 2/0 (search/found) + +Cybersec 48 22229 1 + +JA3 Host Stats: + IP Address # JA3C + 1 192.168.1.160 1 + + + 1 TCP 192.168.1.160:54804 <-> 151.101.66.49:443 [proto: 91.283/TLS.Cybersec][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 11][cat: Cybersecurity/33][24 pkts/2404 bytes <-> 24 pkts/19825 bytes][Goodput ratio: 35/92][0.09 sec][Hostname/SNI: feodotracker.abuse.ch][(Advertised) ALPNs: http/1.1][(Negotiated) ALPN: http/1.1][bytes ratio: -0.784 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 4/4 15/21 5/7][Pkt Len c2s/s2c min/avg/max/stddev: 54/66 100/826 583/1434 109/662][TLSv1.2][JA3C: 2a26b1a62e40d25d4de3babc9d532f30][ServerNames: p2.shared.global.fastly.net,*.12wbt.com,*.2bleacherreport.com,*.3bleacherreport.com,*.4bleacherreport.com,*.8bleacherreport.com,*.abuse.ch,*.acdn-it.ps-pantheon.com,*.cdn.livingmap.com,*.content.plastiq.com,*.dimensions.ai,*.dollarshaveclub.co.uk,*.dollarshaveclub.com,*.dontpayfull.com,*.ebisubook.com,*.foreignaffairs.com,*.fs.jibjab.com,*.fs.unitprints.com,*.ggleap.com,*.goodeggs.com,*.huevosbuenos.com,*.indy.myomnigon.com,*.jwatch.org,*.kingsfordcharcoal.com.au,*.lancenters.com,*.madebywe.com,*.minirodini.com,*.modcloth.net,*.orionlabs.io,*.ps-pantheon.com,*.scodle.com,*.steelseries.com,*.theforeman.org,*.uploads.eversign.com,*.uploads.schoox.com,*.vts.com,*.x.stg1.ebisubook.com,*.yang2020.com,12wbt.com,2bleacherreport.com,3bleacherreport.com,4bleacherreport.com,8bleacherreport.com,abuse.ch,brita.com,cdn.fwupd.org,cdn.livingmap.com,cdn.seated.com,cdn.skillacademy.com,clinicaloptions.com,clorox.com,content-preprod.beaverbrooksweb2.co.uk,content.beaverbrooks.co.uk,content.plastiq.com,coolmathgames.com,copterroyale.coolmathgames.com,d8-dev.coolmathgames.com,deflyio.coolmathgames.com,delivery-api.evadacms.com,dimensions.ai,dollarshaveclub.co.uk,dollarshaveclub.com,dontpayfull.com,eluniverso.com,email.amg-group.co,email.tekoforlife.co.uk,feedmarket.fr,freshstep.com,ggleap.com,goodeggs.com,heap.io,huevosbuenos.com,identity.linuxfoundation.org,joebiden.com,jwatch.org,kingsford.co.nz,kingsfordcharcoal.com.au,lancenters.com,lists.linuxfoundation.org,m-stage.coolmathgames.com,m.coolmathgames.com,madebywe.com,minirodini.com,modcloth.net,orionlabs.io,puritanmedproducts.com,reviews.org,rg-video-staging.ruangguru.com,rg-video.ruangguru.com,ruangguru.com,scodle.com,stage.coolmathgames.com,staging.appblade.com,steelseries.com,stg.platform.eluniverso.com,test.brita.com,test.heap.io,test.joebiden.com,test.ruangguru.com,theforeman.org,video-cdn.quipper.com,videos.calcworkshop.com,vts.com,www.101network.com,www.autos101.com,www.brita.com,www.clorox.com,www.collider.com,www.coolmathgames.com,www.eluniverso.com,www.flinto.com,www.freshstep.com,www.heap.io,www.holagente.com,www.icsydney.com.au,www.joebiden.com,www.kingsford.co.nz,www.mrnatty.com,www.myjewellerystory.com.au,www.myjs.com,www.netacea.com,www.parenting101.com,www.puritanmedproducts.com,www.reviews.org,www.sba.sa,www.shashatcom.sa,www.uat.ontariocolleges.ca,www.vacation101.com,www.walterspeople.co.uk,www.westwayelectricsupply.com][JA3S: ae53107a2e47ea20c72ac44821a728bf][Issuer: C=BE, O=GlobalSign nv-sa, CN=GlobalSign CloudSSL CA - SHA256 - G3][Subject: C=US, ST=California, L=San Francisco, O=Fastly, Inc., CN=p2.shared.global.fastly.net][Firefox][Validity: 2019-11-19 01:31:22 - 2020-08-29 17:19:32][Cipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256][Plen Bins: 12,16,0,4,0,4,4,0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0,0,0,0,0] diff --git a/tests/cfgs/packets_limit_per_flow/config.txt b/tests/cfgs/packets_limit_per_flow/config.txt new file mode 100644 index 00000000000..0e73f76c571 --- /dev/null +++ b/tests/cfgs/packets_limit_per_flow/config.txt @@ -0,0 +1 @@ +--cfg=,packets_limit_per_flow,4 diff --git a/tests/cfgs/packets_limit_per_flow/pcap/tls_verylong_certificate.pcap b/tests/cfgs/packets_limit_per_flow/pcap/tls_verylong_certificate.pcap new file mode 120000 index 00000000000..2f722f28ed6 --- /dev/null +++ b/tests/cfgs/packets_limit_per_flow/pcap/tls_verylong_certificate.pcap @@ -0,0 +1 @@ +../../default/pcap/tls_verylong_certificate.pcap \ No newline at end of file diff --git a/tests/cfgs/packets_limit_per_flow/result/tls_verylong_certificate.pcap.out b/tests/cfgs/packets_limit_per_flow/result/tls_verylong_certificate.pcap.out new file mode 100644 index 00000000000..ef17beecb92 --- /dev/null +++ b/tests/cfgs/packets_limit_per_flow/result/tls_verylong_certificate.pcap.out @@ -0,0 +1,30 @@ +Guessed flow protos: 0 + +DPI Packets (TCP): 5 (5.00 pkts/flow) +Confidence DPI : 1 (flows) +Num dissector calls: 1 (1.00 diss/flow) +LRU cache ookla: 0/0/0 (insert/search/found) +LRU cache bittorrent: 0/0/0 (insert/search/found) +LRU cache zoom: 0/0/0 (insert/search/found) +LRU cache stun: 0/0/0 (insert/search/found) +LRU cache tls_cert: 0/0/0 (insert/search/found) +LRU cache mining: 0/0/0 (insert/search/found) +LRU cache msteams: 0/0/0 (insert/search/found) +LRU cache stun_zoom: 0/0/0 (insert/search/found) +Automa host: 1/1 (search/found) +Automa domain: 1/0 (search/found) +Automa tls cert: 0/0 (search/found) +Automa risk mask: 0/0 (search/found) +Automa common alpns: 1/1 (search/found) +Patricia risk mask: 0/0 (search/found) +Patricia risk: 0/0 (search/found) +Patricia protocols: 2/0 (search/found) + +Cybersec 48 22229 1 + +JA3 Host Stats: + IP Address # JA3C + 1 192.168.1.160 1 + + + 1 TCP 192.168.1.160:54804 <-> 151.101.66.49:443 [proto: 91.283/TLS.Cybersec][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 5][cat: Cybersecurity/33][24 pkts/2404 bytes <-> 24 pkts/19825 bytes][Goodput ratio: 35/92][0.09 sec][Hostname/SNI: feodotracker.abuse.ch][(Advertised) ALPNs: http/1.1][bytes ratio: -0.784 (Download)][IAT c2s/s2c min/avg/max/stddev: 0/0 4/4 15/21 5/7][Pkt Len c2s/s2c min/avg/max/stddev: 54/66 100/826 583/1434 109/662][TLSv1.2][JA3C: 2a26b1a62e40d25d4de3babc9d532f30][Firefox][Plen Bins: 12,16,0,4,0,4,4,0,0,0,0,0,0,0,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,55,0,0,0,0,0] diff --git a/windows/src/ndpi_define.h b/windows/src/ndpi_define.h index 5a47f925408..4fea61aa0fa 100644 --- a/windows/src/ndpi_define.h +++ b/windows/src/ndpi_define.h @@ -152,7 +152,6 @@ /* misc definitions */ #define NDPI_DEFAULT_MAX_TCP_RETRANSMISSION_WINDOW_SIZE 0x10000 -#define NDPI_DEFAULT_MAX_NUM_PKTS_PER_FLOW_TO_DISSECT 32 /* TODO: rebuild all memory areas to have a more aligned memory block here */