diff --git a/.gitignore b/.gitignore index e496d1b5340..c494b9ff71e 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ /fuzz/fuzz_gcrypt_cipher /fuzz/fuzz_ndpi_reader_payload_analyzer /fuzz/fuzz_filecfg_protocols +/fuzz/fuzz_filecfg_categories /fuzz/fuzz_readerutils_workflow /fuzz/fuzz_readerutils_parseprotolist /fuzz/fuzz_ndpi_reader_alloc_fail_seed_corpus.zip @@ -108,6 +109,7 @@ /fuzz/fuzz_libinjection_seed_corpus.zip /fuzz/fuzz_tls_certificate_seed_corpus.zip /fuzz/fuzz_filecfg_protocols_seed_corpus.zip +/fuzz/fuzz_filecfg_categories_seed_corpus.zip /fuzz/fuzz_dga_seed_corpus.zip /fuzz/fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip /fuzz/fuzz_readerutils_workflow_seed_corpus.zip diff --git a/example/categories.txt b/example/categories.txt index 6c5171c5387..815efad7404 100644 --- a/example/categories.txt +++ b/example/categories.txt @@ -1,3 +1,6 @@ # Format: name\tcategory_id internetbadguys.com 100 144.139.247.220 100 +172.30.69.0/24 100 +[2001:1670:8:40a6:a08e:332b:aa69:18dc] 100 +[2001:db8:1::1]/127 100 diff --git a/fuzz/Makefile.am b/fuzz/Makefile.am index 5f6fbb71cd2..360b39a46db 100644 --- a/fuzz/Makefile.am +++ b/fuzz/Makefile.am @@ -8,7 +8,7 @@ bin_PROGRAMS += fuzz_libinjection fuzz_binaryfusefilter #Internal crypto bin_PROGRAMS += fuzz_gcrypt_light fuzz_gcrypt_aes fuzz_gcrypt_gcm fuzz_gcrypt_cipher #Configuration files -bin_PROGRAMS += fuzz_filecfg_protocols +bin_PROGRAMS += fuzz_filecfg_protocols fuzz_filecfg_categories #Reader utils bin_PROGRAMS += fuzz_readerutils_workflow fuzz_readerutils_parseprotolist @@ -485,6 +485,19 @@ fuzz_filecfg_protocols_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \ $(fuzz_filecfg_protocols_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@ +fuzz_filecfg_categories_SOURCES = fuzz_filecfg_categories.c fuzz_common_code.c +fuzz_filecfg_categories_CFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS) +fuzz_filecfg_categories_LDADD = ../src/lib/libndpi.a $(ADDITIONAL_LIBS) +fuzz_filecfg_categories_LDFLAGS = $(LIBS) +if HAS_FUZZLDFLAGS +fuzz_filecfg_categories_CFLAGS += $(LIB_FUZZING_ENGINE) +fuzz_filecfg_categories_LDFLAGS += $(LIB_FUZZING_ENGINE) +endif +# force usage of CXX for linker +fuzz_filecfg_categories_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \ + $(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \ + $(fuzz_filecfg_categories_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@ + fuzz_readerutils_workflow_SOURCES = fuzz_readerutils_workflow.cpp fuzz_common_code.c ../example/reader_util.c fuzz_readerutils_workflow_CXXFLAGS = -I../example/ @NDPI_CFLAGS@ $(CXXFLAGS) -DDISABLE_CUSTOM_ALLOCATOR_ON_READERUTILS fuzz_readerutils_workflow_CFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS) -DDISABLE_CUSTOM_ALLOCATOR_ON_READERUTILS @@ -606,6 +619,10 @@ files_corpus_fuzz_filecfg_protocols := $(wildcard corpus/fuzz_filecfg_protocols fuzz_filecfg_protocols_seed_corpus.zip: $(files_corpus_fuzz_filecfg_protocols) zip -j fuzz_filecfg_protocols_seed_corpus.zip $(files_corpus_fuzz_filecfg_protocols) +files_corpus_fuzz_filecfg_categories := $(wildcard corpus/fuzz_filecfg_categories/*) +fuzz_filecfg_categories_seed_corpus.zip: $(files_corpus_fuzz_filecfg_categories) + zip -j fuzz_filecfg_categories_seed_corpus.zip $(files_corpus_fuzz_filecfg_categories) + files_corpus_fuzz_readerutils_workflow := $(wildcard corpus/fuzz_readerutils_workflow/*) fuzz_readerutils_workflow_seed_corpus.zip: $(files_corpus_fuzz_readerutils_workflow) zip -j fuzz_readerutils_workflow_seed_corpus.zip $(files_corpus_fuzz_readerutils_workflow) @@ -622,7 +639,7 @@ files_corpus_fuzz_ds_domain_classify := $(wildcard corpus/fuzz_ds_domain_classi fuzz_ds_domain_classify_seed_corpus.zip: $(files_corpus_fuzz_ds_domain_classify) zip -j fuzz_ds_domain_classify_seed_corpus.zip $(files_corpus_fuzz_ds_domain_classify) -corpus: fuzz_ndpi_reader_seed_corpus.zip fuzz_ndpi_reader_alloc_fail_seed_corpus.zip fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip fuzz_quic_get_crypto_data_seed_corpus.zip fuzz_config_seed_corpus.zip fuzz_ds_patricia_seed_corpus.zip fuzz_ds_ahocorasick_seed_corpus.zip fuzz_alg_ses_des_seed_corpus.zip fuzz_alg_hw_rsi_outliers_da_seed_corpus.zip fuzz_alg_bins_seed_corpus.zip fuzz_alg_hll_seed_corpus.zip fuzz_alg_jitter_seed_corpus.zip fuzz_ds_libcache_seed_corpus.zip fuzz_community_id_seed_corpus.zip fuzz_ds_tree_seed_corpus.zip fuzz_serialization_seed_corpus.zip fuzz_ds_ptree_seed_corpus.zip fuzz_alg_crc32_md5_seed_corpus.zip fuzz_alg_bytestream_seed_corpus.zip fuzz_libinjection_seed_corpus.zip fuzz_tls_certificate_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_readerutils_workflow_seed_corpus.zip fuzz_readerutils_parseprotolist_seed_corpus.zip fuzz_ds_bitmap64_seed_corpus.zip fuzz_ds_domain_classify_seed_corpus.zip +corpus: fuzz_ndpi_reader_seed_corpus.zip fuzz_ndpi_reader_alloc_fail_seed_corpus.zip fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip fuzz_quic_get_crypto_data_seed_corpus.zip fuzz_config_seed_corpus.zip fuzz_ds_patricia_seed_corpus.zip fuzz_ds_ahocorasick_seed_corpus.zip fuzz_alg_ses_des_seed_corpus.zip fuzz_alg_hw_rsi_outliers_da_seed_corpus.zip fuzz_alg_bins_seed_corpus.zip fuzz_alg_hll_seed_corpus.zip fuzz_alg_jitter_seed_corpus.zip fuzz_ds_libcache_seed_corpus.zip fuzz_community_id_seed_corpus.zip fuzz_ds_tree_seed_corpus.zip fuzz_serialization_seed_corpus.zip fuzz_ds_ptree_seed_corpus.zip fuzz_alg_crc32_md5_seed_corpus.zip fuzz_alg_bytestream_seed_corpus.zip fuzz_libinjection_seed_corpus.zip fuzz_tls_certificate_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_readerutils_workflow_seed_corpus.zip fuzz_readerutils_parseprotolist_seed_corpus.zip fuzz_ds_bitmap64_seed_corpus.zip fuzz_ds_domain_classify_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip cp corpus/fuzz_*seed_corpus.zip . #Create dictionaries exactly as expected by oss-fuzz. @@ -651,6 +668,7 @@ distdir: -o -path './corpus/fuzz_*.zip' \ -o -path './corpus/fuzz_quic_get_crypto_data/*' \ -o -path './corpus/fuzz_filecfg_protocols/*' \ + -o -path './corpus/fuzz_filecfg_categories/*' \ -o -path './corpus/fuzz_readerutils_workflow/*' \ -o -path './corpus/fuzz_readerutils_parseprotolist/*' \ -o -path './corpus/fuzz_config/*' \ diff --git a/fuzz/corpus/fuzz_filecfg_categories/domain.txt b/fuzz/corpus/fuzz_filecfg_categories/domain.txt new file mode 100644 index 00000000000..2598e8dc4a3 --- /dev/null +++ b/fuzz/corpus/fuzz_filecfg_categories/domain.txt @@ -0,0 +1,2 @@ +# Format: name\tcategory_id +internetbadguys.com 100 diff --git a/fuzz/corpus/fuzz_filecfg_categories/ipv4.txt b/fuzz/corpus/fuzz_filecfg_categories/ipv4.txt new file mode 100644 index 00000000000..5f44509573f --- /dev/null +++ b/fuzz/corpus/fuzz_filecfg_categories/ipv4.txt @@ -0,0 +1 @@ +144.139.247.220 100 diff --git a/fuzz/corpus/fuzz_filecfg_categories/ipv4_prefix.txt b/fuzz/corpus/fuzz_filecfg_categories/ipv4_prefix.txt new file mode 100644 index 00000000000..40758d9e53b --- /dev/null +++ b/fuzz/corpus/fuzz_filecfg_categories/ipv4_prefix.txt @@ -0,0 +1 @@ +144.139.247.220/24 100 diff --git a/fuzz/corpus/fuzz_filecfg_categories/ipv6.txt b/fuzz/corpus/fuzz_filecfg_categories/ipv6.txt new file mode 100644 index 00000000000..4a638c09af1 --- /dev/null +++ b/fuzz/corpus/fuzz_filecfg_categories/ipv6.txt @@ -0,0 +1 @@ +[3ffe:507:0:1:200:86ff:fe05:80da] 100 diff --git a/fuzz/corpus/fuzz_filecfg_categories/ipv6_prefix.txt b/fuzz/corpus/fuzz_filecfg_categories/ipv6_prefix.txt new file mode 100644 index 00000000000..05adf16028f --- /dev/null +++ b/fuzz/corpus/fuzz_filecfg_categories/ipv6_prefix.txt @@ -0,0 +1 @@ +[3ffe:507:0:1:200:86ff:fe05:80da]/64 100 diff --git a/fuzz/fuzz_config.cpp b/fuzz/fuzz_config.cpp index 53d5c2fd825..53df3a8c322 100644 --- a/fuzz/fuzz_config.cpp +++ b/fuzz/fuzz_config.cpp @@ -220,6 +220,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { flow.l4_proto, flow.c_address.v4, flow.c_port, flow.s_address.v4, flow.s_port); + } else { + ndpi_find_ipv6_category_userdata(ndpi_info_mod, (struct in6_addr *)flow.c_address.v6); } /* Another "strange" function: fuzz it here, for lack of a better alternative */ ndpi_search_tcp_or_udp(ndpi_info_mod, &flow); diff --git a/fuzz/fuzz_filecfg_categories.c b/fuzz/fuzz_filecfg_categories.c new file mode 100644 index 00000000000..d2e43c99cea --- /dev/null +++ b/fuzz/fuzz_filecfg_categories.c @@ -0,0 +1,46 @@ +#include "ndpi_api.h" +#include "fuzz_common_code.h" + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + struct ndpi_detection_module_struct *ndpi_struct; + FILE *fd; + /* Try to be fast */ + ndpi_init_prefs prefs = ndpi_dont_load_tor_list | + ndpi_dont_load_azure_list | + ndpi_dont_load_whatsapp_list | + ndpi_dont_load_amazon_aws_list | + ndpi_dont_load_ethereum_list | + ndpi_dont_load_zoom_list | + ndpi_dont_load_cloudflare_list | + ndpi_dont_load_microsoft_list | + ndpi_dont_load_google_list | + ndpi_dont_load_google_cloud_list | + ndpi_dont_load_asn_lists | + ndpi_dont_init_risk_ptree | + ndpi_dont_load_cachefly_list | + ndpi_dont_load_protonvpn_list | + ndpi_dont_load_mullvad_list; + NDPI_PROTOCOL_BITMASK all; + NDPI_PROTOCOL_BITMASK debug_bitmask; + + /* To allow memory allocation failures */ + fuzz_set_alloc_callbacks_and_seed(size); + + ndpi_struct = ndpi_init_detection_module(prefs); + NDPI_BITMASK_SET_ALL(all); + ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all); + + NDPI_BITMASK_SET_ALL(debug_bitmask); + ndpi_set_log_level(ndpi_struct, 4); + ndpi_set_debug_bitmask(ndpi_struct, debug_bitmask); + + fd = buffer_to_file(data, size); + ndpi_load_categories_file2(ndpi_struct, fd, NULL); + if(fd) + fclose(fd); + + /* We don't really need to call ndpi_finalize_initialization */ + + ndpi_exit_detection_module(ndpi_struct); + return 0; +} diff --git a/src/include/ndpi_api.h b/src/include/ndpi_api.h index 35fbde85cdb..ee77b537aa9 100644 --- a/src/include/ndpi_api.h +++ b/src/include/ndpi_api.h @@ -810,6 +810,7 @@ extern "C" { * -1 else */ int ndpi_load_categories_file(struct ndpi_detection_module_struct *ndpi_str, const char* path, void *user_data); + int ndpi_load_categories_file2(struct ndpi_detection_module_struct *ndpi_str, FILE *fd, void *user_data); /** * Loads a file (separated by ) of domain names associated with the specified category @@ -1028,11 +1029,17 @@ extern "C" { int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_struct); void* ndpi_find_ipv4_category_userdata(struct ndpi_detection_module_struct *ndpi_str, u_int32_t saddr); + void* ndpi_find_ipv6_category_userdata(struct ndpi_detection_module_struct *ndpi_str, + struct in6_addr *saddr); int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_struct, struct ndpi_flow_struct *flow, u_int32_t saddr, u_int32_t daddr, ndpi_protocol *ret); + int ndpi_fill_ip6_protocol_category(struct ndpi_detection_module_struct *ndpi_str, + struct ndpi_flow_struct *flow, + struct in6_addr *saddr, struct in6_addr *daddr, + ndpi_protocol *ret); int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct, char *name, u_int name_len, ndpi_protocol_category_t *id); void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struct, diff --git a/src/include/ndpi_typedefs.h b/src/include/ndpi_typedefs.h index 25702308027..365638aa38b 100644 --- a/src/include/ndpi_typedefs.h +++ b/src/include/ndpi_typedefs.h @@ -1327,6 +1327,7 @@ struct ndpi_detection_module_struct { ndpi_domain_classify *sc_hostnames, *sc_hostnames_shadow; #endif void *ipAddresses, *ipAddresses_shadow; /* Patricia */ + void *ipAddresses6, *ipAddresses6_shadow; /* Patricia IPv6*/ u_int8_t categories_loaded; } custom_categories; diff --git a/src/lib/ndpi_main.c b/src/lib/ndpi_main.c index 16cffe7868a..a0fb3a1a6cd 100644 --- a/src/lib/ndpi_main.c +++ b/src/lib/ndpi_main.c @@ -3267,6 +3267,8 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs ndpi_str->custom_categories.ipAddresses = ndpi_patricia_new(32 /* IPv4 */); ndpi_str->custom_categories.ipAddresses_shadow = ndpi_patricia_new(32 /* IPv4 */); + ndpi_str->custom_categories.ipAddresses6 = ndpi_patricia_new(128 /* IPv6 */); + ndpi_str->custom_categories.ipAddresses6_shadow = ndpi_patricia_new(128 /* IPv6 */); if(ndpi_str->host_automa.ac_automa) ac_automata_feature(ndpi_str->host_automa.ac_automa,AC_FEATURE_LC); @@ -3294,7 +3296,8 @@ struct ndpi_detection_module_struct *ndpi_init_detection_module(ndpi_init_prefs if(ndpi_str->common_alpns_automa.ac_automa) ac_automata_name(ndpi_str->common_alpns_automa.ac_automa,"content",AC_FEATURE_DEBUG); - if((ndpi_str->custom_categories.ipAddresses == NULL) || (ndpi_str->custom_categories.ipAddresses_shadow == NULL)) { + if((ndpi_str->custom_categories.ipAddresses == NULL) || (ndpi_str->custom_categories.ipAddresses_shadow == NULL) || + (ndpi_str->custom_categories.ipAddresses6 == NULL) || (ndpi_str->custom_categories.ipAddresses6_shadow == NULL)) { NDPI_LOG_ERR(ndpi_str, "[NDPI] Error allocating Patricia trees\n"); ndpi_exit_detection_module(ndpi_str); return(NULL); @@ -3686,6 +3689,9 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str ndpi_protocol_category_t *id) { char ipbuf[64], *ptr; struct in_addr pin; + struct in6_addr pin6; + ndpi_prefix_t prefix; + ndpi_patricia_node_t *node; u_int cp_len = ndpi_min(sizeof(ipbuf) - 1, name_len); if(!ndpi_str->custom_categories.categories_loaded) @@ -3694,8 +3700,9 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str if(cp_len > 0) { memcpy(ipbuf, name_or_ip, cp_len); ipbuf[cp_len] = '\0'; - } else + } else { ipbuf[0] = '\0'; + } ptr = strrchr(ipbuf, '/'); @@ -3703,9 +3710,7 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str ptr[0] = '\0'; if(inet_pton(AF_INET, ipbuf, &pin) == 1) { - /* Search IP */ - ndpi_prefix_t prefix; - ndpi_patricia_node_t *node; + /* Search IPv4 */ /* Make sure all in network byte order otherwise compares wont work */ ndpi_fill_prefix_v4(&prefix, &pin, 32, ((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses)->maxbits); @@ -3713,10 +3718,18 @@ int ndpi_get_custom_category_match(struct ndpi_detection_module_struct *ndpi_str if(node) { *id = node->value.u.uv32.user_value; - return(0); } + return(-1); + } else if(inet_pton(AF_INET6, ipbuf, &pin6) == 1) { + /* Search IPv6 */ + ndpi_fill_prefix_v6(&prefix, &pin6, 128, ((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6)->maxbits); + node = ndpi_patricia_search_best(ndpi_str->custom_categories.ipAddresses6, &prefix); + if(node) { + *id = node->value.u.uv32.user_value; + return(0); + } return(-1); } else { /* Search Host */ @@ -3827,6 +3840,12 @@ void ndpi_exit_detection_module(struct ndpi_detection_module_struct *ndpi_str) { if(ndpi_str->custom_categories.ipAddresses_shadow != NULL) ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses_shadow, free_ptree_data); + if(ndpi_str->custom_categories.ipAddresses6 != NULL) + ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6, free_ptree_data); + + if(ndpi_str->custom_categories.ipAddresses6_shadow != NULL) + ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6_shadow, free_ptree_data); + if(ndpi_str->host_risk_mask_automa.ac_automa != NULL) ac_automata_release((AC_AUTOMATA_t *) ndpi_str->host_risk_mask_automa.ac_automa, 1 /* free patterns strings memory */); @@ -4384,20 +4403,33 @@ int ndpi_handle_rule(struct ndpi_detection_module_struct *ndpi_str, */ int ndpi_load_categories_file(struct ndpi_detection_module_struct *ndpi_str, const char *path, void *user_data) { - char buffer[512], *line, *name, *category, *saveptr; + int rc; FILE *fd; - int len, num = 0; if(!ndpi_str || !path) return(-1); fd = fopen(path, "r"); - if(fd == NULL) { NDPI_LOG_ERR(ndpi_str, "Unable to open file %s [%s]\n", path, strerror(errno)); - return(-1); + return -1; } + rc = ndpi_load_categories_file2(ndpi_str, fd, user_data); + + fclose(fd); + + return rc; +} + +int ndpi_load_categories_file2(struct ndpi_detection_module_struct *ndpi_str, + FILE *fd, void *user_data) { + char buffer[512], *line, *name, *category, *saveptr; + int len, num = 0; + + if(!ndpi_str || !fd) + return(-1); + while(1) { line = fgets(buffer, sizeof(buffer), fd); @@ -4426,8 +4458,6 @@ int ndpi_load_categories_file(struct ndpi_detection_module_struct *ndpi_str, } } - fclose(fd); - /* Not necessay to call ndpi_enable_loaded_categories() as ndpi_set_protocol_detection_bitmask2() will do that @@ -7139,14 +7169,16 @@ int ndpi_load_ip_category(struct ndpi_detection_module_struct *ndpi_str, const char *ip_address_and_mask, ndpi_protocol_category_t category, void *user_data) { - ndpi_patricia_node_t *node; - struct in_addr pin; - int bits = 32; + ndpi_patricia_node_t *node = NULL; + int bits = 32, is_ipv6 = 0; char *ptr; - char ipbuf[64]; + char ipbuf[128]; - if(!ndpi_str->custom_categories.ipAddresses_shadow) - return(-1); + if(ip_address_and_mask[0] == '[') { + is_ipv6 = 1; + bits = 128; + ip_address_and_mask++; /* Strip '[' */ + } strncpy(ipbuf, ip_address_and_mask, sizeof(ipbuf) - 1); ipbuf[sizeof(ipbuf) - 1] = '\0'; @@ -7158,13 +7190,31 @@ int ndpi_load_ip_category(struct ndpi_detection_module_struct *ndpi_str, if(atoi(ptr) >= 0 && atoi(ptr) <= 32) bits = atoi(ptr); } + ptr = strrchr(ipbuf, ']'); + if(ptr) + *ptr = '\0'; /* Strip ']' */ + + if(!is_ipv6 && ndpi_str->custom_categories.ipAddresses_shadow) { + struct in_addr pin; - if(inet_pton(AF_INET, ipbuf, &pin) != 1) { - NDPI_LOG_DBG2(ndpi_str, "Invalid ip/ip+netmask: %s\n", ip_address_and_mask); + if(inet_pton(AF_INET, ipbuf, &pin) != 1) { + NDPI_LOG_DBG2(ndpi_str, "Invalid ip4/ip4+netmask: %s\n", ip_address_and_mask); + return(-1); + } + node = add_to_ptree(ndpi_str->custom_categories.ipAddresses_shadow, AF_INET, &pin, bits); + } else if(is_ipv6 && ndpi_str->custom_categories.ipAddresses6_shadow) { + struct in6_addr pin6; + + if(inet_pton(AF_INET6, ipbuf, &pin6) != 1) { + NDPI_LOG_DBG2(ndpi_str, "Invalid ip6/ip6+netmask: %s\n", ip_address_and_mask); + return(-1); + } + node = add_to_ptree(ndpi_str->custom_categories.ipAddresses6_shadow, AF_INET6, &pin6, bits); + } else { return(-1); } - if((node = add_to_ptree(ndpi_str->custom_categories.ipAddresses_shadow, AF_INET, &pin, bits)) != NULL) { + if(node != NULL) { node->value.u.uv32.user_value = (u_int16_t)category, node->value.u.uv32.additional_user_value = 0; node->custom_user_data = user_data; } @@ -7260,9 +7310,15 @@ int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_str) if(ndpi_str->custom_categories.ipAddresses != NULL) ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses, free_ptree_data); + if(ndpi_str->custom_categories.ipAddresses6 != NULL) + ndpi_patricia_destroy((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6, free_ptree_data); + ndpi_str->custom_categories.ipAddresses = ndpi_str->custom_categories.ipAddresses_shadow; ndpi_str->custom_categories.ipAddresses_shadow = ndpi_patricia_new(32 /* IPv4 */); + ndpi_str->custom_categories.ipAddresses6 = ndpi_str->custom_categories.ipAddresses6_shadow; + ndpi_str->custom_categories.ipAddresses6_shadow = ndpi_patricia_new(128 /* IPv6 */); + ndpi_str->custom_categories.categories_loaded = 1; return(0); @@ -7288,6 +7344,26 @@ void* ndpi_find_ipv4_category_userdata(struct ndpi_detection_module_struct *ndpi return(node ? node->custom_user_data : NULL); } +/* ********************************************************************************* */ + +void* ndpi_find_ipv6_category_userdata(struct ndpi_detection_module_struct *ndpi_str, + struct in6_addr *saddr) { + ndpi_patricia_node_t *node; + + if(!saddr || !ndpi_str || !ndpi_str->custom_categories.ipAddresses6) + node = NULL; + else { + ndpi_prefix_t prefix; + + ndpi_fill_prefix_v6(&prefix, saddr, 128, + ((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6)->maxbits); + node = ndpi_patricia_search_best(ndpi_str->custom_categories.ipAddresses6, &prefix); + } + + return(node ? node->custom_user_data : NULL); +} + + /* ********************************************************************************* */ /* NOTE u_int32_t is represented in network byte order */ @@ -7344,6 +7420,52 @@ int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_str /* ********************************************************************************* */ +int ndpi_fill_ip6_protocol_category(struct ndpi_detection_module_struct *ndpi_str, + struct ndpi_flow_struct *flow, + struct in6_addr *saddr, struct in6_addr *daddr, + ndpi_protocol *ret) { + bool match_client = true; + + ret->custom_category_userdata = NULL; + + if(ndpi_str->custom_categories.categories_loaded && + ndpi_str->custom_categories.ipAddresses6) { + + ndpi_prefix_t prefix; + ndpi_patricia_node_t *node; + + ndpi_fill_prefix_v6(&prefix, saddr, 128, + ((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6)->maxbits); + node = ndpi_patricia_search_best(ndpi_str->custom_categories.ipAddresses6, &prefix); + + if(node == NULL) { + ndpi_fill_prefix_v6(&prefix, daddr, 128, + ((ndpi_patricia_tree_t *) ndpi_str->custom_categories.ipAddresses6)->maxbits); + node = ndpi_patricia_search_best(ndpi_str->custom_categories.ipAddresses6, &prefix); + match_client = false; + } else { + match_client = true; + } + + if(node) { + ret->category = (ndpi_protocol_category_t) node->value.u.uv32.user_value; + ret->custom_category_userdata = node->custom_user_data; + + if((ret->category == CUSTOM_CATEGORY_MALWARE) && (match_client == false)) { + ndpi_set_risk(ndpi_str, flow, NDPI_MALWARE_HOST_CONTACTED, "Client contacted malware host"); + } + + return(1); + } + } + + ret->category = ndpi_get_proto_category(ndpi_str, *ret); + + return(0); +} + +/* ********************************************************************************* */ + void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_str, struct ndpi_flow_struct *flow, ndpi_protocol *ret) { if((ret->master_protocol == NDPI_PROTOCOL_UNKNOWN) && (ret->app_protocol == NDPI_PROTOCOL_UNKNOWN)) @@ -7442,9 +7564,11 @@ static int ndpi_do_guess(struct ndpi_detection_module_struct *ndpi_str, struct n ret->protocol_by_ip = flow->guessed_protocol_id_by_ip; - if(ndpi_str->custom_categories.categories_loaded && packet->iph) { - if(ndpi_str->ndpi_num_custom_protocols != 0) + if(ndpi_str->custom_categories.categories_loaded) { + if(packet->iph) ndpi_fill_ip_protocol_category(ndpi_str, flow, flow->c_address.v4, flow->s_address.v4, ret); + else + ndpi_fill_ip6_protocol_category(ndpi_str, flow, (struct in6_addr *)flow->c_address.v6, (struct in6_addr *)flow->s_address.v6, ret); flow->guessed_header_category = ret->category; } else flow->guessed_header_category = NDPI_PROTOCOL_CATEGORY_UNSPECIFIED; diff --git a/tests/cfgs/default/pcap/custom_categories.pcapng b/tests/cfgs/default/pcap/custom_categories.pcapng new file mode 100644 index 00000000000..293565ff844 Binary files /dev/null and b/tests/cfgs/default/pcap/custom_categories.pcapng differ diff --git a/tests/cfgs/default/result/custom_categories.pcapng.out b/tests/cfgs/default/result/custom_categories.pcapng.out new file mode 100644 index 00000000000..25423013227 --- /dev/null +++ b/tests/cfgs/default/result/custom_categories.pcapng.out @@ -0,0 +1,32 @@ +Guessed flow protos: 0 + +DPI Packets (TCP): 26 (13.00 pkts/flow) +DPI Packets (other): 1 (1.00 pkts/flow) +Confidence DPI : 3 (flows) +Num dissector calls: 23 (7.67 diss/flow) +LRU cache ookla: 0/0/0 (insert/search/found) +LRU cache bittorrent: 0/0/0 (insert/search/found) +LRU cache zoom: 0/0/0 (insert/search/found) +LRU cache stun: 0/0/0 (insert/search/found) +LRU cache tls_cert: 0/0/0 (insert/search/found) +LRU cache mining: 0/0/0 (insert/search/found) +LRU cache msteams: 0/0/0 (insert/search/found) +LRU cache stun_zoom: 0/0/0 (insert/search/found) +Automa host: 0/0 (search/found) +Automa domain: 0/0 (search/found) +Automa tls cert: 0/0 (search/found) +Automa risk mask: 0/0 (search/found) +Automa common alpns: 0/0 (search/found) +Patricia risk mask: 2/0 (search/found) +Patricia risk mask IPv6: 2/0 (search/found) +Patricia risk: 0/0 (search/found) +Patricia risk IPv6: 2/0 (search/found) +Patricia protocols: 2/0 (search/found) +Patricia protocols IPv6: 4/0 (search/found) + +IPSec 1 346 1 +SSH 84 14188 2 + + 1 TCP [2001:db8:1::1]:64720 <-> [2001:db8:200::1]:20868 [proto: 92/SSH][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 16][cat: Malware/100][32 pkts/3639 bytes <-> 30 pkts/6335 bytes][Goodput ratio: 24/59][5.34 sec][Hostname/SNI: SSH-1.5-1.2.26][bytes ratio: -0.270 (Download)][IAT c2s/s2c min/avg/max/stddev: 13/74 184/193 1212/1436 234/283][Pkt Len c2s/s2c min/avg/max/stddev: 86/86 114/211 250/1294 47/257][Risk: ** Known Proto on Non Std Port **][Risk Score: 50][Server: SSH-1.5-1.2.26][Plen Bins: 69,6,0,0,11,2,0,0,2,0,0,0,0,0,2,0,0,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0] + 2 TCP 172.26.219.44:58639 <-> 172.30.69.103:22 [proto: 92/SSH][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 10][cat: Malware/100][11 pkts/2011 bytes <-> 11 pkts/2203 bytes][Goodput ratio: 63/67][0.11 sec][Hostname/SNI: SSH-1.99-OpenSSH_4.3][bytes ratio: -0.046 (Mixed)][IAT c2s/s2c min/avg/max/stddev: 0/0 7/7 39/41 12/13][Pkt Len c2s/s2c min/avg/max/stddev: 66/66 183/200 1026/770 270/223][Risk: ** SSH Obsolete Cli Vers/Cipher **** SSH Obsolete Ser Vers/Cipher **** Client contacted a malware host **][Risk Score: 300][Risk Info: Client contacted malware host / Found cipher arcfour128 / Found cipher arcfour128][HASSH-C: D6593B3202A30B2AA9793A00F8647A0A][Server: SSH-2.0-OpenSSH_6.1][HASSH-S: 500033A73A293E7C36743693D0D4596B][Plen Bins: 31,15,15,0,15,0,0,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0] + 3 ESP [2a01:e34:ef6f:4340:94be:5dac:c20a:d2a0]:0 -> [2001:1670:8:40a6:a08e:332b:aa69:18dc]:0 [VLAN: 121][proto: 79/IPSec][IP: 0/Unknown][Encrypted][Confidence: DPI][DPI packets: 1][cat: Malware/100][1 pkts/346 bytes -> 0 pkts/0 bytes][Goodput ratio: 0/0][< 1 sec][Risk: ** Unidirectional Traffic **** Client contacted a malware host **][Risk Score: 160][Risk Info: No server to client traffic / Client contacted malware host][Plen Bins: 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]