IPv6: add support for custom categories (#2126)

.gitignore

@@ -83,6 +83,7 @@
 /fuzz/fuzz_gcrypt_cipher
 /fuzz/fuzz_ndpi_reader_payload_analyzer
 /fuzz/fuzz_filecfg_protocols
+/fuzz/fuzz_filecfg_categories
 /fuzz/fuzz_readerutils_workflow
 /fuzz/fuzz_readerutils_parseprotolist
 /fuzz/fuzz_ndpi_reader_alloc_fail_seed_corpus.zip
@@ -108,6 +109,7 @@
 /fuzz/fuzz_libinjection_seed_corpus.zip
 /fuzz/fuzz_tls_certificate_seed_corpus.zip
 /fuzz/fuzz_filecfg_protocols_seed_corpus.zip
+/fuzz/fuzz_filecfg_categories_seed_corpus.zip
 /fuzz/fuzz_dga_seed_corpus.zip
 /fuzz/fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip
 /fuzz/fuzz_readerutils_workflow_seed_corpus.zip

example/categories.txt

@@ -1,3 +1,6 @@
 #  Format: name\tcategory_id
 internetbadguys.com	100
 144.139.247.220	100
+172.30.69.0/24	100
+[2001:1670:8:40a6:a08e:332b:aa69:18dc]	100
+[2001:db8:1::1]/127	100

fuzz/Makefile.am

@@ -8,7 +8,7 @@ bin_PROGRAMS += fuzz_libinjection fuzz_binaryfusefilter
 #Internal crypto
 bin_PROGRAMS += fuzz_gcrypt_light fuzz_gcrypt_aes fuzz_gcrypt_gcm fuzz_gcrypt_cipher
 #Configuration files
-bin_PROGRAMS += fuzz_filecfg_protocols
+bin_PROGRAMS += fuzz_filecfg_protocols fuzz_filecfg_categories
 #Reader utils
 bin_PROGRAMS += fuzz_readerutils_workflow fuzz_readerutils_parseprotolist
 
@@ -485,6 +485,19 @@ fuzz_filecfg_protocols_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
     $(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \
     $(fuzz_filecfg_protocols_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
 
+fuzz_filecfg_categories_SOURCES = fuzz_filecfg_categories.c fuzz_common_code.c
+fuzz_filecfg_categories_CFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS)
+fuzz_filecfg_categories_LDADD = ../src/lib/libndpi.a $(ADDITIONAL_LIBS)
+fuzz_filecfg_categories_LDFLAGS = $(LIBS)
+if HAS_FUZZLDFLAGS
+fuzz_filecfg_categories_CFLAGS += $(LIB_FUZZING_ENGINE)
+fuzz_filecfg_categories_LDFLAGS += $(LIB_FUZZING_ENGINE)
+endif
+# force usage of CXX for linker
+fuzz_filecfg_categories_LINK=$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) \
+    $(LIBTOOLFLAGS) --mode=link $(CXX) @NDPI_CFLAGS@ $(AM_CXXFLAGS) $(CXXFLAGS) \
+    $(fuzz_filecfg_categories_LDFLAGS) @NDPI_LDFLAGS@ $(LDFLAGS) -o $@
+
 fuzz_readerutils_workflow_SOURCES = fuzz_readerutils_workflow.cpp fuzz_common_code.c ../example/reader_util.c
 fuzz_readerutils_workflow_CXXFLAGS = -I../example/ @NDPI_CFLAGS@ $(CXXFLAGS) -DDISABLE_CUSTOM_ALLOCATOR_ON_READERUTILS
 fuzz_readerutils_workflow_CFLAGS = @NDPI_CFLAGS@ $(CXXFLAGS) -DDISABLE_CUSTOM_ALLOCATOR_ON_READERUTILS
@@ -606,6 +619,10 @@ files_corpus_fuzz_filecfg_protocols :=  $(wildcard corpus/fuzz_filecfg_protocols
 fuzz_filecfg_protocols_seed_corpus.zip: $(files_corpus_fuzz_filecfg_protocols)
 	zip -j fuzz_filecfg_protocols_seed_corpus.zip $(files_corpus_fuzz_filecfg_protocols)
 
+files_corpus_fuzz_filecfg_categories :=  $(wildcard corpus/fuzz_filecfg_categories/*)
+fuzz_filecfg_categories_seed_corpus.zip: $(files_corpus_fuzz_filecfg_categories)
+	zip -j fuzz_filecfg_categories_seed_corpus.zip $(files_corpus_fuzz_filecfg_categories)
+
 files_corpus_fuzz_readerutils_workflow :=  $(wildcard corpus/fuzz_readerutils_workflow/*)
 fuzz_readerutils_workflow_seed_corpus.zip: $(files_corpus_fuzz_readerutils_workflow)
 	zip -j fuzz_readerutils_workflow_seed_corpus.zip $(files_corpus_fuzz_readerutils_workflow)
@@ -622,7 +639,7 @@ files_corpus_fuzz_ds_domain_classify :=  $(wildcard corpus/fuzz_ds_domain_classi
 fuzz_ds_domain_classify_seed_corpus.zip: $(files_corpus_fuzz_ds_domain_classify)
 	zip -j fuzz_ds_domain_classify_seed_corpus.zip $(files_corpus_fuzz_ds_domain_classify)
 
-corpus: fuzz_ndpi_reader_seed_corpus.zip fuzz_ndpi_reader_alloc_fail_seed_corpus.zip fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip fuzz_quic_get_crypto_data_seed_corpus.zip fuzz_config_seed_corpus.zip fuzz_ds_patricia_seed_corpus.zip fuzz_ds_ahocorasick_seed_corpus.zip fuzz_alg_ses_des_seed_corpus.zip fuzz_alg_hw_rsi_outliers_da_seed_corpus.zip fuzz_alg_bins_seed_corpus.zip fuzz_alg_hll_seed_corpus.zip fuzz_alg_jitter_seed_corpus.zip fuzz_ds_libcache_seed_corpus.zip fuzz_community_id_seed_corpus.zip fuzz_ds_tree_seed_corpus.zip fuzz_serialization_seed_corpus.zip fuzz_ds_ptree_seed_corpus.zip fuzz_alg_crc32_md5_seed_corpus.zip fuzz_alg_bytestream_seed_corpus.zip fuzz_libinjection_seed_corpus.zip fuzz_tls_certificate_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_readerutils_workflow_seed_corpus.zip fuzz_readerutils_parseprotolist_seed_corpus.zip fuzz_ds_bitmap64_seed_corpus.zip fuzz_ds_domain_classify_seed_corpus.zip
+corpus: fuzz_ndpi_reader_seed_corpus.zip fuzz_ndpi_reader_alloc_fail_seed_corpus.zip fuzz_ndpi_reader_payload_analyzer_seed_corpus.zip fuzz_quic_get_crypto_data_seed_corpus.zip fuzz_config_seed_corpus.zip fuzz_ds_patricia_seed_corpus.zip fuzz_ds_ahocorasick_seed_corpus.zip fuzz_alg_ses_des_seed_corpus.zip fuzz_alg_hw_rsi_outliers_da_seed_corpus.zip fuzz_alg_bins_seed_corpus.zip fuzz_alg_hll_seed_corpus.zip fuzz_alg_jitter_seed_corpus.zip fuzz_ds_libcache_seed_corpus.zip fuzz_community_id_seed_corpus.zip fuzz_ds_tree_seed_corpus.zip fuzz_serialization_seed_corpus.zip fuzz_ds_ptree_seed_corpus.zip fuzz_alg_crc32_md5_seed_corpus.zip fuzz_alg_bytestream_seed_corpus.zip fuzz_libinjection_seed_corpus.zip fuzz_tls_certificate_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip fuzz_readerutils_workflow_seed_corpus.zip fuzz_readerutils_parseprotolist_seed_corpus.zip fuzz_ds_bitmap64_seed_corpus.zip fuzz_ds_domain_classify_seed_corpus.zip fuzz_filecfg_protocols_seed_corpus.zip
 	cp corpus/fuzz_*seed_corpus.zip .
 
 #Create dictionaries exactly as expected by oss-fuzz.
@@ -651,6 +668,7 @@ distdir:
 		-o -path './corpus/fuzz_*.zip' \
 		-o -path './corpus/fuzz_quic_get_crypto_data/*' \
 		-o -path './corpus/fuzz_filecfg_protocols/*' \
+		-o -path './corpus/fuzz_filecfg_categories/*' \
 		-o -path './corpus/fuzz_readerutils_workflow/*' \
 		-o -path './corpus/fuzz_readerutils_parseprotolist/*' \
 		-o -path './corpus/fuzz_config/*' \

fuzz/corpus/fuzz_filecfg_categories/domain.txt

@@ -0,0 +1,2 @@
+#  Format: name\tcategory_id
+internetbadguys.com	100

fuzz/corpus/fuzz_filecfg_categories/ipv4.txt

@@ -0,0 +1 @@
+144.139.247.220	100

fuzz/corpus/fuzz_filecfg_categories/ipv4_prefix.txt

@@ -0,0 +1 @@
+144.139.247.220/24	100

fuzz/corpus/fuzz_filecfg_categories/ipv6.txt

@@ -0,0 +1 @@
+[3ffe:507:0:1:200:86ff:fe05:80da]	100

fuzz/corpus/fuzz_filecfg_categories/ipv6_prefix.txt

@@ -0,0 +1 @@
+[3ffe:507:0:1:200:86ff:fe05:80da]/64	100

fuzz/fuzz_config.cpp

@@ -220,6 +220,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
                                         flow.l4_proto,
                                         flow.c_address.v4, flow.c_port,
                                         flow.s_address.v4, flow.s_port);
+    } else {
+      ndpi_find_ipv6_category_userdata(ndpi_info_mod, (struct in6_addr *)flow.c_address.v6);
     }
     /* Another "strange" function: fuzz it here, for lack of a better alternative */
     ndpi_search_tcp_or_udp(ndpi_info_mod, &flow);

fuzz/fuzz_filecfg_categories.c

@@ -0,0 +1,46 @@
+#include "ndpi_api.h"
+#include "fuzz_common_code.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  struct ndpi_detection_module_struct *ndpi_struct;
+  FILE *fd;
+  /* Try to be fast */
+  ndpi_init_prefs prefs = ndpi_dont_load_tor_list |
+			  ndpi_dont_load_azure_list |
+			  ndpi_dont_load_whatsapp_list |
+			  ndpi_dont_load_amazon_aws_list |
+			  ndpi_dont_load_ethereum_list |
+			  ndpi_dont_load_zoom_list |
+			  ndpi_dont_load_cloudflare_list |
+			  ndpi_dont_load_microsoft_list |
+			  ndpi_dont_load_google_list |
+			  ndpi_dont_load_google_cloud_list |
+			  ndpi_dont_load_asn_lists |
+			  ndpi_dont_init_risk_ptree |
+			  ndpi_dont_load_cachefly_list |
+			  ndpi_dont_load_protonvpn_list |
+			  ndpi_dont_load_mullvad_list;
+  NDPI_PROTOCOL_BITMASK all;
+  NDPI_PROTOCOL_BITMASK debug_bitmask;
+
+  /* To allow memory allocation failures */
+  fuzz_set_alloc_callbacks_and_seed(size);
+
+  ndpi_struct = ndpi_init_detection_module(prefs);
+  NDPI_BITMASK_SET_ALL(all);
+  ndpi_set_protocol_detection_bitmask2(ndpi_struct, &all);
+
+  NDPI_BITMASK_SET_ALL(debug_bitmask);
+  ndpi_set_log_level(ndpi_struct, 4);
+  ndpi_set_debug_bitmask(ndpi_struct, debug_bitmask);
+
+  fd = buffer_to_file(data, size);
+  ndpi_load_categories_file2(ndpi_struct, fd, NULL);
+  if(fd)
+    fclose(fd);
+
+  /* We don't really need to call ndpi_finalize_initialization */
+
+  ndpi_exit_detection_module(ndpi_struct);
+  return 0;
+}

src/include/ndpi_api.h

@@ -810,6 +810,7 @@ extern "C" {
    *          -1 else
    */
   int ndpi_load_categories_file(struct ndpi_detection_module_struct *ndpi_str, const char* path, void *user_data);
+  int ndpi_load_categories_file2(struct ndpi_detection_module_struct *ndpi_str, FILE *fd, void *user_data);
 
   /**
    * Loads a file (separated by <cr>) of domain names associated with the specified category
@@ -1028,11 +1029,17 @@ extern "C" {
   int ndpi_enable_loaded_categories(struct ndpi_detection_module_struct *ndpi_struct);
   void* ndpi_find_ipv4_category_userdata(struct ndpi_detection_module_struct *ndpi_str,
 					 u_int32_t saddr);
+  void* ndpi_find_ipv6_category_userdata(struct ndpi_detection_module_struct *ndpi_str,
+					 struct in6_addr *saddr);
   int ndpi_fill_ip_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,
 				     struct ndpi_flow_struct *flow,
 				     u_int32_t saddr,
 				     u_int32_t daddr,
 				     ndpi_protocol *ret);
+  int ndpi_fill_ip6_protocol_category(struct ndpi_detection_module_struct *ndpi_str,
+				      struct ndpi_flow_struct *flow,
+				      struct in6_addr *saddr, struct in6_addr *daddr,
+				      ndpi_protocol *ret);
   int ndpi_match_custom_category(struct ndpi_detection_module_struct *ndpi_struct,
 				 char *name, u_int name_len, ndpi_protocol_category_t *id);
   void ndpi_fill_protocol_category(struct ndpi_detection_module_struct *ndpi_struct,

src/include/ndpi_typedefs.h

@@ -1327,6 +1327,7 @@ struct ndpi_detection_module_struct {
     ndpi_domain_classify *sc_hostnames, *sc_hostnames_shadow;
 #endif
     void *ipAddresses, *ipAddresses_shadow; /* Patricia */
+    void *ipAddresses6, *ipAddresses6_shadow; /* Patricia IPv6*/
     u_int8_t categories_loaded;
   } custom_categories;