diff --git a/.gitignore b/.gitignore index f80e5c682..e0832dded 100644 --- a/.gitignore +++ b/.gitignore @@ -275,6 +275,9 @@ FakesAssemblies/ .ntvs_analysis.dat node_modules/ +# Visual Studio Settings +.vscode + # Visual Studio 6 build log *.plg diff --git a/apps/build_memory_index.cpp b/apps/build_memory_index.cpp index 544e42dee..6fc0da0c9 100644 --- a/apps/build_memory_index.cpp +++ b/apps/build_memory_index.cpp @@ -70,6 +70,9 @@ int main(int argc, char **argv) program_options_utils::FILTERED_LBUILD); optional_configs.add_options()("label_type", po::value(&label_type)->default_value("uint"), program_options_utils::LABEL_TYPE_DESCRIPTION); + //optional_configs.add_options()("reduce_prune", po::bool_switch(&reduce_prune)->default_value(false),"Reduce Prune"); + + //std::cout<<"reduce_prune: "< /home/rakri/avarhade/DiskANN/prec40M/KK4_R90/reduce_prune_r90_l100.txt \ No newline at end of file diff --git a/run-search.sh b/run-search.sh new file mode 100644 index 000000000..167251e34 --- /dev/null +++ b/run-search.sh @@ -0,0 +1,12 @@ +cd build +# ./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_KK_4_reduce_prune_memory_index_r90_l100 --query_file /nvmessd1/fbv4/queries384d.bin --gt_file /nvmessd1/fbv4/gt100_prec40M --result_path /home/rakri/avarhade/Dump/tmp -K 10 -L 10 20 30 40 50 60 70 80 90 100 -T 1 > /home/rakri/avarhade/DiskANN/prec40M/KK4_R90/reduce_prune_r90_k10.txt +# ./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_KK_4_reduce_prune_memory_index_r90_l100 --query_file /nvmessd1/fbv4/queries384d.bin --gt_file /nvmessd1/fbv4/gt100_prec40M --result_path /home/rakri/avarhade/Dump/tmp -K 25 -L 30 40 50 60 70 80 90 100 -T 1 > /home/rakri/avarhade/DiskANN/prec40M/KK4_R90/reduce_prune_r90_k25.txt +# ./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_KK_4_reduce_prune_memory_index_r90_l100 --query_file /nvmessd1/fbv4/queries384d.bin --gt_file /nvmessd1/fbv4/gt100_prec40M --result_path /home/rakri/avarhade/Dump/tmp -K 50 -L 50 60 70 80 90 100 -T 1 > /home/rakri/avarhade/DiskANN/prec40M/KK4_R90/reduce_prune_r90_k50.txt +# ./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_KK_4_reduce_prune_memory_index_r90_l100 --query_file /nvmessd1/fbv4/queries384d.bin --gt_file /nvmessd1/fbv4/gt100_prec40M --result_path /home/rakri/avarhade/Dump/tmp -K 100 -L 100 -T 1 > /home/rakri/avarhade/DiskANN/prec40M/KK4_R90/reduce_prune_r90_k100.txt + +./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_KK_1.5_reduce_prune_memory_index_r64_l100 --query_file /nvmessd1/fbv4/queries384d.bin --gt_file /nvmessd1/fbv4/gt100_prec40M --result_path /home/rakri/avarhade/Dump/tmp -K 50 -L 50 60 70 80 90 100 -T 1 >> /home/rakri/avarhade/DiskANN/prec40M/KK1.5/KK_1.5_reduce_prune_r64_k50.txt +./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_KK_3_reduce_prune_memory_index_r64_l100 --query_file /nvmessd1/fbv4/queries384d.bin --gt_file /nvmessd1/fbv4/gt100_prec40M --result_path /home/rakri/avarhade/Dump/tmp -K 50 -L 50 60 70 80 90 100 -T 1 >> /home/rakri/avarhade/DiskANN/prec40M/KK3/KK_3_reduce_prune_r64_k50.txt +./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_KK_4_reduce_prune_memory_index_r64_l100 --query_file /nvmessd1/fbv4/queries384d.bin --gt_file /nvmessd1/fbv4/gt100_prec40M --result_path /home/rakri/avarhade/Dump/tmp -K 50 -L 50 60 70 80 90 100 -T 1 >> /home/rakri/avarhade/DiskANN/prec40M/KK4/KK_4_reduce_prune_r64_k50.txt + + +./apps/search_memory_index --data_type int8 --dist_fn l2 --index_path_prefix /nvmessd1/fbv4/avarhade/prec40M_reduce_prune_memory_index_r64_l100 --query_file /nvmessd1/fbv4/queries384d.bin --gt_file /nvmessd1/fbv4/gt100_prec40M --result_path /home/rakri/avarhade/Dump/tmp -K 50 -L 50 60 70 80 90 100 -T 1 >> /home/rakri/avarhade/DiskANN/prec40M/reduce_prune_r64_k50.txt \ No newline at end of file diff --git a/src/index.cpp b/src/index.cpp index bf93344fa..6fd8c851b 100644 --- a/src/index.cpp +++ b/src/index.cpp @@ -4,7 +4,7 @@ #include #include - +#include #include "boost/dynamic_bitset.hpp" #include "index_factory.h" #include "memory_mapper.h" @@ -25,6 +25,9 @@ #define MAX_POINTS_FOR_USING_BITSET 10000000 +//bool reduce_prune = false; +std::atomic count_prune = 0; + namespace diskann { // Initialize an index with metric m, load the data of type T with filename @@ -1032,7 +1035,7 @@ void Index::search_for_point_and_prune(int location, uint32_t L } auto &pool = scratch->pool(); - + //diskann::cout<<"Pool Size (search_point_and_prune) "<::occlude_list(const uint32_t location, std::vector &result, InMemQueryScratch *scratch, const tsl::robin_set *const delete_set_ptr) -{ +{ + count_prune++; if (pool.size() == 0) return; @@ -1067,6 +1071,7 @@ void Index::occlude_list(const uint32_t location, std::vector maxc) pool.resize(maxc); + std::vector &occlude_factor = scratch->occlude_factor(); // occlude_list can be called with the same scratch more than once by // search_for_point_and_add_link through inter_insert. @@ -1074,7 +1079,8 @@ void Index::occlude_list(const uint32_t location, std::vector::prune_neighbors(const uint32_t location, std::vecto pruned_list.clear(); pruned_list.reserve(range); + //diskann::cout<<"Pool Size (prune_neighbors before occlude) "< 1) { for (const auto &node : pool) @@ -1319,8 +1329,18 @@ template void Index reduced_pruned_list = pruned_list; + bool reduce_prune = true; + if (reduce_prune){ + //diskann::cout << "Reducing pruned list for node " << std::endl; + float KK = 4; + uint32_t reduced_pruned_list_size = (uint32_t)(_indexingRange/KK*(1+KK*(node_ctr/(visit_order.size())))); + reduced_pruned_list_size = std::min(reduced_pruned_list_size, (uint32_t)reduced_pruned_list.size()); + reduced_pruned_list.resize(reduced_pruned_list_size); + } - _graph_store->set_neighbours(node, pruned_list); + _graph_store->set_neighbours(node, reduced_pruned_list); assert(_graph_store->get_neighbours((location_t)node).size() <= _indexingRange); } @@ -1556,7 +1576,7 @@ void Index::build_with_data_populated(const std::vector & } diskann::cout << "Index built with degree: max:" << max << " avg:" << (float)total / (float)(_nd + _num_frozen_pts) << " min:" << min << " count(deg<2):" << cnt << std::endl; - + diskann::cout << "Robust Prune Calls: " << count_prune << std::endl; _has_built = true; } template