Add patch for wtoring data in ordered vector instead of vector
12 subnets, total size: 18176 IP std::map lookup: Incoming traffic 6740435 pps 3085 mbps 0 flows sorted vector lookup: new: Incoming traffic 7129775 pps 3263 mbps 0 flows Speedup: 400 kpps. CPU: E5-2407 2.20GHz 4 cores
This commit is contained in:
parent
a966603c2b
commit
1211a513cd
|
@ -0,0 +1,135 @@
|
|||
diff --git a/src/fastnetmon.cpp b/src/fastnetmon.cpp
|
||||
index 5308244..2a67506 100644
|
||||
--- a/src/fastnetmon.cpp
|
||||
+++ b/src/fastnetmon.cpp
|
||||
@@ -191,6 +191,12 @@ map_of_vector_counters SubnetVectorMap;
|
||||
// Flow tracking structures
|
||||
map_of_vector_counters_for_flow SubnetVectorMapFlow;
|
||||
|
||||
+typedef std::vector<unsigned long int> lookup_vector_indexes_t;
|
||||
+typedef std::vector<vector_of_counters> lookup_vector_data_t;
|
||||
+
|
||||
+lookup_vector_indexes_t lookup_vector_indexes;
|
||||
+lookup_vector_data_t lookup_vector_data;
|
||||
+
|
||||
/* End of our data structs */
|
||||
|
||||
boost::mutex data_counters_mutex;
|
||||
@@ -969,12 +975,22 @@ bool load_our_networks_list() {
|
||||
|
||||
/* Preallocate data structures */
|
||||
|
||||
- patricia_process (lookup_tree, (void_fn_t)subnet_vectors_allocator);
|
||||
-
|
||||
logger<<log4cpp::Priority::INFO<<"We start total zerofication of counters";
|
||||
zeroify_all_counters();
|
||||
logger<<log4cpp::Priority::INFO<<"We finished zerofication";
|
||||
|
||||
+ // We could try allocate vector from std::map
|
||||
+ patricia_process (lookup_tree, (void_fn_t)subnet_vectors_allocator);
|
||||
+ lookup_vector_indexes.reserve( SubnetVectorMap.size() );
|
||||
+ lookup_vector_data.reserve( SubnetVectorMap.size() );
|
||||
+
|
||||
+ // Copy pointers to the vector
|
||||
+ // We do not need any sort here because vector is already sorted because std::map nature
|
||||
+ for( map_of_vector_counters::iterator ii = SubnetVectorMap.begin(); ii != SubnetVectorMap.end(); ++ii) {
|
||||
+ lookup_vector_indexes.push_back(ii->first);
|
||||
+ lookup_vector_data.push_back(ii->second);
|
||||
+ }
|
||||
+
|
||||
logger<<log4cpp::Priority::INFO<<"We loaded "<<networks_list_as_string.size()<<" subnets to our in-memory list of networks";
|
||||
logger<<log4cpp::Priority::INFO<<"Total number of monitored hosts (total size of all networks): "
|
||||
<<total_number_of_hosts_in_our_networks;
|
||||
@@ -1005,14 +1021,17 @@ void process_packet(simple_packet& current_packet) {
|
||||
}
|
||||
|
||||
// Try to find map key for this subnet
|
||||
- map_of_vector_counters::iterator itr;
|
||||
+ vector_of_counters* itr = NULL;
|
||||
|
||||
if (packet_direction == OUTGOING or packet_direction == INCOMING) {
|
||||
- itr = SubnetVectorMap.find(subnet);
|
||||
+ lookup_vector_indexes_t::iterator fast_iter = std::lower_bound(lookup_vector_indexes.begin(), lookup_vector_indexes.end(), subnet);
|
||||
|
||||
- if (itr == SubnetVectorMap.end()) {
|
||||
+ if (fast_iter != lookup_vector_indexes.end() && *fast_iter == subnet) {
|
||||
+ // Load data from data vector from same index
|
||||
+ itr = &lookup_vector_data[ fast_iter - lookup_vector_indexes.begin() ];
|
||||
+ } else {
|
||||
logger<< log4cpp::Priority::ERROR<<"Can't find vector address in subnet map";
|
||||
- return;
|
||||
+ return;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1046,16 +1065,16 @@ void process_packet(simple_packet& current_packet) {
|
||||
if (packet_direction == OUTGOING) {
|
||||
int64_t shift_in_vector = (int64_t)ntohl(current_packet.src_ip) - (int64_t)subnet_in_host_byte_order;
|
||||
|
||||
- if (shift_in_vector < 0 or shift_in_vector >= itr->second.size()) {
|
||||
+ if (shift_in_vector < 0 or shift_in_vector >= itr->size()) {
|
||||
logger<< log4cpp::Priority::ERROR<<"We tried to access to element with index "<<shift_in_vector
|
||||
- <<" which located outside allocated vector with size "<<itr->second.size();
|
||||
+ <<" which located outside allocated vector with size "<<itr->size();
|
||||
|
||||
logger<< log4cpp::Priority::ERROR<<"We expect issues with this packet in OUTGOING direction: "<<print_simple_packet(current_packet);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
- map_element* current_element = &itr->second[shift_in_vector];
|
||||
+ map_element* current_element = &(*itr)[shift_in_vector];
|
||||
|
||||
// Main packet/bytes counter
|
||||
__sync_fetch_and_add(¤t_element->out_packets, sampled_number_of_packets);
|
||||
@@ -1135,16 +1154,16 @@ void process_packet(simple_packet& current_packet) {
|
||||
} else if (packet_direction == INCOMING) {
|
||||
int64_t shift_in_vector = (int64_t)ntohl(current_packet.dst_ip) - (int64_t)subnet_in_host_byte_order;
|
||||
|
||||
- if (shift_in_vector < 0 or shift_in_vector >= itr->second.size()) {
|
||||
+ if (shift_in_vector < 0 or shift_in_vector >= itr->size()) {
|
||||
logger<< log4cpp::Priority::ERROR<<"We tried to access to element with index "<<shift_in_vector
|
||||
- <<" which located outside allocated vector with size "<<itr->second.size();
|
||||
+ <<" which located outside allocated vector with size "<<itr->size();
|
||||
|
||||
logger<< log4cpp::Priority::INFO<<"We expect issues with this packet in INCOMING direction: "<<print_simple_packet(current_packet);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
- map_element* current_element = &itr->second[shift_in_vector];
|
||||
+ map_element* current_element = &(*itr)[shift_in_vector];
|
||||
|
||||
// Main packet/bytes counter
|
||||
__sync_fetch_and_add(¤t_element->in_packets, sampled_number_of_packets);
|
||||
@@ -1307,15 +1326,17 @@ void recalculate_speed() {
|
||||
uint64_t incoming_total_flows = 0;
|
||||
uint64_t outgoing_total_flows = 0;
|
||||
|
||||
- for (map_of_vector_counters::iterator itr = SubnetVectorMap.begin(); itr != SubnetVectorMap.end(); ++itr) {
|
||||
- for (vector_of_counters::iterator vector_itr = itr->second.begin(); vector_itr != itr->second.end(); ++vector_itr) {
|
||||
- int current_index = vector_itr - itr->second.begin();
|
||||
+ for (lookup_vector_indexes_t::iterator itr = lookup_vector_indexes.begin(); itr != lookup_vector_indexes.end(); ++itr) {
|
||||
+ vector_of_counters* vector_pointer = &lookup_vector_data[ itr - lookup_vector_indexes.begin() ];
|
||||
+
|
||||
+ for (vector_of_counters::iterator vector_itr = vector_pointer->begin(); vector_itr != vector_pointer->end(); ++vector_itr) {
|
||||
+ int current_index = vector_itr - vector_pointer->begin();
|
||||
|
||||
// New element
|
||||
map_element new_speed_element;
|
||||
|
||||
// convert to host order for math operations
|
||||
- uint32_t subnet_ip = ntohl(itr->first);
|
||||
+ uint32_t subnet_ip = ntohl(*itr);
|
||||
uint32_t client_ip_in_host_bytes_order = subnet_ip + current_index;
|
||||
|
||||
// covnert to our standard network byte order
|
||||
@@ -1364,7 +1385,7 @@ void recalculate_speed() {
|
||||
new_speed_element.icmp_in_bytes = uint64_t((double)vector_itr->icmp_in_bytes / speed_calc_period);
|
||||
new_speed_element.icmp_out_bytes = uint64_t((double)vector_itr->icmp_out_bytes / speed_calc_period);
|
||||
|
||||
- conntrack_main_struct* flow_counter_ptr = &SubnetVectorMapFlow[itr->first][current_index];
|
||||
+ conntrack_main_struct* flow_counter_ptr = &SubnetVectorMapFlow[*itr][current_index];
|
||||
|
||||
// todo: optimize this operations!
|
||||
uint64_t total_out_flows =
|
Loading…
Reference in New Issue