2013-10-21 11:43:54 +02:00
/* Author: pavel.odintsov@gmail.com */
/* License: GPLv2 */
2013-10-18 12:16:55 +02:00
# include <stdio.h>
# include <stdlib.h>
# include <errno.h>
# include <string.h>
# include <unistd.h>
# include <signal.h>
2013-10-21 11:43:54 +02:00
# include <time.h>
2014-06-30 10:49:55 +02:00
# include <math.h>
2013-10-21 11:43:54 +02:00
2013-10-18 12:16:55 +02:00
# include <sys/socket.h>
2014-06-22 14:25:12 +02:00
# include <sys/resource.h>
2014-12-05 12:02:46 +01:00
# include <sys/stat.h>
2013-10-18 12:16:55 +02:00
# include <arpa/inet.h>
# include <netinet/ip.h>
# include <netinet/tcp.h>
# include <netinet/udp.h>
# include <netinet/ip_icmp.h>
2013-10-21 11:43:54 +02:00
# include <netinet/if_ether.h>
# include <netinet/in.h>
2013-10-18 12:16:55 +02:00
2014-06-25 21:59:43 +02:00
# include "libpatricia/patricia.h"
2014-12-01 22:08:38 +01:00
# include "fastnetmon_types.h"
2015-03-21 12:24:31 +01:00
# include "fast_library.h"
2015-01-23 17:49:37 +01:00
// Plugins
2014-12-01 22:08:38 +01:00
# include "sflow_plugin/sflow_collector.h"
2015-01-23 17:49:37 +01:00
# include "netflow_plugin/netflow_collector.h"
2015-01-27 10:56:16 +01:00
# include "pcap_plugin/pcap_collector.h"
2015-03-10 15:17:52 +01:00
# include "netmap_plugin/netmap_collector.h"
2015-01-27 10:56:16 +01:00
2015-02-03 20:55:44 +01:00
# ifdef PF_RING
# include "pfring_plugin/pfring_collector.h"
# endif
2015-01-27 10:56:16 +01:00
// Yes, maybe it's not an good idea but with this we can guarantee working code in example plugin
2015-01-26 10:10:35 +01:00
# include "example_plugin/example_collector.h"
2014-06-26 17:08:19 +02:00
2015-01-27 10:56:16 +01:00
2013-10-18 12:16:55 +02:00
# include <algorithm>
# include <iostream>
# include <map>
2014-06-09 11:08:19 +02:00
# include <fstream>
2013-11-15 15:35:44 +01:00
2013-10-18 12:16:55 +02:00
# include <vector>
# include <utility>
# include <sstream>
2014-06-29 13:28:56 +02:00
# include <boost/thread.hpp>
# include <boost/thread/mutex.hpp>
2014-10-22 15:27:15 +02:00
# include <boost/regex.hpp>
2013-10-21 16:47:31 +02:00
2014-06-25 21:59:43 +02:00
// log4cpp logging facility
2014-06-24 12:16:22 +02:00
# include "log4cpp/Category.hh"
# include "log4cpp/Appender.hh"
# include "log4cpp/FileAppender.hh"
# include "log4cpp/OstreamAppender.hh"
# include "log4cpp/Layout.hh"
# include "log4cpp/BasicLayout.hh"
# include "log4cpp/PatternLayout.hh"
# include "log4cpp/Priority.hh"
2014-06-25 21:59:43 +02:00
// Boost libs
2013-10-18 12:16:55 +02:00
# include <boost/algorithm/string.hpp>
2014-06-23 14:48:25 +02:00
2013-12-28 20:11:20 +01:00
# ifdef GEOIP
# include "GeoIP.h"
# endif
2013-10-20 00:10:19 +02:00
# ifdef REDIS
# include <hiredis/hiredis.h>
# endif
2013-10-18 12:21:41 +02:00
2015-05-02 22:53:45 +02:00
std : : string pid_path = " /var/run/fastnetmon.pid " ;
2015-01-27 20:29:51 +01:00
std : : string global_config_path = " /etc/fastnetmon.conf " ;
2014-11-23 20:17:56 +01:00
2014-06-28 23:04:31 +02:00
time_t last_call_of_traffic_recalculation ;
2014-12-21 12:52:22 +01:00
// Variable with all data from main screen
2015-01-27 20:29:51 +01:00
std : : string screen_data_stats = " " ;
2014-12-21 12:52:22 +01:00
2015-01-27 10:56:16 +01:00
// Global map with parsed config file
2015-01-27 20:29:51 +01:00
std : : map < std : : string , std : : string > configuration_map ;
2015-01-27 10:56:16 +01:00
2013-11-15 15:35:44 +01:00
/* Configuration block, we must move it to configuration file */
2013-10-21 12:27:43 +02:00
# ifdef REDIS
2014-07-04 10:02:44 +02:00
unsigned int redis_port = 6379 ;
2015-01-27 20:29:51 +01:00
std : : string redis_host = " 127.0.0.1 " ;
2014-06-09 12:33:07 +02:00
// because it's additional and very specific feature we should disable it by default
bool redis_enabled = false ;
2013-10-21 12:27:43 +02:00
# endif
2013-10-18 12:16:55 +02:00
2014-12-02 16:07:33 +01:00
bool enable_ban_for_pps = false ;
bool enable_ban_for_bandwidth = false ;
2014-12-02 15:46:46 +01:00
bool enable_ban_for_flows_per_second = false ;
2014-12-02 11:50:05 +01:00
bool enable_conection_tracking = true ;
2014-12-02 13:43:34 +01:00
bool enable_data_collection_from_mirror = true ;
2015-03-10 15:17:52 +01:00
bool enable_netmap_collection = false ;
2014-12-02 13:43:34 +01:00
bool enable_sflow_collection = false ;
2015-01-23 17:37:23 +01:00
bool enable_netflow_collection = false ;
2015-01-27 10:56:16 +01:00
bool enable_pcap_collection = false ;
2015-01-23 17:37:23 +01:00
2014-10-23 10:51:01 +02:00
// Time consumed by reaclculation for all IPs
2014-12-16 12:59:24 +01:00
struct timeval speed_calculation_time ;
// Time consumed by drawing stats for all IPs
struct timeval drawing_thread_execution_time ;
2014-10-23 10:51:01 +02:00
2015-05-12 11:07:45 +02:00
// Global thread group for packet capture threads
boost : : thread_group packet_capture_plugin_thread_group ;
2014-06-30 12:25:41 +02:00
// Total number of hosts in our networks
// We need this as global variable because it's very important value for configuring data structures
2014-07-04 10:02:44 +02:00
unsigned int total_number_of_hosts_in_our_networks = 0 ;
2014-06-30 12:25:41 +02:00
2013-12-28 20:11:20 +01:00
# ifdef GEOIP
GeoIP * geo_ip = NULL ;
# endif
2014-06-25 21:59:43 +02:00
patricia_tree_t * lookup_tree , * whitelist_tree ;
2014-07-04 10:02:44 +02:00
bool DEBUG = 0 ;
2013-10-21 12:27:43 +02:00
2014-12-15 20:08:39 +01:00
// flag about dumping all packets to log
2014-06-20 17:18:27 +02:00
bool DEBUG_DUMP_ALL_PACKETS = false ;
2014-11-27 22:16:27 +01:00
// Period for update screen for console version of tool
2014-07-04 10:02:44 +02:00
unsigned int check_period = 3 ;
2013-10-21 12:27:43 +02:00
2014-10-21 09:20:18 +02:00
// Standard ban time in seconds for all attacks but you can tune this value
int standard_ban_time = 1800 ;
2014-11-27 22:16:27 +01:00
// We calc average pps/bps for this time
double average_calculation_amount = 15 ;
2015-01-26 14:10:25 +01:00
// Show average or absolute value of speed
bool print_average_traffic_counts = true ;
2014-11-27 22:16:27 +01:00
2015-04-27 14:49:03 +02:00
// Key used for sorting clients in output. Allowed sort params: packets/bytes/flows
2015-01-27 20:29:51 +01:00
std : : string sort_parameter = " packets " ;
2013-10-21 15:43:00 +02:00
2014-06-09 11:08:19 +02:00
// Path to notify script
2015-01-27 20:29:51 +01:00
std : : string notify_script_path = " /usr/local/bin/notify_about_attack.sh " ;
2014-06-09 11:08:19 +02:00
2015-04-27 18:11:08 +02:00
// Path to file with networks for whitelising
std : : string white_list_path = " /etc/networks_whitelist " ;
// Path to file with all networks listing
std : : string networks_list_path = " /etc/networks_list " ;
2013-11-15 15:35:44 +01:00
// Number of lines in programm output
2014-07-04 10:02:44 +02:00
unsigned int max_ips_in_list = 7 ;
2013-10-21 12:27:43 +02:00
2013-11-15 15:35:44 +01:00
// We must ban IP if it exceeed this limit in PPS
2014-10-29 10:21:53 +01:00
unsigned int ban_threshold_pps = 20000 ;
2014-11-14 22:34:29 +01:00
// We must ban IP of it exceed this limit for number of flows in any direction
unsigned int ban_threshold_flows = 3500 ;
2014-10-29 10:21:53 +01:00
// We must ban client if it exceed 1GBps
unsigned int ban_threshold_mbps = 1000 ;
2013-10-21 12:27:43 +02:00
2013-11-15 15:35:44 +01:00
// Number of lines for sending ben attack details to email
2014-07-04 10:02:44 +02:00
unsigned int ban_details_records_count = 500 ;
2013-10-21 12:27:43 +02:00
2014-06-09 14:47:11 +02:00
// log file
2014-06-24 12:16:22 +02:00
log4cpp : : Category & logger = log4cpp : : Category : : getRoot ( ) ;
2015-01-27 20:29:51 +01:00
std : : string log_file_path = " /var/log/fastnetmon.log " ;
std : : string attack_details_folder = " /var/log/fastnetmon_attacks " ;
2014-06-09 14:47:11 +02:00
2013-11-15 15:35:44 +01:00
/* Configuration block ends */
2013-10-21 12:27:43 +02:00
2014-06-26 20:27:55 +02:00
// We count total number of incoming/outgoing/internal and other traffic type packets/bytes
// And initilize by 0 all fields
2014-06-30 09:04:06 +02:00
total_counter_element total_counters [ 4 ] ;
2014-06-30 13:10:07 +02:00
total_counter_element total_speed_counters [ 4 ] ;
2013-11-15 15:35:44 +01:00
2014-12-10 16:08:12 +01:00
// Total amount of non parsed packets
2014-12-16 15:20:04 +01:00
uint64_t total_unparsed_packets = 0 ;
2014-12-01 22:08:38 +01:00
2014-12-16 15:20:04 +01:00
uint64_t incoming_total_flows_speed = 0 ;
uint64_t outgoing_total_flows_speed = 0 ;
2014-11-14 11:45:14 +01:00
2014-10-23 16:08:58 +02:00
map_of_vector_counters SubnetVectorMap ;
2014-10-23 10:51:01 +02:00
2014-11-13 14:13:28 +01:00
// Flow tracking structures
map_of_vector_counters_for_flow SubnetVectorMapFlow ;
2013-11-15 15:35:44 +01:00
/* End of our data structs */
2013-10-18 12:16:55 +02:00
2014-06-29 13:28:56 +02:00
boost : : mutex data_counters_mutex ;
boost : : mutex speed_counters_mutex ;
boost : : mutex total_counters_mutex ;
2014-12-19 16:50:00 +01:00
boost : : mutex ban_list_details_mutex ;
2014-10-21 09:20:18 +02:00
boost : : mutex ban_list_mutex ;
2014-10-22 04:29:55 +02:00
boost : : mutex flow_counter ;
2013-10-21 21:45:33 +02:00
2014-10-22 04:29:55 +02:00
// map for flows
2015-01-27 20:29:51 +01:00
std : : map < uint64_t , int > FlowCounter ;
2014-10-22 04:29:55 +02:00
2014-11-15 01:10:04 +01:00
// Struct for string speed per IP
2014-06-25 17:32:32 +02:00
map_for_counters SpeedCounter ;
2014-11-27 22:16:27 +01:00
// Struct for storing average speed per IP for specified interval
map_for_counters SpeedCounterAverage ;
2013-12-28 20:11:20 +01:00
# ifdef GEOIP
map_for_counters GeoIpCounter ;
# endif
2014-11-15 01:10:04 +01:00
// In ddos info we store attack power and direction
2015-01-27 20:29:51 +01:00
std : : map < uint32_t , banlist_item > ban_list ;
std : : map < uint32_t , std : : vector < simple_packet > > ban_list_details ;
2013-10-18 12:16:55 +02:00
2015-01-27 20:29:51 +01:00
std : : vector < subnet > our_networks ;
std : : vector < subnet > whitelist_networks ;
2013-10-18 12:16:55 +02:00
2014-11-15 01:10:04 +01:00
// Ban enable/disable flag
2014-11-13 09:50:17 +01:00
bool we_do_real_ban = true ;
2015-04-26 11:47:37 +02:00
// ExaBGP support flag
bool exabgp_enabled = false ;
2015-04-26 14:35:01 +02:00
std : : string exabgp_community = " " ;
2015-05-11 21:56:18 +02:00
std : : string exabgp_command_pipe = " /var/run/exabgp.cmd " ;
2015-04-26 14:35:01 +02:00
std : : string exabgp_next_hop = " " ;
2015-04-26 11:47:37 +02:00
2015-05-10 20:42:49 +02:00
// Graphite monitoring
bool graphite_enabled = false ;
std : : string graphite_host = " 127.0.0.1 " ;
unsigned short int graphite_port = 2003 ;
std : : string graphite_prefix = " fastnetmon. " ;
2015-01-28 08:49:12 +01:00
bool process_incoming_traffic = true ;
bool process_outgoing_traffic = true ;
2014-11-15 01:10:04 +01:00
// Prototypes
2014-12-03 15:56:08 +01:00
# ifdef HWFILTER_LOCKING
2015-01-27 20:29:51 +01:00
void block_all_traffic_with_82599_hardware_filtering ( std : : string client_ip_as_string ) ;
2014-12-03 15:56:08 +01:00
# endif
2014-12-05 12:02:46 +01:00
2015-05-07 15:51:10 +02:00
std : : string get_printable_attack_name ( attack_type_t attack ) ;
attack_type_t detect_attack_type ( attack_details & current_attack ) ;
2015-03-13 16:25:13 +01:00
bool we_should_ban_this_ip ( map_element * current_average_speed_element ) ;
2014-12-16 15:20:04 +01:00
unsigned int get_max_used_protocol ( uint64_t tcp , uint64_t udp , uint64_t icmp ) ;
2015-01-27 20:29:51 +01:00
void print_attack_details_to_file ( std : : string details , std : : string client_ip_as_string , attack_details current_attack ) ;
std : : string print_ban_thresholds ( ) ;
2014-11-23 20:17:56 +01:00
bool load_configuration_file ( ) ;
2015-01-27 20:29:51 +01:00
std : : string print_flow_tracking_for_ip ( conntrack_main_struct & conntrack_element , std : : string client_ip ) ;
2014-11-15 00:41:54 +01:00
void convert_integer_to_conntrack_hash_struct ( packed_session * packed_connection_data , packed_conntrack_hash * unpacked_data ) ;
2014-11-13 16:01:15 +01:00
uint64_t convert_conntrack_hash_struct_to_integer ( packed_conntrack_hash * struct_value ) ;
2014-10-21 09:20:18 +02:00
void cleanup_ban_list ( ) ;
2015-01-27 20:29:51 +01:00
std : : string get_attack_description ( uint32_t client_ip , attack_details & current_attack ) ;
2014-07-04 12:04:19 +02:00
void send_attack_details ( uint32_t client_ip , attack_details current_attack_details ) ;
2014-06-30 15:33:14 +02:00
void free_up_all_resources ( ) ;
2015-01-27 20:29:51 +01:00
std : : string print_ddos_attack_details ( ) ;
2015-03-13 17:24:57 +01:00
void execute_ip_ban ( uint32_t client_ip , map_element new_speed_element , map_element current_speed_element , std : : string flow_attack_details ) ;
2014-10-23 16:08:58 +02:00
direction get_packet_direction ( uint32_t src_ip , uint32_t dst_ip , unsigned long & subnet ) ;
2014-06-25 17:32:32 +02:00
void recalculate_speed ( ) ;
2015-01-27 20:29:51 +01:00
std : : string print_channel_speed ( std : : string traffic_type , direction packet_direction ) ;
2014-06-21 16:11:53 +02:00
void process_packet ( simple_packet & current_packet ) ;
2014-12-16 12:59:24 +01:00
void traffic_draw_programm ( ) ;
2015-04-27 16:52:16 +02:00
void interruption_signal_handler ( int signal_number ) ;
2013-10-18 12:16:55 +02:00
2014-12-04 15:45:42 +01:00
/* Class for custom comparison fields by different fields */
class TrafficComparatorClass {
private :
sort_type sort_field ;
direction sort_direction ;
public :
TrafficComparatorClass ( direction sort_direction , sort_type sort_field ) {
this - > sort_field = sort_field ;
this - > sort_direction = sort_direction ;
}
2014-11-14 10:55:23 +01:00
2014-12-04 15:45:42 +01:00
bool operator ( ) ( pair_of_map_elements a , pair_of_map_elements b ) {
if ( sort_field = = FLOWS ) {
if ( sort_direction = = INCOMING ) {
return a . second . in_flows > b . second . in_flows ;
} else if ( sort_direction = = OUTGOING ) {
return a . second . out_flows > b . second . out_flows ;
} else {
return false ;
}
} else if ( sort_field = = PACKETS ) {
if ( sort_direction = = INCOMING ) {
return a . second . in_packets > b . second . in_packets ;
} else if ( sort_direction = = OUTGOING ) {
return a . second . out_packets > b . second . out_packets ;
} else {
return false ;
}
} else if ( sort_field = = BYTES ) {
if ( sort_direction = = INCOMING ) {
return a . second . in_bytes > b . second . in_bytes ;
} else if ( sort_direction = = OUTGOING ) {
return a . second . out_bytes > b . second . out_bytes ;
} else {
return false ;
}
} else {
return false ;
}
}
} ;
2014-11-14 10:55:23 +01:00
2015-01-27 20:29:51 +01:00
std : : string get_direction_name ( direction direction_value ) {
std : : string direction_name ;
2013-10-21 11:43:54 +02:00
switch ( direction_value ) {
case INCOMING : direction_name = " incoming " ; break ;
case OUTGOING : direction_name = " outgoing " ; break ;
case INTERNAL : direction_name = " internal " ; break ;
case OTHER : direction_name = " other " ; break ;
default : direction_name = " unknown " ; break ;
}
return direction_name ;
}
2015-04-27 16:52:16 +02:00
void sigpipe_handler_for_popen ( int signo ) {
logger < < log4cpp : : Priority : : ERROR < < " Sorry but we experienced error with popen. "
< < " Please check your scripts. It should receive data on stdin! " ;
exit ( 1 ) ;
}
2014-06-09 13:53:31 +02:00
// exec command and pass data to it stdin
2015-01-27 20:29:51 +01:00
bool exec_with_stdin_params ( std : : string cmd , std : : string params ) {
2013-10-19 16:40:54 +02:00
FILE * pipe = popen ( cmd . c_str ( ) , " w " ) ;
2014-12-26 09:57:30 +01:00
if ( ! pipe ) {
logger < < log4cpp : : Priority : : ERROR < < " Can't execute programm " < < cmd < < " error code: " < < errno < < " error text: " < < strerror ( errno ) ;
return false ;
}
2013-10-19 16:40:54 +02:00
2015-04-27 16:52:16 +02:00
int fputs_ret = fputs ( params . c_str ( ) , pipe ) ;
if ( fputs_ret ) {
pclose ( pipe ) ;
2013-10-19 16:40:54 +02:00
return true ;
} else {
2014-12-26 09:57:30 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Can't pass data to stdin of programm " < < cmd ;
2015-04-27 16:52:16 +02:00
pclose ( pipe ) ;
2013-10-19 16:40:54 +02:00
return false ;
}
}
2013-12-28 20:11:20 +01:00
# ifdef GEOIP
bool geoip_init ( ) {
// load GeoIP ASN database to memory
geo_ip = GeoIP_open ( " /root/fastnetmon/GeoIPASNum.dat " , GEOIP_MEMORY_CACHE ) ;
if ( geo_ip = = NULL ) {
return false ;
} else {
return true ;
}
}
# endif
2013-10-20 00:10:19 +02:00
# ifdef REDIS
2015-04-13 11:30:20 +02:00
redisContext * redis_init_connection ( ) {
2013-10-21 11:43:54 +02:00
struct timeval timeout = { 1 , 500000 } ; // 1.5 seconds
2015-04-13 11:30:20 +02:00
redisContext * redis_context = redisConnectWithTimeout ( redis_host . c_str ( ) , redis_port , timeout ) ;
2013-10-21 11:43:54 +02:00
if ( redis_context - > err ) {
2014-06-24 12:16:22 +02:00
logger < < log4cpp : : Priority : : INFO < < " Connection error: " < < redis_context - > errstr ;
2015-04-13 11:30:20 +02:00
return NULL ;
2013-10-21 11:43:54 +02:00
}
2014-11-15 01:10:04 +01:00
// We should check connection with ping because redis do not check connection
2013-10-21 11:43:54 +02:00
redisReply * reply = ( redisReply * ) redisCommand ( redis_context , " PING " ) ;
if ( reply ) {
freeReplyObject ( reply ) ;
} else {
2015-04-13 11:30:20 +02:00
return NULL ;
2013-10-21 11:43:54 +02:00
}
2015-04-13 11:30:20 +02:00
return redis_context ;
2013-10-21 11:43:54 +02:00
}
2015-04-13 11:30:20 +02:00
# endif
2013-10-21 11:43:54 +02:00
2015-04-13 11:30:20 +02:00
# ifdef REDIS
void store_data_in_redis ( std : : string key_name , std : : string attack_details ) {
redisReply * reply = NULL ;
redisContext * redis_context = redis_init_connection ( ) ;
2013-10-20 00:10:19 +02:00
2013-10-21 11:43:54 +02:00
if ( ! redis_context ) {
2015-04-13 11:30:20 +02:00
logger < < log4cpp : : Priority : : INFO < < " Could not initiate connection to Redis " ;
2013-10-21 11:43:54 +02:00
return ;
2013-10-20 00:10:19 +02:00
}
2013-10-21 11:43:54 +02:00
2015-04-13 11:30:20 +02:00
reply = ( redisReply * ) redisCommand ( redis_context , " SET %s %s " , key_name . c_str ( ) , attack_details . c_str ( ) ) ;
2013-10-21 11:43:54 +02:00
2014-11-15 01:10:04 +01:00
// If we store data correctly ...
2013-10-21 11:43:54 +02:00
if ( ! reply ) {
2014-12-19 09:43:22 +01:00
logger . error ( " Can't increment traffic in redis error_code: %d error_string: %s " , redis_context - > err , redis_context - > errstr ) ;
2013-10-21 11:43:54 +02:00
2014-11-15 01:10:04 +01:00
// Handle redis server restart corectly
2013-10-21 11:43:54 +02:00
if ( redis_context - > err = = 1 or redis_context - > err = = 3 ) {
2015-04-13 11:30:20 +02:00
// Connection refused
logger . error ( " Unfortunately we can't store data in Redis because server reject connection " ) ;
2013-10-21 11:43:54 +02:00
}
} else {
freeReplyObject ( reply ) ;
}
2015-04-13 11:30:20 +02:00
redisFree ( redis_context ) ;
2013-10-20 00:10:19 +02:00
}
# endif
2015-01-27 20:29:51 +01:00
std : : string draw_table ( map_for_counters & my_map_packets , direction data_direction , bool do_redis_update , sort_type sort_item ) {
2014-11-14 11:45:14 +01:00
std : : vector < pair_of_map_elements > vector_for_sort ;
2013-10-18 12:16:55 +02:00
2015-01-27 20:29:51 +01:00
std : : stringstream output_buffer ;
2014-06-24 12:42:43 +02:00
2014-11-14 11:45:14 +01:00
// Preallocate memory for sort vector
vector_for_sort . reserve ( my_map_packets . size ( ) ) ;
2014-06-23 14:48:25 +02:00
2014-11-14 11:45:14 +01:00
for ( map_for_counters : : iterator ii = my_map_packets . begin ( ) ; ii ! = my_map_packets . end ( ) ; + + ii ) {
2014-11-15 01:10:04 +01:00
// store all elements into vector for sorting
2015-01-27 20:29:51 +01:00
vector_for_sort . push_back ( std : : make_pair ( ( * ii ) . first , ( * ii ) . second ) ) ;
2014-11-14 11:45:14 +01:00
}
2014-12-04 15:45:42 +01:00
if ( data_direction = = INCOMING or data_direction = = OUTGOING ) {
std : : sort ( vector_for_sort . begin ( ) , vector_for_sort . end ( ) , TrafficComparatorClass ( data_direction , sort_item ) ) ;
2014-11-14 11:45:14 +01:00
} else {
2015-02-21 15:23:22 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Unexpected bahaviour on sort function " ;
return " Internal error " ;
2014-11-14 11:45:14 +01:00
}
2015-05-10 20:42:49 +02:00
graphite_data_t graphite_data ;
2014-11-14 11:45:14 +01:00
unsigned int element_number = 0 ;
2015-02-21 15:23:22 +01:00
// TODO: fix this code because iteraton over over millions of IPs is very CPU intensive
2015-01-27 20:29:51 +01:00
for ( std : : vector < pair_of_map_elements > : : iterator ii = vector_for_sort . begin ( ) ; ii ! = vector_for_sort . end ( ) ; + + ii ) {
2014-11-14 11:45:14 +01:00
uint32_t client_ip = ( * ii ) . first ;
2015-01-27 20:29:51 +01:00
std : : string client_ip_as_string = convert_ip_as_uint_to_string ( ( * ii ) . first ) ;
2014-11-14 11:45:14 +01:00
2014-12-16 15:20:04 +01:00
uint64_t pps = 0 ;
uint64_t bps = 0 ;
uint64_t flows = 0 ;
2014-11-14 11:45:14 +01:00
2014-12-16 15:20:04 +01:00
uint64_t pps_average = 0 ;
uint64_t bps_average = 0 ;
uint64_t flows_average = 0 ;
2014-12-18 16:30:10 +01:00
// TODO: replace map by vector iteration
2014-12-02 15:11:27 +01:00
map_element * current_average_speed_element = & SpeedCounterAverage [ client_ip ] ;
2014-12-18 16:30:10 +01:00
map_element * current_speed_element = & SpeedCounter [ client_ip ] ;
2014-12-02 15:11:27 +01:00
2014-11-15 01:10:04 +01:00
// Create polymorphic pps, byte and flow counters
2014-11-14 11:45:14 +01:00
if ( data_direction = = INCOMING ) {
2014-12-04 22:01:08 +01:00
pps = current_speed_element - > in_packets ;
bps = current_speed_element - > in_bytes ;
flows = current_speed_element - > in_flows ;
2014-12-18 16:30:10 +01:00
2014-12-02 16:30:25 +01:00
pps_average = current_average_speed_element - > in_packets ;
bps_average = current_average_speed_element - > in_bytes ;
flows_average = current_average_speed_element - > in_flows ;
2014-11-14 11:45:14 +01:00
} else if ( data_direction = = OUTGOING ) {
2014-12-04 22:01:08 +01:00
pps = current_speed_element - > out_packets ;
bps = current_speed_element - > out_bytes ;
flows = current_speed_element - > out_flows ;
2014-12-02 16:30:25 +01:00
2014-12-02 15:11:27 +01:00
pps_average = current_average_speed_element - > out_packets ;
bps_average = current_average_speed_element - > out_bytes ;
2014-12-02 16:30:25 +01:00
flows_average = current_average_speed_element - > out_flows ;
2014-11-14 11:45:14 +01:00
}
2014-12-16 15:20:04 +01:00
uint64_t mbps = convert_speed_to_mbps ( bps ) ;
uint64_t mbps_average = convert_speed_to_mbps ( bps_average ) ;
2014-11-14 11:45:14 +01:00
2014-11-15 01:10:04 +01:00
// Print first max_ips_in_list elements in list, we will show top 20 "huge" channel loaders
2014-11-14 11:45:14 +01:00
if ( element_number < max_ips_in_list ) {
2015-01-27 20:29:51 +01:00
std : : string is_banned = ban_list . count ( client_ip ) > 0 ? " *banned* " : " " ;
2014-11-14 11:45:14 +01:00
// We use setw for alignment
2014-11-27 22:16:27 +01:00
output_buffer < < client_ip_as_string < < " \t \t " ;
2015-05-10 20:42:49 +02:00
if ( graphite_enabled ) {
std : : string direction_as_string ;
if ( data_direction = = INCOMING ) {
direction_as_string = " incoming " ;
} else if ( data_direction = = OUTGOING ) {
direction_as_string = " outgoing " ;
}
std : : string ip_as_string_with_dash_delimiters = client_ip_as_string ;
// Replace dots by dashes
std : : replace ( ip_as_string_with_dash_delimiters . begin ( ) , ip_as_string_with_dash_delimiters . end ( ) , ' . ' , ' _ ' ) ;
graphite_data [ graphite_prefix + ip_as_string_with_dash_delimiters + " . " + direction_as_string + " .pps " ] = pps ;
graphite_data [ graphite_prefix + ip_as_string_with_dash_delimiters + " . " + direction_as_string + " .mbps " ] = mbps ;
graphite_data [ graphite_prefix + ip_as_string_with_dash_delimiters + " . " + direction_as_string + " .flows " ] = flows ;
}
2014-11-27 22:16:27 +01:00
if ( print_average_traffic_counts ) {
2015-01-27 20:29:51 +01:00
output_buffer < < std : : setw ( 6 ) < < pps_average < < " pps " ;
output_buffer < < std : : setw ( 6 ) < < mbps_average < < " mbps " ;
output_buffer < < std : : setw ( 6 ) < < flows_average < < " flows " ;
2014-11-27 22:16:27 +01:00
} else {
2015-01-27 20:29:51 +01:00
output_buffer < < std : : setw ( 6 ) < < pps < < " pps " ;
output_buffer < < std : : setw ( 6 ) < < mbps < < " mbps " ;
output_buffer < < std : : setw ( 6 ) < < flows < < " flows " ;
2014-11-27 22:16:27 +01:00
}
2015-01-27 20:29:51 +01:00
output_buffer < < is_banned < < std : : endl ;
2014-11-14 11:45:14 +01:00
}
2013-10-31 20:09:02 +01:00
2014-11-14 11:45:14 +01:00
element_number + + ;
}
2014-06-24 12:42:43 +02:00
2015-05-11 14:28:14 +02:00
if ( graphite_enabled ) {
bool graphite_put_result = store_data_to_graphite ( graphite_port , graphite_host , graphite_data ) ;
2015-05-10 20:42:49 +02:00
2015-05-11 14:28:14 +02:00
if ( ! graphite_put_result ) {
logger < < log4cpp : : Priority : : ERROR < < " Can't store data to Graphite " ;
}
2015-05-10 20:42:49 +02:00
}
2015-05-11 14:28:14 +02:00
2014-06-24 12:42:43 +02:00
return output_buffer . str ( ) ;
2013-10-18 12:16:55 +02:00
}
2015-03-21 12:24:31 +01:00
// TODO: move to lirbary
2014-06-09 11:57:28 +02:00
// read whole file to vector
2015-01-27 20:29:51 +01:00
std : : vector < std : : string > read_file_to_vector ( std : : string file_name ) {
std : : vector < std : : string > data ;
std : : string line ;
2014-06-09 11:57:28 +02:00
2015-01-27 20:29:51 +01:00
std : : ifstream reading_file ;
2014-06-30 09:04:06 +02:00
reading_file . open ( file_name . c_str ( ) , std : : ifstream : : in ) ;
2014-06-09 11:57:28 +02:00
if ( reading_file . is_open ( ) ) {
while ( getline ( reading_file , line ) ) {
data . push_back ( line ) ;
}
2014-06-22 21:13:44 +02:00
} else {
2014-12-19 09:43:22 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Can't open file: " < < file_name ;
2014-06-09 11:57:28 +02:00
}
return data ;
}
2014-06-09 11:08:19 +02:00
// Load configuration
2014-11-23 20:17:56 +01:00
bool load_configuration_file ( ) {
2015-01-27 20:29:51 +01:00
std : : ifstream config_file ( global_config_path . c_str ( ) ) ;
std : : string line ;
2014-06-09 11:08:19 +02:00
2014-11-23 20:17:56 +01:00
if ( ! config_file . is_open ( ) ) {
2014-12-19 09:43:22 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Can't open config file " ;
2014-11-23 20:17:56 +01:00
return false ;
}
2014-06-09 11:08:19 +02:00
2014-11-23 20:17:56 +01:00
while ( getline ( config_file , line ) ) {
2015-04-27 14:48:11 +02:00
std : : vector < std : : string > parsed_config ;
if ( line . find ( " # " ) = = 0 or line . empty ( ) ) {
// Ignore comments line
continue ;
}
2014-12-19 09:43:22 +01:00
boost : : split ( parsed_config , line , boost : : is_any_of ( " = " ) , boost : : token_compress_on ) ;
if ( parsed_config . size ( ) = = 2 ) {
configuration_map [ parsed_config [ 0 ] ] = parsed_config [ 1 ] ;
} else {
2015-02-18 20:31:55 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Can't parse config line: ' " < < line < < " ' " ;
2014-12-19 09:43:22 +01:00
}
2014-11-23 20:17:56 +01:00
}
2014-10-29 10:21:53 +01:00
2014-12-12 14:10:20 +01:00
if ( configuration_map . count ( " enable_connection_tracking " ) ) {
if ( configuration_map [ " enable_connection_tracking " ] = = " on " ) {
enable_conection_tracking = true ;
} else {
enable_conection_tracking = false ;
}
}
2014-12-03 16:03:47 +01:00
if ( configuration_map . count ( " ban_time " ) ! = 0 ) {
standard_ban_time = convert_string_to_integer ( configuration_map [ " ban_time " ] ) ;
}
2014-12-02 15:11:27 +01:00
if ( configuration_map . count ( " average_calculation_time " ) ! = 0 ) {
average_calculation_amount = convert_string_to_integer ( configuration_map [ " average_calculation_time " ] ) ;
}
2014-12-02 13:43:34 +01:00
if ( configuration_map . count ( " threshold_pps " ) ! = 0 ) {
2014-11-23 20:17:56 +01:00
ban_threshold_pps = convert_string_to_integer ( configuration_map [ " threshold_pps " ] ) ;
}
2014-06-09 11:08:19 +02:00
2014-11-23 20:17:56 +01:00
if ( configuration_map . count ( " threshold_mbps " ) ! = 0 ) {
ban_threshold_mbps = convert_string_to_integer ( configuration_map [ " threshold_mbps " ] ) ;
}
2014-12-03 15:25:49 +01:00
if ( configuration_map . count ( " threshold_flows " ) ! = 0 ) {
2014-11-23 20:17:56 +01:00
ban_threshold_flows = convert_string_to_integer ( configuration_map [ " threshold_flows " ] ) ;
}
2014-11-16 15:41:30 +01:00
2014-12-05 10:46:01 +01:00
if ( configuration_map . count ( " enable_ban " ) ! = 0 ) {
if ( configuration_map [ " enable_ban " ] = = " on " ) {
we_do_real_ban = true ;
} else {
we_do_real_ban = false ;
}
}
2015-04-26 11:47:37 +02:00
if ( configuration_map . count ( " exabgp " ) ! = 0 ) {
if ( configuration_map [ " exabgp " ] = = " on " ) {
exabgp_enabled = true ;
} else {
exabgp_enabled = false ;
}
}
if ( exabgp_enabled ) {
//TODO: add community format validation
exabgp_community = configuration_map [ " exabgp_community " ] ;
if ( exabgp_community . empty ( ) ) {
logger < < log4cpp : : Priority : : ERROR < < " You enabled exabgp but not specified community, we disable exabgp support " ;
exabgp_enabled = false ;
}
}
if ( exabgp_enabled ) {
exabgp_command_pipe = configuration_map [ " exabgp_command_pipe " ] ;
if ( exabgp_command_pipe . empty ( ) ) {
logger < < log4cpp : : Priority : : ERROR
< < " You enabled exabgp but not specified exabgp_command_pipe, so we disable exabgp support " ;
exabgp_enabled = false ;
}
}
2015-04-26 14:35:01 +02:00
if ( exabgp_enabled ) {
exabgp_next_hop = configuration_map [ " exabgp_next_hop " ] ;
if ( exabgp_next_hop . empty ( ) ) {
logger < < log4cpp : : Priority : : ERROR
< < " You enabled exabgp but not specified exabgp_next_hop, so we disable exabgp support " ;
exabgp_enabled = false ;
}
2015-05-02 18:00:33 +02:00
if ( exabgp_enabled ) {
logger < < log4cpp : : Priority : : INFO < < " ExaBGP support initialized correctly " ;
}
}
2015-04-26 11:47:37 +02:00
2014-12-02 13:43:34 +01:00
if ( configuration_map . count ( " sflow " ) ! = 0 ) {
if ( configuration_map [ " sflow " ] = = " on " ) {
enable_sflow_collection = true ;
} else {
enable_sflow_collection = false ;
}
}
2015-01-23 17:37:23 +01:00
if ( configuration_map . count ( " netflow " ) ! = 0 ) {
if ( configuration_map [ " netflow " ] = = " on " ) {
enable_netflow_collection = true ;
} else {
enable_netflow_collection = false ;
}
}
2015-05-10 20:42:49 +02:00
// Graphite
if ( configuration_map . count ( " graphite " ) ! = 0 ) {
graphite_enabled = configuration_map [ " graphite " ] = = " on " ? true : false ;
}
if ( configuration_map . count ( " graphite_host " ) ! = 0 ) {
graphite_host = configuration_map [ " graphite_host " ] ;
}
if ( configuration_map . count ( " graphite_port " ) ! = 0 ) {
graphite_port = convert_string_to_integer ( configuration_map [ " graphite_port " ] ) ;
}
2015-01-28 08:49:12 +01:00
if ( configuration_map . count ( " process_incoming_traffic " ) ! = 0 ) {
process_incoming_traffic = configuration_map [ " process_incoming_traffic " ] = = " on " ? true : false ;
}
if ( configuration_map . count ( " process_outgoing_traffic " ) ! = 0 ) {
process_outgoing_traffic = configuration_map [ " process_outgoing_traffic " ] = = " on " ? true : false ;
}
2014-12-02 13:43:34 +01:00
if ( configuration_map . count ( " mirror " ) ! = 0 ) {
if ( configuration_map [ " mirror " ] = = " on " ) {
enable_data_collection_from_mirror = true ;
} else {
enable_data_collection_from_mirror = false ;
}
}
2015-03-10 15:17:52 +01:00
if ( configuration_map . count ( " mirror_netmap " ) ! = 0 ) {
if ( configuration_map [ " mirror_netmap " ] = = " on " ) {
enable_netmap_collection = true ;
} else {
enable_netmap_collection = false ;
}
}
if ( enable_netmap_collection & & enable_data_collection_from_mirror ) {
logger < < log4cpp : : Priority : : ERROR < < " You have enabled pfring and netmap data collection from mirror which strictly prohibited, please select one " ;
exit ( 1 ) ;
}
2015-01-27 10:56:16 +01:00
if ( configuration_map . count ( " pcap " ) ! = 0 ) {
if ( configuration_map [ " pcap " ] = = " on " ) {
enable_pcap_collection = true ;
} else {
enable_pcap_collection = false ;
}
}
2014-12-02 16:07:33 +01:00
if ( configuration_map . count ( " ban_for_pps " ) ! = 0 ) {
if ( configuration_map [ " ban_for_pps " ] = = " on " ) {
enable_ban_for_pps = true ;
} else {
enable_ban_for_pps = false ;
}
}
if ( configuration_map . count ( " ban_for_bandwidth " ) ! = 0 ) {
if ( configuration_map [ " ban_for_bandwidth " ] = = " on " ) {
enable_ban_for_bandwidth = true ;
} else {
enable_ban_for_bandwidth = false ;
}
}
if ( configuration_map . count ( " ban_for_flows " ) ! = 0 ) {
if ( configuration_map [ " ban_for_flows " ] = = " on " ) {
enable_ban_for_flows_per_second = true ;
} else {
enable_ban_for_flows_per_second = false ;
}
}
2015-04-27 18:21:13 +02:00
if ( configuration_map . count ( " white_list_path " ) ! = 0 ) {
white_list_path = configuration_map [ " white_list_path " ] ;
}
if ( configuration_map . count ( " networks_list_path " ) ! = 0 ) {
networks_list_path = configuration_map [ " networks_list_path " ] ;
}
2014-06-09 11:57:28 +02:00
# ifdef REDIS
2014-11-23 20:17:56 +01:00
if ( configuration_map . count ( " redis_port " ) ! = 0 ) {
redis_port = convert_string_to_integer ( configuration_map [ " redis_port " ] ) ;
}
2014-06-09 11:08:19 +02:00
2014-11-23 20:17:56 +01:00
if ( configuration_map . count ( " redis_host " ) ! = 0 ) {
redis_host = configuration_map [ " redis_host " ] ;
}
2014-06-09 12:33:07 +02:00
2014-11-23 20:17:56 +01:00
if ( configuration_map . count ( " redis_enabled " ) ! = 0 ) {
if ( configuration_map [ " redis_enabled " ] = = " yes " ) {
redis_enabled = true ;
} else {
redis_enabled = false ;
}
}
2014-06-09 11:57:28 +02:00
# endif
2014-06-09 11:08:19 +02:00
2014-11-23 20:17:56 +01:00
if ( configuration_map . count ( " ban_details_records_count " ) ! = 0 ) {
ban_details_records_count = convert_string_to_integer ( configuration_map [ " ban_details_records_count " ] ) ;
}
2014-06-09 11:08:19 +02:00
2014-11-23 20:17:56 +01:00
if ( configuration_map . count ( " check_period " ) ! = 0 ) {
2014-12-16 15:20:04 +01:00
check_period = convert_string_to_integer ( configuration_map [ " check_period " ] ) ;
2014-11-23 20:17:56 +01:00
}
2014-06-09 11:08:19 +02:00
2014-11-23 20:17:56 +01:00
if ( configuration_map . count ( " sort_parameter " ) ! = 0 ) {
sort_parameter = configuration_map [ " sort_parameter " ] ;
}
2014-06-09 11:08:19 +02:00
2014-11-23 20:17:56 +01:00
if ( configuration_map . count ( " max_ips_in_list " ) ! = 0 ) {
max_ips_in_list = convert_string_to_integer ( configuration_map [ " max_ips_in_list " ] ) ;
}
2014-06-09 11:08:19 +02:00
2014-11-23 20:17:56 +01:00
if ( configuration_map . count ( " notify_script_path " ) ! = 0 ) {
notify_script_path = configuration_map [ " notify_script_path " ] ;
2014-06-09 11:08:19 +02:00
}
2014-11-23 20:17:56 +01:00
return true ;
2014-06-09 11:08:19 +02:00
}
2014-06-22 21:04:43 +02:00
/* Enable core dumps for simplify debug tasks */
2014-06-22 14:25:12 +02:00
void enable_core_dumps ( ) {
struct rlimit rlim ;
2014-06-22 21:04:43 +02:00
int result = getrlimit ( RLIMIT_CORE , & rlim ) ;
if ( result ) {
2014-12-19 09:43:22 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Can't get current rlimit for RLIMIT_CORE " ;
2014-06-22 21:04:43 +02:00
return ;
} else {
2014-06-22 14:25:12 +02:00
rlim . rlim_cur = rlim . rlim_max ;
setrlimit ( RLIMIT_CORE , & rlim ) ;
}
}
2013-11-15 15:35:44 +01:00
2014-10-23 16:08:58 +02:00
void subnet_vectors_allocator ( prefix_t * prefix , void * data ) {
// Network byte order
uint32_t subnet_as_integer = prefix - > add . sin . s_addr ;
u_short bitlen = prefix - > bitlen ;
2015-01-27 21:00:44 +01:00
double base = 2 ;
int network_size_in_ips = pow ( base , 32 - bitlen ) ;
2014-10-23 16:08:58 +02:00
//logger<< log4cpp::Priority::INFO<<"Subnet: "<<prefix->add.sin.s_addr<<" network size: "<<network_size_in_ips;
2015-01-22 17:24:36 +01:00
logger < < log4cpp : : Priority : : INFO < < " I will allocate " < < network_size_in_ips < < " records for subnet " < < subnet_as_integer < < " cidr mask: " < < bitlen ;
2014-10-23 16:08:58 +02:00
// Initialize map element
SubnetVectorMap [ subnet_as_integer ] = vector_of_counters ( network_size_in_ips ) ;
// Zeroify all vector elements
map_element zero_map_element ;
memset ( & zero_map_element , 0 , sizeof ( zero_map_element ) ) ;
std : : fill ( SubnetVectorMap [ subnet_as_integer ] . begin ( ) , SubnetVectorMap [ subnet_as_integer ] . end ( ) , zero_map_element ) ;
2014-11-13 14:13:28 +01:00
2014-11-13 16:01:15 +01:00
// Initilize map element
SubnetVectorMapFlow [ subnet_as_integer ] = vector_of_flow_counters ( network_size_in_ips ) ;
2014-11-16 18:09:16 +01:00
// On creating it initilizes by zeros
2014-11-13 14:13:28 +01:00
conntrack_main_struct zero_conntrack_main_struct ;
std : : fill ( SubnetVectorMapFlow [ subnet_as_integer ] . begin ( ) , SubnetVectorMapFlow [ subnet_as_integer ] . end ( ) , zero_conntrack_main_struct ) ;
2014-10-23 16:08:58 +02:00
}
void zeroify_all_counters ( ) {
map_element zero_map_element ;
memset ( & zero_map_element , 0 , sizeof ( zero_map_element ) ) ;
2015-03-15 18:58:40 +01:00
for ( map_of_vector_counters : : iterator itr = SubnetVectorMap . begin ( ) ; itr ! = SubnetVectorMap . end ( ) ; + + itr ) {
2014-11-13 14:13:28 +01:00
//logger<< log4cpp::Priority::INFO<<"Zeroify "<<itr->first;
2014-10-23 16:08:58 +02:00
std : : fill ( itr - > second . begin ( ) , itr - > second . end ( ) , zero_map_element ) ;
}
}
2014-11-13 14:13:28 +01:00
void zeroify_all_flow_counters ( ) {
2014-11-16 18:09:16 +01:00
// On creating it initilizes by zeros
2014-11-13 14:13:28 +01:00
conntrack_main_struct zero_conntrack_main_struct ;
2014-11-13 16:01:15 +01:00
// Iterate over map
2015-03-15 18:58:40 +01:00
for ( map_of_vector_counters_for_flow : : iterator itr = SubnetVectorMapFlow . begin ( ) ; itr ! = SubnetVectorMapFlow . end ( ) ; + + itr ) {
2014-11-13 16:01:15 +01:00
// Iterate over vector
2015-03-15 18:58:40 +01:00
for ( vector_of_flow_counters : : iterator vector_iterator = itr - > second . begin ( ) ; vector_iterator ! = itr - > second . end ( ) ; + + vector_iterator ) {
2014-11-13 16:01:15 +01:00
// TODO: rewrite this monkey code
vector_iterator - > in_tcp . clear ( ) ;
vector_iterator - > in_udp . clear ( ) ;
vector_iterator - > in_icmp . clear ( ) ;
vector_iterator - > in_other . clear ( ) ;
vector_iterator - > out_tcp . clear ( ) ;
vector_iterator - > out_udp . clear ( ) ;
vector_iterator - > out_icmp . clear ( ) ;
vector_iterator - > out_other . clear ( ) ;
}
2014-11-13 14:13:28 +01:00
}
}
2014-06-22 14:25:12 +02:00
bool load_our_networks_list ( ) {
2015-04-27 18:11:08 +02:00
if ( file_exists ( white_list_path ) ) {
std : : vector < std : : string > network_list_from_config = read_file_to_vector ( white_list_path ) ;
2014-06-09 14:47:11 +02:00
2015-01-27 20:29:51 +01:00
for ( std : : vector < std : : string > : : iterator ii = network_list_from_config . begin ( ) ; ii ! = network_list_from_config . end ( ) ; + + ii ) {
2014-12-19 09:43:22 +01:00
if ( ii - > length ( ) > 0 & & is_cidr_subnet ( ii - > c_str ( ) ) ) {
2014-10-21 12:21:48 +02:00
make_and_lookup ( whitelist_tree , const_cast < char * > ( ii - > c_str ( ) ) ) ;
2014-10-22 15:27:15 +02:00
} else {
2014-12-19 09:43:22 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Can't parse line from whitelist: " < < * ii ;
2014-10-21 12:21:48 +02:00
}
2014-06-25 21:59:43 +02:00
}
2014-06-24 12:16:22 +02:00
logger < < log4cpp : : Priority : : INFO < < " We loaded " < < network_list_from_config . size ( ) < < " networks from whitelist file " ;
2014-06-09 14:47:11 +02:00
}
2015-01-27 20:29:51 +01:00
std : : vector < std : : string > networks_list_as_string ;
2014-11-15 01:10:04 +01:00
// We can bould "our subnets" automatically here
2013-10-18 12:59:58 +02:00
if ( file_exists ( " /proc/vz/version " ) ) {
2014-06-24 12:16:22 +02:00
logger < < log4cpp : : Priority : : INFO < < " We found OpenVZ " ;
2014-11-15 01:10:04 +01:00
// Add /32 CIDR mask for every IP here
2015-01-27 20:29:51 +01:00
std : : vector < std : : string > openvz_ips = read_file_to_vector ( " /proc/vz/veip " ) ;
for ( std : : vector < std : : string > : : iterator ii = openvz_ips . begin ( ) ; ii ! = openvz_ips . end ( ) ; + + ii ) {
2014-06-09 13:53:31 +02:00
// skip IPv6 addresses
if ( strstr ( ii - > c_str ( ) , " : " ) ! = NULL ) {
continue ;
}
// skip header
if ( strstr ( ii - > c_str ( ) , " Version " ) ! = NULL ) {
continue ;
}
2015-01-27 20:29:51 +01:00
std : : vector < std : : string > subnet_as_string ;
2014-06-09 13:53:31 +02:00
split ( subnet_as_string , * ii , boost : : is_any_of ( " " ) , boost : : token_compress_on ) ;
2015-01-27 20:29:51 +01:00
std : : string openvz_subnet = subnet_as_string [ 1 ] + " /32 " ;
2014-06-09 13:53:31 +02:00
networks_list_as_string . push_back ( openvz_subnet ) ;
2014-06-09 12:33:07 +02:00
}
2014-06-30 10:49:55 +02:00
2015-05-11 18:00:35 +02:00
logger < < log4cpp : : Priority : : INFO < < " We loaded " < < networks_list_as_string . size ( ) < < " networks from /proc/vz/veip " ;
}
if ( file_exists ( " /sbin/ip " ) ) {
logger < < log4cpp : : Priority : : INFO < < " We are working on Linux and could use ip tool for detecting local IP's " ;
ip_addresses_list_t ip_list = get_local_ip_addresses_list ( ) ;
logger < < log4cpp : : Priority : : INFO < < " We found " < < ip_list . size ( ) < < " local IP addresses and will monitor they " ;
for ( ip_addresses_list_t : : iterator iter = ip_list . begin ( ) ; iter ! = ip_list . end ( ) ; + + iter ) {
networks_list_as_string . push_back ( * iter + " /32 " ) ;
}
2013-10-18 12:16:55 +02:00
}
2013-10-18 12:59:58 +02:00
2015-04-27 18:11:08 +02:00
if ( file_exists ( networks_list_path ) ) {
2015-01-27 20:29:51 +01:00
std : : vector < std : : string > network_list_from_config = read_file_to_vector ( " /etc/networks_list " ) ;
2013-10-18 12:59:58 +02:00
networks_list_as_string . insert ( networks_list_as_string . end ( ) , network_list_from_config . begin ( ) , network_list_from_config . end ( ) ) ;
2014-06-09 16:28:56 +02:00
2014-06-24 12:16:22 +02:00
logger < < log4cpp : : Priority : : INFO < < " We loaded " < < network_list_from_config . size ( ) < < " networks from networks file " ;
2013-10-18 12:59:58 +02:00
}
2013-10-18 12:16:55 +02:00
2014-11-15 01:10:04 +01:00
// Some consistency checks
2013-10-18 12:16:55 +02:00
assert ( convert_ip_as_string_to_uint ( " 255.255.255.0 " ) = = convert_cidr_to_binary_netmask ( 24 ) ) ;
assert ( convert_ip_as_string_to_uint ( " 255.255.255.255 " ) = = convert_cidr_to_binary_netmask ( 32 ) ) ;
2015-02-18 20:31:55 +01:00
for ( std : : vector < std : : string > : : iterator ii = networks_list_as_string . begin ( ) ; ii ! = networks_list_as_string . end ( ) ; + + ii ) {
if ( ii - > length ( ) = = 0 ) {
// Skip blank lines in subnet list file silently
continue ;
}
2015-01-27 21:00:44 +01:00
2015-02-18 20:31:55 +01:00
if ( ! is_cidr_subnet ( ii - > c_str ( ) ) ) {
logger < < log4cpp : : Priority : : ERROR < < " Can't parse line from subnet list: ' " < < * ii < < " ' " ;
continue ;
}
std : : string network_address_in_cidr_form = * ii ;
unsigned int cidr_mask = get_cidr_mask_from_network_as_string ( network_address_in_cidr_form ) ;
std : : string network_address = get_net_address_from_network_as_string ( network_address_in_cidr_form ) ;
2014-12-16 11:56:08 +01:00
2015-02-18 20:31:55 +01:00
double base = 2 ;
total_number_of_hosts_in_our_networks + = pow ( base , 32 - cidr_mask ) ;
// Make sure it's "subnet address" and not an host address
uint32_t subnet_address_as_uint = convert_ip_as_string_to_uint ( network_address ) ;
uint32_t subnet_address_netmask_binary = convert_cidr_to_binary_netmask ( cidr_mask ) ;
uint32_t generated_subnet_address = subnet_address_as_uint & subnet_address_netmask_binary ;
if ( subnet_address_as_uint ! = generated_subnet_address ) {
std : : string new_network_address_as_string
= convert_ip_as_uint_to_string ( generated_subnet_address ) + " / " + convert_int_to_string ( cidr_mask ) ;
logger < < log4cpp : : Priority : : WARN < < " We will use " < < new_network_address_as_string
< < " instead of " < < network_address_in_cidr_form < < " because it's host address " ;
network_address_in_cidr_form = new_network_address_as_string ;
2014-10-21 12:21:48 +02:00
}
2015-02-18 20:31:55 +01:00
make_and_lookup ( lookup_tree , const_cast < char * > ( network_address_in_cidr_form . c_str ( ) ) ) ;
2014-06-25 21:59:43 +02:00
}
2014-10-23 16:08:58 +02:00
/* Preallocate data structures */
patricia_process ( lookup_tree , ( void_fn_t ) subnet_vectors_allocator ) ;
logger < < log4cpp : : Priority : : INFO < < " We start total zerofication of counters " ;
zeroify_all_counters ( ) ;
2015-02-18 20:31:55 +01:00
logger < < log4cpp : : Priority : : INFO < < " We finished zerofication " ;
2014-10-23 16:08:58 +02:00
2014-06-30 10:49:55 +02:00
logger < < log4cpp : : Priority : : INFO < < " We loaded " < < networks_list_as_string . size ( ) < < " subnets to our in-memory list of networks " ;
logger < < log4cpp : : Priority : : INFO < < " Total number of monitored hosts (total size of all networks): "
< < total_number_of_hosts_in_our_networks ;
2014-06-09 16:28:56 +02:00
return true ;
}
2014-11-15 01:10:04 +01:00
/* Process simple unified packet */
2014-06-21 16:11:53 +02:00
void process_packet ( simple_packet & current_packet ) {
2014-06-20 17:18:27 +02:00
// Packets dump is very useful for bug hunting
if ( DEBUG_DUMP_ALL_PACKETS ) {
2014-06-24 12:16:22 +02:00
logger < < log4cpp : : Priority : : INFO < < " Dump: " < < print_simple_packet ( current_packet ) ;
2014-06-20 17:18:27 +02:00
}
2013-10-18 15:28:00 +02:00
2014-10-23 16:08:58 +02:00
// Subnet for found IPs
unsigned long subnet = 0 ;
direction packet_direction = get_packet_direction ( current_packet . src_ip , current_packet . dst_ip , subnet ) ;
2015-01-28 08:49:12 +01:00
// Skip processing of specific traffic direction
if ( ( packet_direction = = INCOMING & & ! process_incoming_traffic ) or ( packet_direction = = OUTGOING & & ! process_outgoing_traffic ) ) {
return ;
}
2014-10-23 16:08:58 +02:00
uint32_t subnet_in_host_byte_order = 0 ;
// We operate in host bytes order and need to convert subnet
if ( subnet ! = 0 ) {
subnet_in_host_byte_order = ntohl ( subnet ) ;
}
// Try to find map key for this subnet
map_of_vector_counters : : iterator itr ;
if ( packet_direction = = OUTGOING or packet_direction = = INCOMING ) {
itr = SubnetVectorMap . find ( subnet ) ;
if ( itr = = SubnetVectorMap . end ( ) ) {
2014-12-19 09:43:22 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Can't find vector address in subnet map " ;
2014-10-23 16:08:58 +02:00
return ;
}
}
2013-10-21 21:45:33 +02:00
2014-11-13 14:13:28 +01:00
map_of_vector_counters_for_flow : : iterator itr_flow ;
2014-12-02 11:50:05 +01:00
if ( enable_conection_tracking ) {
if ( packet_direction = = OUTGOING or packet_direction = = INCOMING ) {
itr_flow = SubnetVectorMapFlow . find ( subnet ) ;
2014-11-13 14:13:28 +01:00
2014-12-02 11:50:05 +01:00
if ( itr_flow = = SubnetVectorMapFlow . end ( ) ) {
2014-12-19 09:43:22 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Can't find vector address in subnet flow map " ;
2014-12-02 11:50:05 +01:00
return ;
}
2014-11-13 14:13:28 +01:00
}
}
2015-01-26 09:09:03 +01:00
/* Because we support mirroring, sflow and netflow we should support different cases:
- One packet passed for processing ( mirror )
- Multiple packets ( " flows " ) passed for processing ( netflow )
- One sampled packed passed for processing ( netflow )
- Another combinations of this three options
*/
2015-03-22 22:30:35 +01:00
uint64_t sampled_number_of_packets = current_packet . number_of_packets * current_packet . sample_ratio ;
uint64_t sampled_number_of_bytes = current_packet . length * current_packet . sample_ratio ;
2014-12-02 10:30:20 +01:00
2014-12-18 16:30:10 +01:00
__sync_fetch_and_add ( & total_counters [ packet_direction ] . packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & total_counters [ packet_direction ] . bytes , sampled_number_of_bytes ) ;
2014-12-19 16:50:00 +01:00
// Incerementi main and per protocol packet counters
if ( packet_direction = = OUTGOING ) {
2015-02-14 12:01:43 +01:00
int64_t shift_in_vector = ( int64_t ) ntohl ( current_packet . src_ip ) - ( int64_t ) subnet_in_host_byte_order ;
2015-01-22 17:24:36 +01:00
2015-02-14 12:01:43 +01:00
if ( shift_in_vector < 0 or shift_in_vector > = itr - > second . size ( ) ) {
2015-01-23 10:54:41 +01:00
logger < < log4cpp : : Priority : : ERROR < < " We tried to access to element with index " < < shift_in_vector
2015-01-23 16:27:49 +01:00
< < " which located outside allocated vector with size " < < itr - > second . size ( ) ;
2015-01-23 10:54:41 +01:00
2015-02-14 12:01:43 +01:00
logger < < log4cpp : : Priority : : ERROR < < " We expect issues with this packet in OUTGOING direction: " < < print_simple_packet ( current_packet ) ;
2015-01-23 10:54:41 +01:00
2015-01-22 17:24:36 +01:00
return ;
}
2014-12-19 16:50:00 +01:00
map_element * current_element = & itr - > second [ shift_in_vector ] ;
2014-06-26 20:27:55 +02:00
2014-12-19 16:55:43 +01:00
// Main packet/bytes counter
2014-12-19 16:50:00 +01:00
__sync_fetch_and_add ( & current_element - > out_packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & current_element - > out_bytes , sampled_number_of_bytes ) ;
2014-10-23 16:08:58 +02:00
2015-05-07 15:09:43 +02:00
// Fragmented IP packets
2015-05-08 09:54:39 +02:00
if ( current_packet . ip_fragmented ) {
__sync_fetch_and_add ( & current_element - > fragmented_out_packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & current_element - > fragmented_out_bytes , sampled_number_of_bytes ) ;
}
2015-05-07 15:09:43 +02:00
2014-12-19 16:50:00 +01:00
conntrack_main_struct * current_element_flow = NULL ;
if ( enable_conection_tracking ) {
current_element_flow = & itr_flow - > second [ shift_in_vector ] ;
}
2014-06-28 12:12:56 +02:00
2014-12-19 16:50:00 +01:00
// Collect data when ban client
2015-03-15 18:58:40 +01:00
if ( ! ban_list_details . empty ( ) & & ban_list_details . count ( current_packet . src_ip ) > 0 & &
2014-12-19 16:50:00 +01:00
ban_list_details [ current_packet . src_ip ] . size ( ) < ban_details_records_count ) {
2014-12-02 11:50:05 +01:00
2014-12-19 16:50:00 +01:00
ban_list_details_mutex . lock ( ) ;
ban_list_details [ current_packet . src_ip ] . push_back ( current_packet ) ;
ban_list_details_mutex . unlock ( ) ;
}
2014-12-02 11:50:05 +01:00
2014-12-19 16:50:00 +01:00
uint64_t connection_tracking_hash = 0 ;
2013-10-18 15:28:00 +02:00
2014-12-19 16:50:00 +01:00
if ( enable_conection_tracking ) {
packed_conntrack_hash flow_tracking_structure ;
flow_tracking_structure . opposite_ip = current_packet . dst_ip ;
flow_tracking_structure . src_port = current_packet . source_port ;
flow_tracking_structure . dst_port = current_packet . destination_port ;
2014-11-13 16:01:15 +01:00
2014-12-19 16:50:00 +01:00
// convert this struct to 64 bit integer
connection_tracking_hash = convert_conntrack_hash_struct_to_integer ( & flow_tracking_structure ) ;
}
2014-11-13 16:01:15 +01:00
2014-12-19 16:50:00 +01:00
if ( current_packet . protocol = = IPPROTO_TCP ) {
__sync_fetch_and_add ( & current_element - > tcp_out_packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & current_element - > tcp_out_bytes , sampled_number_of_bytes ) ;
2014-12-18 16:30:10 +01:00
2015-05-07 11:51:48 +02:00
if ( extract_bit_value ( current_packet . flags , TCP_SYN_FLAG_SHIFT ) ) {
__sync_fetch_and_add ( & current_element - > tcp_syn_out_packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & current_element - > tcp_syn_out_bytes , sampled_number_of_bytes ) ;
}
2014-12-19 16:50:00 +01:00
if ( enable_conection_tracking ) {
flow_counter . lock ( ) ;
conntrack_key_struct * conntrack_key_struct_ptr = & current_element_flow - > out_tcp [ connection_tracking_hash ] ;
2014-11-14 10:55:23 +01:00
2014-12-19 16:50:00 +01:00
conntrack_key_struct_ptr - > packets + = sampled_number_of_packets ;
conntrack_key_struct_ptr - > bytes + = sampled_number_of_bytes ;
2014-12-02 11:50:05 +01:00
2014-12-19 16:50:00 +01:00
flow_counter . unlock ( ) ;
}
} else if ( current_packet . protocol = = IPPROTO_UDP ) {
__sync_fetch_and_add ( & current_element - > udp_out_packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & current_element - > udp_out_bytes , sampled_number_of_bytes ) ;
2014-11-13 16:01:15 +01:00
2014-12-19 16:50:00 +01:00
if ( enable_conection_tracking ) {
flow_counter . lock ( ) ;
conntrack_key_struct * conntrack_key_struct_ptr = & current_element_flow - > out_udp [ connection_tracking_hash ] ;
2014-11-14 10:55:23 +01:00
2014-12-19 16:50:00 +01:00
conntrack_key_struct_ptr - > packets + = sampled_number_of_packets ;
conntrack_key_struct_ptr - > bytes + = sampled_number_of_bytes ;
2014-11-14 10:55:23 +01:00
2014-12-19 16:50:00 +01:00
flow_counter . unlock ( ) ;
2014-12-02 11:50:05 +01:00
}
2014-12-19 16:50:00 +01:00
} else if ( current_packet . protocol = = IPPROTO_ICMP ) {
__sync_fetch_and_add ( & current_element - > icmp_out_packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & current_element - > icmp_out_bytes , sampled_number_of_bytes ) ;
2014-12-19 16:55:43 +01:00
// no flow tracking for icmp
2014-12-19 16:50:00 +01:00
} else {
2014-10-19 13:30:25 +02:00
2014-12-19 16:50:00 +01:00
}
} else if ( packet_direction = = INCOMING ) {
2015-02-14 12:01:43 +01:00
int64_t shift_in_vector = ( int64_t ) ntohl ( current_packet . dst_ip ) - ( int64_t ) subnet_in_host_byte_order ;
2015-01-22 17:24:36 +01:00
2015-02-14 12:01:43 +01:00
if ( shift_in_vector < 0 or shift_in_vector > = itr - > second . size ( ) ) {
2015-01-23 10:54:41 +01:00
logger < < log4cpp : : Priority : : ERROR < < " We tried to access to element with index " < < shift_in_vector
2015-01-23 16:27:49 +01:00
< < " which located outside allocated vector with size " < < itr - > second . size ( ) ;
2015-01-23 10:54:41 +01:00
logger < < log4cpp : : Priority : : INFO < < " We expect issues with this packet in INCOMING direction: " < < print_simple_packet ( current_packet ) ;
2015-01-22 17:24:36 +01:00
return ;
}
2014-12-19 16:50:00 +01:00
map_element * current_element = & itr - > second [ shift_in_vector ] ;
2014-12-19 16:55:43 +01:00
// Main packet/bytes counter
2014-12-19 16:50:00 +01:00
__sync_fetch_and_add ( & current_element - > in_packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & current_element - > in_bytes , sampled_number_of_bytes ) ;
2014-11-13 16:25:45 +01:00
2015-05-07 15:09:43 +02:00
// Count fragmented IP packets
2015-05-08 09:54:39 +02:00
if ( current_packet . ip_fragmented ) {
__sync_fetch_and_add ( & current_element - > fragmented_in_packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & current_element - > fragmented_in_bytes , sampled_number_of_bytes ) ;
}
2015-05-07 15:09:43 +02:00
2014-12-19 16:50:00 +01:00
conntrack_main_struct * current_element_flow = NULL ;
2014-12-02 11:50:05 +01:00
2014-12-19 16:50:00 +01:00
if ( enable_conection_tracking ) {
current_element_flow = & itr_flow - > second [ shift_in_vector ] ;
}
2014-12-02 11:50:05 +01:00
2014-12-19 16:50:00 +01:00
uint64_t connection_tracking_hash = 0 ;
if ( enable_conection_tracking ) {
packed_conntrack_hash flow_tracking_structure ;
flow_tracking_structure . opposite_ip = current_packet . src_ip ;
flow_tracking_structure . src_port = current_packet . source_port ;
flow_tracking_structure . dst_port = current_packet . destination_port ;
// convert this struct to 64 bit integer
connection_tracking_hash = convert_conntrack_hash_struct_to_integer ( & flow_tracking_structure ) ;
}
2013-10-18 15:28:00 +02:00
2014-12-19 16:50:00 +01:00
// Collect attack details
2015-03-15 18:58:40 +01:00
if ( ! ban_list_details . empty ( ) & & ban_list_details . count ( current_packet . dst_ip ) > 0 & &
2014-12-19 16:50:00 +01:00
ban_list_details [ current_packet . dst_ip ] . size ( ) < ban_details_records_count ) {
2014-11-13 16:25:45 +01:00
2014-12-19 16:50:00 +01:00
ban_list_details_mutex . lock ( ) ;
ban_list_details [ current_packet . dst_ip ] . push_back ( current_packet ) ;
ban_list_details_mutex . unlock ( ) ;
}
2014-11-14 10:55:23 +01:00
2014-12-19 16:50:00 +01:00
if ( current_packet . protocol = = IPPROTO_TCP ) {
__sync_fetch_and_add ( & current_element - > tcp_in_packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & current_element - > tcp_in_bytes , sampled_number_of_bytes ) ;
2014-11-14 10:55:23 +01:00
2015-05-07 11:51:48 +02:00
if ( extract_bit_value ( current_packet . flags , TCP_SYN_FLAG_SHIFT ) ) {
__sync_fetch_and_add ( & current_element - > tcp_syn_in_packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & current_element - > tcp_syn_in_bytes , sampled_number_of_bytes ) ;
}
2014-12-19 16:50:00 +01:00
if ( enable_conection_tracking ) {
flow_counter . lock ( ) ;
conntrack_key_struct * conntrack_key_struct_ptr = & current_element_flow - > in_tcp [ connection_tracking_hash ] ;
2014-11-13 16:25:45 +01:00
2014-12-19 16:50:00 +01:00
conntrack_key_struct_ptr - > packets + = sampled_number_of_packets ;
conntrack_key_struct_ptr - > bytes + = sampled_number_of_bytes ;
2014-11-13 16:25:45 +01:00
2014-12-19 16:50:00 +01:00
flow_counter . unlock ( ) ;
}
} else if ( current_packet . protocol = = IPPROTO_UDP ) {
__sync_fetch_and_add ( & current_element - > udp_in_packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & current_element - > udp_in_bytes , sampled_number_of_bytes ) ;
2014-11-14 10:55:23 +01:00
2014-12-19 16:50:00 +01:00
if ( enable_conection_tracking ) {
flow_counter . lock ( ) ;
conntrack_key_struct * conntrack_key_struct_ptr = & current_element_flow - > in_udp [ connection_tracking_hash ] ;
2014-12-05 14:09:25 +01:00
2014-12-19 16:50:00 +01:00
conntrack_key_struct_ptr - > packets + = sampled_number_of_packets ;
conntrack_key_struct_ptr - > bytes + = sampled_number_of_bytes ;
flow_counter . unlock ( ) ;
2014-12-02 11:50:05 +01:00
}
2014-12-19 16:50:00 +01:00
} else if ( current_packet . protocol = = IPPROTO_ICMP ) {
__sync_fetch_and_add ( & current_element - > icmp_in_packets , sampled_number_of_packets ) ;
__sync_fetch_and_add ( & current_element - > icmp_in_bytes , sampled_number_of_bytes ) ;
2014-10-19 13:30:25 +02:00
2015-05-06 23:09:37 +02:00
// no flow tracking for icmp
2014-12-19 16:50:00 +01:00
} else {
// TBD
2014-12-02 11:50:05 +01:00
}
2014-12-19 16:50:00 +01:00
} else if ( packet_direction = = INTERNAL ) {
2013-10-18 12:16:55 +02:00
}
2014-06-25 17:32:32 +02:00
}
2013-10-18 12:16:55 +02:00
2013-12-28 20:11:20 +01:00
# ifdef GEOIP
2014-07-04 10:02:44 +02:00
unsigned int get_asn_for_ip ( uint32_t ip ) {
2014-06-25 17:32:32 +02:00
char * asn_raw = GeoIP_org_by_name ( geo_ip , convert_ip_as_uint_to_string ( remote_ip ) . c_str ( ) ) ;
uint32_t asn_number = 0 ;
if ( asn_raw = = NULL ) {
asn_number = 0 ;
} else {
// split string: AS1299 TeliaSonera International Carrier
2015-01-27 20:29:51 +01:00
std : : vector < std : : string > asn_as_string ;
2014-06-25 17:32:32 +02:00
split ( asn_as_string , asn_raw , boost : : is_any_of ( " " ) , boost : : token_compress_on ) ;
2013-12-28 20:11:20 +01:00
2014-06-25 17:32:32 +02:00
// free up original string
free ( asn_raw ) ;
2013-12-28 20:11:20 +01:00
2014-06-25 17:32:32 +02:00
// extract raw number
asn_number = convert_string_to_integer ( asn_as_string [ 0 ] . substr ( 2 ) ) ;
2013-12-28 20:11:20 +01:00
}
2014-06-25 17:32:32 +02:00
return asn_number ;
2013-10-21 16:47:31 +02:00
}
2014-06-25 17:32:32 +02:00
# endif
2013-10-21 16:47:31 +02:00
2013-10-21 21:45:33 +02:00
// void* void* data
2014-11-27 22:16:27 +01:00
// It's not an calculation thread, it's vizualization thread :)
2013-10-21 21:45:33 +02:00
void calculation_thread ( ) {
2014-06-25 17:32:32 +02:00
// we need wait one second for calculating speed by recalculate_speed
2014-12-16 15:20:04 +01:00
//#include <sys/prctl.h>
2014-10-23 10:51:01 +02:00
//prctl(PR_SET_NAME , "fastnetmon calc thread", 0, 0, 0);
2014-12-16 15:20:04 +01:00
// Sleep for a half second for shift against calculatiuon thread
boost : : this_thread : : sleep ( boost : : posix_time : : milliseconds ( 500 ) ) ;
2013-10-21 21:45:33 +02:00
while ( 1 ) {
2015-04-27 09:54:51 +02:00
// Available only from boost 1.54: boost::this_thread::sleep_for( boost::chrono::seconds(check_period) );
2014-06-29 13:28:56 +02:00
boost : : this_thread : : sleep ( boost : : posix_time : : seconds ( check_period ) ) ;
2014-12-16 12:59:24 +01:00
traffic_draw_programm ( ) ;
2013-10-21 21:45:33 +02:00
}
}
2014-06-25 17:32:32 +02:00
void recalculate_speed_thread_handler ( ) {
while ( 1 ) {
// recalculate data every one second
2015-04-27 09:54:51 +02:00
// Available only from boost 1.54: boost::this_thread::sleep_for( boost::chrono::seconds(1) );
2014-06-29 13:28:56 +02:00
boost : : this_thread : : sleep ( boost : : posix_time : : seconds ( 1 ) ) ;
2014-06-25 17:32:32 +02:00
recalculate_speed ( ) ;
}
}
2014-11-15 01:10:04 +01:00
/* Calculate speed for all connnections */
2014-06-25 17:32:32 +02:00
void recalculate_speed ( ) {
2014-12-16 15:20:04 +01:00
//logger<< log4cpp::Priority::INFO<<"We run recalculate_speed";
2014-06-28 13:27:57 +02:00
2014-12-16 12:59:24 +01:00
struct timeval start_calc_time ;
gettimeofday ( & start_calc_time , NULL ) ;
2014-06-28 23:04:31 +02:00
double speed_calc_period = 1 ;
2014-12-16 12:59:24 +01:00
time_t start_time ;
time ( & start_time ) ;
2014-06-28 23:04:31 +02:00
2014-12-05 14:31:55 +01:00
// If we got 1+ seconds lag we should use new "delta" or skip this step
2014-12-16 12:59:24 +01:00
double time_difference = difftime ( start_time , last_call_of_traffic_recalculation ) ;
2014-06-28 23:04:31 +02:00
if ( time_difference < 1 ) {
// It could occur on programm start
2014-12-05 14:31:55 +01:00
logger < < log4cpp : : Priority : : INFO < < " We skip one iteration of speed_calc because it runs so early! " ;
2014-06-28 23:04:31 +02:00
return ;
} else if ( int ( time_difference ) = = 1 ) {
2014-11-15 01:10:04 +01:00
// All fine, we run on time
2014-06-28 23:04:31 +02:00
} else {
logger < < log4cpp : : Priority : : INFO < < " Time from last run of speed_recalc is soooo big, we got ugly lags: " < < time_difference ;
speed_calc_period = time_difference ;
}
2014-10-23 16:08:58 +02:00
map_element zero_map_element ;
memset ( & zero_map_element , 0 , sizeof ( zero_map_element ) ) ;
2014-11-14 11:45:14 +01:00
2014-12-16 15:20:04 +01:00
uint64_t incoming_total_flows = 0 ;
uint64_t outgoing_total_flows = 0 ;
2014-11-14 11:45:14 +01:00
2014-10-23 16:08:58 +02:00
for ( map_of_vector_counters : : iterator itr = SubnetVectorMap . begin ( ) ; itr ! = SubnetVectorMap . end ( ) ; + + itr ) {
for ( vector_of_counters : : iterator vector_itr = itr - > second . begin ( ) ; vector_itr ! = itr - > second . end ( ) ; + + vector_itr ) {
int current_index = vector_itr - itr - > second . begin ( ) ;
2014-12-05 14:47:57 +01:00
// New element
map_element new_speed_element ;
2014-10-23 16:08:58 +02:00
// convert to host order for math operations
uint32_t subnet_ip = ntohl ( itr - > first ) ;
uint32_t client_ip_in_host_bytes_order = subnet_ip + current_index ;
// covnert to our standard network byte order
uint32_t client_ip = htonl ( client_ip_in_host_bytes_order ) ;
2014-12-16 15:20:04 +01:00
new_speed_element . in_packets = uint64_t ( ( double ) vector_itr - > in_packets / speed_calc_period ) ;
new_speed_element . out_packets = uint64_t ( ( double ) vector_itr - > out_packets / speed_calc_period ) ;
2014-10-23 16:08:58 +02:00
2014-12-16 15:20:04 +01:00
new_speed_element . in_bytes = uint64_t ( ( double ) vector_itr - > in_bytes / speed_calc_period ) ;
new_speed_element . out_bytes = uint64_t ( ( double ) vector_itr - > out_bytes / speed_calc_period ) ;
2014-10-23 16:08:58 +02:00
2015-05-07 15:09:43 +02:00
// Fragmented
new_speed_element . fragmented_in_packets = uint64_t ( ( double ) vector_itr - > fragmented_in_packets / speed_calc_period ) ;
new_speed_element . fragmented_out_packets = uint64_t ( ( double ) vector_itr - > fragmented_out_packets / speed_calc_period ) ;
new_speed_element . fragmented_in_bytes = uint64_t ( ( double ) vector_itr - > fragmented_in_bytes / speed_calc_period ) ;
new_speed_element . fragmented_out_bytes = uint64_t ( ( double ) vector_itr - > fragmented_out_bytes / speed_calc_period ) ;
2014-12-05 15:34:59 +01:00
// By protocol counters
// TCP
2014-12-16 15:20:04 +01:00
new_speed_element . tcp_in_packets = uint64_t ( ( double ) vector_itr - > tcp_in_packets / speed_calc_period ) ;
new_speed_element . tcp_out_packets = uint64_t ( ( double ) vector_itr - > tcp_out_packets / speed_calc_period ) ;
2014-12-05 15:34:59 +01:00
2014-12-16 15:20:04 +01:00
new_speed_element . tcp_in_bytes = uint64_t ( ( double ) vector_itr - > tcp_in_bytes / speed_calc_period ) ;
new_speed_element . tcp_out_bytes = uint64_t ( ( double ) vector_itr - > tcp_out_bytes / speed_calc_period ) ;
2014-12-05 15:34:59 +01:00
2015-05-07 11:51:48 +02:00
// TCP syn
new_speed_element . tcp_syn_in_packets = uint64_t ( ( double ) vector_itr - > tcp_syn_in_packets / speed_calc_period ) ;
new_speed_element . tcp_syn_out_packets = uint64_t ( ( double ) vector_itr - > tcp_syn_out_packets / speed_calc_period ) ;
new_speed_element . tcp_syn_in_bytes = uint64_t ( ( double ) vector_itr - > tcp_syn_in_bytes / speed_calc_period ) ;
new_speed_element . tcp_syn_out_bytes = uint64_t ( ( double ) vector_itr - > tcp_syn_out_bytes / speed_calc_period ) ;
2014-12-05 15:34:59 +01:00
// UDP
2014-12-16 15:20:04 +01:00
new_speed_element . udp_in_packets = uint64_t ( ( double ) vector_itr - > udp_in_packets / speed_calc_period ) ;
new_speed_element . udp_out_packets = uint64_t ( ( double ) vector_itr - > udp_out_packets / speed_calc_period ) ;
2014-12-05 15:34:59 +01:00
2014-12-16 15:20:04 +01:00
new_speed_element . udp_in_bytes = uint64_t ( ( double ) vector_itr - > udp_in_bytes / speed_calc_period ) ;
new_speed_element . udp_out_bytes = uint64_t ( ( double ) vector_itr - > udp_out_bytes / speed_calc_period ) ;
2014-12-05 15:34:59 +01:00
// ICMP
2014-12-16 15:20:04 +01:00
new_speed_element . icmp_in_packets = uint64_t ( ( double ) vector_itr - > icmp_in_packets / speed_calc_period ) ;
new_speed_element . icmp_out_packets = uint64_t ( ( double ) vector_itr - > icmp_out_packets / speed_calc_period ) ;
2014-12-05 15:34:59 +01:00
2014-12-16 15:20:04 +01:00
new_speed_element . icmp_in_bytes = uint64_t ( ( double ) vector_itr - > icmp_in_bytes / speed_calc_period ) ;
new_speed_element . icmp_out_bytes = uint64_t ( ( double ) vector_itr - > icmp_out_bytes / speed_calc_period ) ;
2014-12-05 15:34:59 +01:00
2014-11-14 10:55:23 +01:00
conntrack_main_struct * flow_counter_ptr = & SubnetVectorMapFlow [ itr - > first ] [ current_index ] ;
// todo: optimize this operations!
2014-12-16 15:20:04 +01:00
uint64_t total_out_flows =
( uint64_t ) flow_counter_ptr - > out_tcp . size ( ) +
( uint64_t ) flow_counter_ptr - > out_udp . size ( ) +
( uint64_t ) flow_counter_ptr - > out_icmp . size ( ) +
( uint64_t ) flow_counter_ptr - > out_other . size ( ) ;
2014-11-14 10:55:23 +01:00
2014-12-16 15:20:04 +01:00
uint64_t total_in_flows =
( uint64_t ) flow_counter_ptr - > in_tcp . size ( ) +
( uint64_t ) flow_counter_ptr - > in_udp . size ( ) +
( uint64_t ) flow_counter_ptr - > in_icmp . size ( ) +
( uint64_t ) flow_counter_ptr - > in_other . size ( ) ;
2014-11-14 10:55:23 +01:00
2014-12-16 15:20:04 +01:00
new_speed_element . out_flows = uint64_t ( ( double ) total_out_flows / speed_calc_period ) ;
new_speed_element . in_flows = uint64_t ( ( double ) total_in_flows / speed_calc_period ) ;
2014-11-14 10:55:23 +01:00
2014-11-14 11:45:14 +01:00
// Increment global counter
2014-12-05 14:47:57 +01:00
incoming_total_flows + = new_speed_element . in_flows ;
outgoing_total_flows + = new_speed_element . out_flows ;
2014-11-14 11:45:14 +01:00
2014-12-02 15:46:46 +01:00
/* Moving average recalculation */
// http://en.wikipedia.org/wiki/Moving_average#Application_to_measuring_computer_performance
2014-12-16 15:20:04 +01:00
//double speed_calc_period = 1;
2014-12-02 15:46:46 +01:00
double exp_power = - speed_calc_period / average_calculation_amount ;
double exp_value = exp ( exp_power ) ;
map_element * current_average_speed_element = & SpeedCounterAverage [ client_ip ] ;
2014-12-16 15:20:04 +01:00
current_average_speed_element - > in_bytes = uint64_t ( new_speed_element . in_bytes + exp_value *
2014-12-05 14:47:57 +01:00
( ( double ) current_average_speed_element - > in_bytes - ( double ) new_speed_element . in_bytes ) ) ;
2014-12-16 15:20:04 +01:00
current_average_speed_element - > out_bytes = uint64_t ( new_speed_element . out_bytes + exp_value *
2014-12-05 14:47:57 +01:00
( ( double ) current_average_speed_element - > out_bytes - ( double ) new_speed_element . out_bytes ) ) ;
2014-12-02 15:46:46 +01:00
2014-12-16 15:20:04 +01:00
current_average_speed_element - > in_packets = uint64_t ( new_speed_element . in_packets + exp_value *
2014-12-05 14:47:57 +01:00
( ( double ) current_average_speed_element - > in_packets - ( double ) new_speed_element . in_packets ) ) ;
2014-12-16 15:20:04 +01:00
current_average_speed_element - > out_packets = uint64_t ( new_speed_element . out_packets + exp_value *
2014-12-05 14:47:57 +01:00
( ( double ) current_average_speed_element - > out_packets - ( double ) new_speed_element . out_packets ) ) ;
2014-12-02 16:15:26 +01:00
2014-12-16 15:20:04 +01:00
current_average_speed_element - > out_flows = uint64_t ( new_speed_element . out_flows + exp_value *
2014-12-05 14:47:57 +01:00
( ( double ) current_average_speed_element - > out_flows - ( double ) new_speed_element . out_flows ) ) ;
2014-12-16 15:20:04 +01:00
current_average_speed_element - > in_flows = uint64_t ( new_speed_element . in_flows + exp_value *
2014-12-05 14:47:57 +01:00
( ( double ) current_average_speed_element - > in_flows - ( double ) new_speed_element . in_flows ) ) ;
2014-12-02 16:15:26 +01:00
2014-12-02 16:34:49 +01:00
/* Moving average recalculation end */
2014-12-02 15:46:46 +01:00
2015-03-13 16:25:13 +01:00
if ( we_should_ban_this_ip ( current_average_speed_element ) ) {
2015-01-27 20:29:51 +01:00
std : : string flow_attack_details = " " ;
2014-12-19 16:50:00 +01:00
if ( enable_conection_tracking ) {
flow_attack_details = print_flow_tracking_for_ip ( * flow_counter_ptr , convert_ip_as_uint_to_string ( client_ip ) ) ;
}
2014-12-02 16:07:33 +01:00
// TODO: we should pass type of ddos ban source (pps, flowd, bandwidth)!
2015-03-13 17:24:57 +01:00
execute_ip_ban ( client_ip , new_speed_element , * current_average_speed_element , flow_attack_details ) ;
2014-11-13 09:01:52 +01:00
}
2014-10-23 16:08:58 +02:00
speed_counters_mutex . lock ( ) ;
2014-12-05 14:47:57 +01:00
//map_element* current_speed_element = &SpeedCounter[client_ip];
//*current_speed_element = new_speed_element;
SpeedCounter [ client_ip ] = new_speed_element ;
2014-10-23 16:08:58 +02:00
speed_counters_mutex . unlock ( ) ;
data_counters_mutex . lock ( ) ;
* vector_itr = zero_map_element ;
data_counters_mutex . unlock ( ) ;
}
}
2014-11-14 11:45:14 +01:00
// Calculate global flow speed
2014-12-16 15:20:04 +01:00
incoming_total_flows_speed = uint64_t ( ( double ) incoming_total_flows / ( double ) speed_calc_period ) ;
outgoing_total_flows_speed = uint64_t ( ( double ) outgoing_total_flows / ( double ) speed_calc_period ) ;
2014-11-14 11:45:14 +01:00
2014-12-02 11:50:05 +01:00
if ( enable_conection_tracking ) {
// Clean Flow Counter
flow_counter . lock ( ) ;
zeroify_all_flow_counters ( ) ;
flow_counter . unlock ( ) ;
}
2014-10-22 04:29:55 +02:00
2014-07-04 10:02:44 +02:00
for ( unsigned int index = 0 ; index < 4 ; index + + ) {
2014-12-16 15:20:04 +01:00
total_speed_counters [ index ] . bytes = uint64_t ( ( double ) total_counters [ index ] . bytes / ( double ) speed_calc_period ) ;
total_speed_counters [ index ] . packets = uint64_t ( ( double ) total_counters [ index ] . packets / ( double ) speed_calc_period ) ;
2014-06-30 13:10:07 +02:00
// nullify data counters after speed calculation
2014-12-18 16:30:10 +01:00
//total_counters_mutex.lock();
2014-06-30 13:10:07 +02:00
total_counters [ index ] . bytes = 0 ;
total_counters [ index ] . packets = 0 ;
2014-12-18 16:30:10 +01:00
//total_counters_mutex.unlock();
2014-06-30 13:10:07 +02:00
}
2014-11-15 01:10:04 +01:00
// Set time of previous startup
2014-06-28 23:04:31 +02:00
time ( & last_call_of_traffic_recalculation ) ;
2014-12-16 12:59:24 +01:00
struct timeval finish_calc_time ;
gettimeofday ( & finish_calc_time , NULL ) ;
timeval_subtract ( & speed_calculation_time , & finish_calc_time , & start_calc_time ) ;
2014-06-25 17:32:32 +02:00
}
2014-06-09 14:47:11 +02:00
2015-01-27 20:29:51 +01:00
void print_screen_contents_into_file ( std : : string screen_data_stats_param ) {
std : : ofstream screen_data_file ;
screen_data_file . open ( " /tmp/fastnetmon.dat " , std : : ios : : trunc ) ;
2014-12-21 12:52:22 +01:00
if ( screen_data_file . is_open ( ) ) {
screen_data_file < < screen_data_stats_param ;
screen_data_file . close ( ) ;
} else {
logger < < log4cpp : : Priority : : ERROR < < " Can't print programm screen into file " ;
}
}
2014-12-16 12:59:24 +01:00
void traffic_draw_programm ( ) {
2015-01-27 20:29:51 +01:00
std : : stringstream output_buffer ;
2014-12-16 15:20:04 +01:00
//logger<<log4cpp::Priority::INFO<<"Draw table call";
2014-10-23 10:51:01 +02:00
struct timeval start_calc_time ;
gettimeofday ( & start_calc_time , NULL ) ;
2013-10-21 21:45:33 +02:00
2014-06-25 17:39:59 +02:00
sort_type sorter ;
if ( sort_parameter = = " packets " ) {
sorter = PACKETS ;
} else if ( sort_parameter = = " bytes " ) {
sorter = BYTES ;
2014-11-14 10:55:23 +01:00
} else if ( sort_parameter = = " flows " ) {
sorter = FLOWS ;
2014-06-25 17:39:59 +02:00
} else {
logger < < log4cpp : : Priority : : INFO < < " Unexpected sorter type: " < < sort_parameter ;
sorter = PACKETS ;
}
2013-10-21 15:43:00 +02:00
2014-10-29 10:21:53 +01:00
output_buffer < < " FastNetMon v1.0 FastVPS Eesti OU (c) VPS and dedicated: http://FastVPS.host " < < " \n "
2014-12-22 16:39:46 +01:00
< < " IPs ordered by: " < < sort_parameter < < " \n " ;
2013-10-18 12:16:55 +02:00
2015-01-27 20:29:51 +01:00
output_buffer < < print_channel_speed ( " Incoming traffic " , INCOMING ) < < std : : endl ;
2014-06-25 17:39:59 +02:00
output_buffer < < draw_table ( SpeedCounter , INCOMING , true , sorter ) ;
2013-10-18 12:16:55 +02:00
2015-01-27 20:29:51 +01:00
output_buffer < < std : : endl ;
2014-06-22 12:57:02 +02:00
2015-01-27 20:29:51 +01:00
output_buffer < < print_channel_speed ( " Outgoing traffic " , OUTGOING ) < < std : : endl ;
2014-06-25 17:39:59 +02:00
output_buffer < < draw_table ( SpeedCounter , OUTGOING , false , sorter ) ;
2013-10-18 12:16:55 +02:00
2015-01-27 20:29:51 +01:00
output_buffer < < std : : endl ;
2013-10-18 12:16:55 +02:00
2015-01-27 20:29:51 +01:00
output_buffer < < print_channel_speed ( " Internal traffic " , INTERNAL ) < < std : : endl ;
2013-10-18 12:16:55 +02:00
2015-01-27 20:29:51 +01:00
output_buffer < < std : : endl ;
2013-10-18 12:16:55 +02:00
2015-01-27 20:29:51 +01:00
output_buffer < < print_channel_speed ( " Other traffic " , OTHER ) < < std : : endl ;
2013-10-18 12:16:55 +02:00
2015-01-27 20:29:51 +01:00
output_buffer < < std : : endl ;
2013-12-28 20:36:22 +01:00
2015-01-27 10:56:16 +01:00
if ( enable_pcap_collection ) {
output_buffer < < get_pcap_stats ( ) < < " \n " ;
}
2014-11-14 10:55:23 +01:00
// Application statistics
2014-12-16 15:20:04 +01:00
output_buffer < < " Screen updated in: \t \t " < < drawing_thread_execution_time . tv_sec < < " sec " < < drawing_thread_execution_time . tv_usec < < " microseconds \n " ;
2014-12-16 12:59:24 +01:00
output_buffer < < " Traffic calculated in: \t \t " < < speed_calculation_time . tv_sec < < " sec " < < speed_calculation_time . tv_usec < < " microseconds \n " ;
2014-12-10 16:08:12 +01:00
output_buffer < < " Total amount of not processed packets: " < < total_unparsed_packets < < " \n " ;
2014-11-27 22:16:27 +01:00
2015-01-22 16:55:59 +01:00
# ifdef PF_RING
if ( enable_data_collection_from_mirror ) {
output_buffer < < get_pf_ring_stats ( ) ;
}
2014-12-16 12:59:24 +01:00
# endif
2014-12-05 09:46:26 +01:00
2014-12-04 17:07:44 +01:00
// Print thresholds
2015-05-07 11:14:19 +02:00
output_buffer < < " \n " < < print_ban_thresholds ( ) ;
2014-12-04 17:07:44 +01:00
2014-11-15 01:10:04 +01:00
if ( ! ban_list . empty ( ) ) {
2015-01-27 20:29:51 +01:00
output_buffer < < std : : endl < < " Ban list: " < < std : : endl ;
2014-12-05 12:02:46 +01:00
output_buffer < < print_ddos_attack_details ( ) ;
2014-11-15 01:10:04 +01:00
}
2014-12-21 12:52:22 +01:00
screen_data_stats = output_buffer . str ( ) ;
2014-12-22 15:00:41 +01:00
2014-12-21 12:52:22 +01:00
// Print screen contents into file
print_screen_contents_into_file ( screen_data_stats ) ;
2014-10-23 10:51:01 +02:00
struct timeval end_calc_time ;
gettimeofday ( & end_calc_time , NULL ) ;
2014-12-16 12:59:24 +01:00
timeval_subtract ( & drawing_thread_execution_time , & end_calc_time , & start_calc_time ) ;
2013-10-18 12:16:55 +02:00
}
2014-06-22 12:57:02 +02:00
// pretty print channel speed in pps and MBit
2015-01-27 20:29:51 +01:00
std : : string print_channel_speed ( std : : string traffic_type , direction packet_direction ) {
2014-12-16 15:20:04 +01:00
uint64_t speed_in_pps = total_speed_counters [ packet_direction ] . packets ;
uint64_t speed_in_bps = total_speed_counters [ packet_direction ] . bytes ;
2014-06-25 17:32:32 +02:00
2014-07-04 10:02:44 +02:00
unsigned int number_of_tabs = 1 ;
2014-06-22 12:57:02 +02:00
// We need this for correct alignment of blocks
if ( traffic_type = = " Other traffic " ) {
number_of_tabs = 2 ;
}
std : : stringstream stream ;
stream < < traffic_type ;
2014-07-04 10:02:44 +02:00
for ( unsigned int i = 0 ; i < number_of_tabs ; i + + ) {
2014-06-22 12:57:02 +02:00
stream < < " \t " ;
}
2014-12-16 15:20:04 +01:00
uint64_t speed_in_mbps = convert_speed_to_mbps ( speed_in_bps ) ;
2014-06-22 12:57:02 +02:00
2015-01-27 20:29:51 +01:00
stream < < std : : setw ( 6 ) < < speed_in_pps < < " pps " < < std : : setw ( 6 ) < < speed_in_mbps < < " mbps " ;
2014-11-14 11:45:14 +01:00
if ( traffic_type = = " Incoming traffic " or traffic_type = = " Outgoing traffic " ) {
if ( packet_direction = = INCOMING ) {
2015-01-27 20:29:51 +01:00
stream < < " " < < std : : setw ( 6 ) < < incoming_total_flows_speed < < " flows " ;
2014-11-14 11:45:14 +01:00
} else if ( packet_direction = = OUTGOING ) {
2015-01-27 20:29:51 +01:00
stream < < " " < < std : : setw ( 6 ) < < outgoing_total_flows_speed < < " flows " ;
2014-11-14 11:45:14 +01:00
}
2015-05-10 20:42:49 +02:00
if ( graphite_enabled ) {
graphite_data_t graphite_data ;
std : : string direction_as_string ;
if ( packet_direction = = INCOMING ) {
direction_as_string = " incoming " ;
graphite_data [ graphite_prefix + direction_as_string + " flows " ] = incoming_total_flows_speed ;
} else if ( packet_direction = = OUTGOING ) {
direction_as_string = " outgoing " ;
graphite_data [ graphite_prefix + direction_as_string + " flows " ] = outgoing_total_flows_speed ;
}
graphite_data [ graphite_prefix + direction_as_string + " .pps " ] = speed_in_pps ;
graphite_data [ graphite_prefix + direction_as_string + " .mbps " ] = speed_in_mbps ;
bool graphite_put_result = store_data_to_graphite ( graphite_port , graphite_host , graphite_data ) ;
if ( ! graphite_put_result ) {
logger < < log4cpp : : Priority : : ERROR < < " Can't store data to Graphite " ;
}
}
2014-11-14 11:45:14 +01:00
}
2015-05-10 20:42:49 +02:00
2014-06-22 12:57:02 +02:00
return stream . str ( ) ;
}
2013-10-18 12:16:55 +02:00
2014-06-24 12:16:22 +02:00
void init_logging ( ) {
log4cpp : : PatternLayout * layout = new log4cpp : : PatternLayout ( ) ;
layout - > setConversionPattern ( " %d [%p] %m%n " ) ;
log4cpp : : Appender * appender = new log4cpp : : FileAppender ( " default " , log_file_path ) ;
appender - > setLayout ( layout ) ;
logger . setPriority ( log4cpp : : Priority : : INFO ) ;
logger . addAppender ( appender ) ;
logger . info ( " Logger initialized! " ) ;
}
2015-05-02 21:18:56 +02:00
// Call fork function
int do_fork ( ) {
int status = 0 ;
switch ( fork ( ) ) {
case 0 :
// It's child
break ;
case - 1 :
/* fork failed */
status = - 1 ;
break ;
default :
// We should close master process with _exit(0)
// We should not call exit() because it will destroy all global variables for programm
_exit ( 0 ) ;
}
return status ;
}
void redirect_fds ( ) {
// Close stdin, stdout and stderr
close ( 0 ) ;
close ( 1 ) ;
close ( 2 ) ;
if ( open ( " /dev/null " , O_RDWR ) ! = 0 ) {
// We can't notify anybody now
exit ( 1 ) ;
}
// Create copy of zero decriptor for 1 and 2 fd's
dup ( 0 ) ;
dup ( 0 ) ;
}
2013-10-18 12:16:55 +02:00
int main ( int argc , char * * argv ) {
2015-05-02 21:18:56 +02:00
bool daemonize = false ;
if ( argc > 1 ) {
if ( strstr ( argv [ 1 ] , " --daemonize " ) ! = NULL ) {
daemonize = true ;
}
}
// We use ideas from here https://github.com/bmc/daemonize/blob/master/daemon.c
if ( daemonize ) {
int status = 0 ;
printf ( " We will run in daemonized mode \n " ) ;
if ( ( status = do_fork ( ) ) < 0 ) {
// fork failed
status = - 1 ;
} else if ( setsid ( ) < 0 ) {
// Create new session
status = - 1 ;
} else if ( ( status = do_fork ( ) ) < 0 ) {
status = - 1 ;
} else {
// Clear inherited umask
umask ( 0 ) ;
// Chdir to root
chdir ( " / " ) ;
// close all descriptors because we are daemon!
redirect_fds ( ) ;
}
}
2015-05-02 22:53:45 +02:00
// enable core dumps
enable_core_dumps ( ) ;
init_logging ( ) ;
if ( file_exists ( pid_path ) ) {
pid_t pid_from_file = 0 ;
if ( read_pid_from_file ( pid_from_file , pid_path ) ) {
// We could read pid
if ( pid_from_file > 0 ) {
// We use signal zero for check process existence
int kill_result = kill ( pid_from_file , 0 ) ;
if ( kill_result = = 0 ) {
logger < < log4cpp : : Priority : : ERROR < < " FastNetMon is already running with pid: " < < pid_from_file ;
exit ( 1 ) ;
} else {
// Yes, we have pid with pid but it's zero
}
} else {
// pid from file is broken, we assume tool is not running
}
} else {
// We can't open file, let's assume it's broken and tool is not running
}
} else {
// no pid file
}
// If we not failed in check steps we could run toolkit
print_pid_to_file ( getpid ( ) , pid_path ) ;
2014-06-25 21:59:43 +02:00
lookup_tree = New_Patricia ( 32 ) ;
whitelist_tree = New_Patricia ( 32 ) ;
2014-06-30 09:04:06 +02:00
// nullify total counters
for ( int index = 0 ; index < 4 ; index + + ) {
total_counters [ index ] . bytes = 0 ;
2014-06-30 13:10:07 +02:00
total_counters [ index ] . packets = 0 ;
total_speed_counters [ index ] . bytes = 0 ;
total_speed_counters [ index ] . packets = 0 ;
2014-06-30 09:04:06 +02:00
}
2014-12-05 12:02:46 +01:00
/* Create folder for attack details */
if ( ! folder_exists ( attack_details_folder ) ) {
int mkdir_result = mkdir ( attack_details_folder . c_str ( ) , S_IRWXU ) ;
if ( mkdir_result ! = 0 ) {
2014-12-19 09:43:22 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Can't create folder for attack details: " < < attack_details_folder ;
2014-12-05 12:02:46 +01:00
exit ( 1 ) ;
}
}
2014-06-20 17:18:27 +02:00
if ( getenv ( " DUMP_ALL_PACKETS " ) ! = NULL ) {
DEBUG_DUMP_ALL_PACKETS = true ;
}
2014-11-13 09:50:17 +01:00
2014-11-13 16:01:15 +01:00
if ( sizeof ( packed_conntrack_hash ) ! = sizeof ( uint64_t ) or sizeof ( packed_conntrack_hash ) ! = 8 ) {
2014-12-05 12:02:46 +01:00
logger < < log4cpp : : Priority : : INFO < < " Assertion about size of packed_conntrack_hash, it's " < < sizeof ( packed_conntrack_hash ) < < " instead 8 " ;
2014-11-13 16:01:15 +01:00
exit ( 1 ) ;
}
2014-06-20 17:18:27 +02:00
2014-06-24 12:16:22 +02:00
logger < < log4cpp : : Priority : : INFO < < " Read configuration file " ;
2014-06-09 14:47:11 +02:00
2014-11-23 20:17:56 +01:00
bool load_config_result = load_configuration_file ( ) ;
if ( ! load_config_result ) {
2015-02-01 14:02:30 +01:00
fprintf ( stderr , " Can't open config file %s, please create it! \n " , global_config_path . c_str ( ) ) ;
2014-11-23 20:19:52 +01:00
exit ( 1 ) ;
2014-11-23 20:17:56 +01:00
}
2014-06-09 13:15:22 +02:00
2014-06-28 12:12:56 +02:00
load_our_networks_list ( ) ;
2014-11-15 01:10:04 +01:00
// Setup CTRL+C handler
2015-04-27 16:52:16 +02:00
signal ( SIGINT , interruption_signal_handler ) ;
/* Without this SIGPIPE error could shutdown toolkit on call of exec_with_stdin_params */
signal ( SIGPIPE , sigpipe_handler_for_popen ) ;
2014-06-28 12:12:56 +02:00
2013-12-28 20:11:20 +01:00
# ifdef GEOIP
2014-11-15 01:10:04 +01:00
// Init GeoIP
2013-12-28 20:11:20 +01:00
if ( ! geoip_init ( ) ) {
2014-12-19 09:43:22 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Can't load geoip tables " ;
2013-12-28 20:11:20 +01:00
exit ( 1 ) ;
}
# endif
2014-11-15 01:10:04 +01:00
// Init previous run date
2014-06-30 15:33:14 +02:00
time ( & last_call_of_traffic_recalculation ) ;
2014-12-19 16:50:00 +01:00
// Run screen draw thread
2014-06-29 13:28:56 +02:00
boost : : thread calc_thread ( calculation_thread ) ;
2014-10-23 10:51:01 +02:00
2014-06-26 11:30:38 +02:00
// start thread for recalculating speed in realtime
2014-06-29 13:28:56 +02:00
boost : : thread recalculate_speed_thread ( recalculate_speed_thread_handler ) ;
2014-12-19 16:50:00 +01:00
2014-11-15 01:10:04 +01:00
// Run banlist cleaner thread
2014-10-21 09:20:18 +02:00
boost : : thread cleanup_ban_list_thread ( cleanup_ban_list ) ;
2013-10-21 16:47:31 +02:00
2015-02-03 20:55:44 +01:00
# ifdef PF_RING
2014-12-02 13:43:34 +01:00
if ( enable_data_collection_from_mirror ) {
2015-05-12 11:07:45 +02:00
packet_capture_plugin_thread_group . add_thread ( new boost : : thread ( start_pfring_collection , process_packet ) ) ;
2014-12-02 13:43:34 +01:00
}
2015-02-03 20:55:44 +01:00
# endif
2015-01-22 16:51:04 +01:00
2015-03-10 15:17:52 +01:00
// netmap processing
if ( enable_netmap_collection ) {
2015-05-12 11:07:45 +02:00
packet_capture_plugin_thread_group . add_thread ( new boost : : thread ( start_netmap_collection , process_packet ) ) ;
2015-03-10 15:17:52 +01:00
}
2014-12-02 13:43:34 +01:00
if ( enable_sflow_collection ) {
2015-05-12 11:07:45 +02:00
packet_capture_plugin_thread_group . add_thread ( new boost : : thread ( start_sflow_collection , process_packet ) ) ;
2014-12-02 13:43:34 +01:00
}
2014-06-30 15:33:14 +02:00
2015-01-23 17:49:37 +01:00
if ( enable_netflow_collection ) {
2015-05-12 11:07:45 +02:00
packet_capture_plugin_thread_group . add_thread ( new boost : : thread ( start_netflow_collection , process_packet ) ) ;
2015-01-23 17:49:37 +01:00
}
2015-01-27 10:56:16 +01:00
if ( enable_pcap_collection ) {
2015-05-12 11:07:45 +02:00
packet_capture_plugin_thread_group . add_thread ( new boost : : thread ( start_pcap_collection , process_packet ) ) ;
2014-12-02 13:43:34 +01:00
}
2014-12-01 22:08:38 +01:00
2015-05-12 11:07:45 +02:00
// Wait for all threads in capture thread group
packet_capture_plugin_thread_group . join_all ( ) ;
2015-03-10 15:17:52 +01:00
2014-06-30 15:33:14 +02:00
recalculate_speed_thread . join ( ) ;
calc_thread . join ( ) ;
free_up_all_resources ( ) ;
return 0 ;
}
void free_up_all_resources ( ) {
2013-12-28 20:11:20 +01:00
# ifdef GEOIP
// Free up geoip handle
GeoIP_delete ( geo_ip ) ;
# endif
2014-06-25 21:59:43 +02:00
Destroy_Patricia ( lookup_tree , ( void_fn_t ) 0 ) ;
Destroy_Patricia ( whitelist_tree , ( void_fn_t ) 0 ) ;
2014-10-22 16:36:44 +02:00
}
2013-11-15 15:35:44 +01:00
// For correct programm shutdown by CTRL+C
2015-04-27 16:52:16 +02:00
void interruption_signal_handler ( int signal_number ) {
2013-10-18 12:16:55 +02:00
2015-01-27 10:56:16 +01:00
if ( enable_pcap_collection ) {
stop_pcap_collection ( ) ;
}
2013-10-18 12:16:55 +02:00
2014-03-11 20:48:15 +01:00
# ifdef PF_RING
2015-02-03 20:55:44 +01:00
stop_pfring_collection ( ) ;
2014-03-11 20:48:15 +01:00
# endif
2015-05-12 11:07:45 +02:00
// packet_capture_plugin_thread_group.interrupt_all();
// Wait some time for threads finishing
// sleep 3;
2013-10-18 12:16:55 +02:00
exit ( 1 ) ;
}
2014-06-26 09:59:03 +02:00
/* Get traffic type: check it belongs to our IPs */
2014-10-23 16:08:58 +02:00
direction get_packet_direction ( uint32_t src_ip , uint32_t dst_ip , unsigned long & subnet ) {
2014-06-26 09:59:03 +02:00
direction packet_direction ;
bool our_ip_is_destination = false ;
bool our_ip_is_source = false ;
prefix_t prefix_for_check_adreess ;
prefix_for_check_adreess . family = AF_INET ;
prefix_for_check_adreess . bitlen = 32 ;
2014-10-23 16:08:58 +02:00
patricia_node_t * found_patrica_node = NULL ;
2014-11-16 16:54:21 +01:00
prefix_for_check_adreess . add . sin . s_addr = dst_ip ;
2014-10-23 16:08:58 +02:00
unsigned long destination_subnet = 0 ;
2014-12-19 13:48:58 +01:00
found_patrica_node = patricia_search_best2 ( lookup_tree , & prefix_for_check_adreess , 1 ) ;
2014-11-28 09:37:52 +01:00
if ( found_patrica_node ) {
2014-06-26 09:59:03 +02:00
our_ip_is_destination = true ;
2014-10-23 16:08:58 +02:00
destination_subnet = found_patrica_node - > prefix - > add . sin . s_addr ;
2014-06-26 09:59:03 +02:00
}
2014-11-16 16:54:21 +01:00
found_patrica_node = NULL ;
2014-06-26 09:59:03 +02:00
prefix_for_check_adreess . add . sin . s_addr = src_ip ;
2014-10-23 16:08:58 +02:00
unsigned long source_subnet = 0 ;
2014-12-19 13:48:58 +01:00
found_patrica_node = patricia_search_best2 ( lookup_tree , & prefix_for_check_adreess , 1 ) ;
2014-11-28 09:37:52 +01:00
if ( found_patrica_node ) {
2014-06-26 09:59:03 +02:00
our_ip_is_source = true ;
2014-10-23 16:08:58 +02:00
source_subnet = found_patrica_node - > prefix - > add . sin . s_addr ;
2014-11-16 16:54:21 +01:00
}
2014-06-26 09:59:03 +02:00
2014-10-23 16:08:58 +02:00
subnet = 0 ;
2014-06-26 09:59:03 +02:00
if ( our_ip_is_source & & our_ip_is_destination ) {
packet_direction = INTERNAL ;
} else if ( our_ip_is_source ) {
2014-10-23 16:08:58 +02:00
subnet = source_subnet ;
2014-06-26 09:59:03 +02:00
packet_direction = OUTGOING ;
} else if ( our_ip_is_destination ) {
2014-10-23 16:08:58 +02:00
subnet = destination_subnet ;
2014-06-26 09:59:03 +02:00
packet_direction = INCOMING ;
} else {
packet_direction = OTHER ;
}
return packet_direction ;
}
2014-06-26 11:30:38 +02:00
2014-12-05 16:48:12 +01:00
unsigned int detect_attack_protocol ( map_element & speed_element , direction attack_direction ) {
if ( attack_direction = = INCOMING ) {
return get_max_used_protocol ( speed_element . tcp_in_packets , speed_element . udp_in_packets , speed_element . icmp_in_packets ) ;
} else {
// OUTGOING
return get_max_used_protocol ( speed_element . tcp_out_packets , speed_element . udp_out_packets , speed_element . icmp_out_packets ) ;
}
}
2015-01-09 23:43:23 +01:00
# define my_max_on_defines(a, b) (a > b ? a : b)
2014-12-16 15:20:04 +01:00
unsigned int get_max_used_protocol ( uint64_t tcp , uint64_t udp , uint64_t icmp ) {
2015-01-09 23:43:23 +01:00
unsigned int max = my_max_on_defines ( my_max_on_defines ( udp , tcp ) , icmp ) ;
2014-12-05 16:48:12 +01:00
if ( max = = tcp ) {
return IPPROTO_TCP ;
} else if ( max = = udp ) {
return IPPROTO_UDP ;
} else if ( max = = icmp ) {
return IPPROTO_ICMP ;
}
return 0 ;
}
2015-04-26 14:35:01 +02:00
void exabgp_ban_manage ( std : : string action , std : : string ip_as_string ) {
/* Buffer for BGP message */
char bgp_message [ 256 ] ;
std : : string ip_as_string_with_mask = ip_as_string + " /32 " ;
int exabgp_pipe = open ( exabgp_command_pipe . c_str ( ) , O_WRONLY ) ;
if ( exabgp_pipe < = 0 ) {
2015-05-11 21:56:18 +02:00
logger < < log4cpp : : Priority : : ERROR < < " Can't open ExaBGP pipe " < < exabgp_command_pipe < < " Ban is not executed " ;
2015-04-26 14:35:01 +02:00
return ;
}
if ( action = = " ban " ) {
sprintf ( bgp_message , " announce route %s next-hop %s community %s \n " ,
ip_as_string_with_mask . c_str ( ) , exabgp_next_hop . c_str ( ) , exabgp_community . c_str ( ) ) ;
} else {
sprintf ( bgp_message , " withdraw route %s \n " , ip_as_string_with_mask . c_str ( ) ) ;
}
int wrote_bytes = write ( exabgp_pipe , bgp_message , strlen ( bgp_message ) ) ;
if ( wrote_bytes ! = strlen ( bgp_message ) ) {
logger < < log4cpp : : Priority : : ERROR < < " Can't write message to ExaBGP pipe " ;
}
close ( exabgp_pipe ) ;
}
2015-03-13 17:24:57 +01:00
void execute_ip_ban ( uint32_t client_ip , map_element speed_element , map_element average_speed_element , std : : string flow_attack_details ) {
2014-12-05 16:01:17 +01:00
struct attack_details current_attack ;
2014-12-16 15:20:04 +01:00
uint64_t pps = 0 ;
2014-07-04 09:30:57 +02:00
2015-03-13 17:24:57 +01:00
uint64_t in_pps = average_speed_element . in_packets ;
uint64_t out_pps = average_speed_element . out_packets ;
uint64_t in_bps = average_speed_element . in_bytes ;
uint64_t out_bps = average_speed_element . out_bytes ;
uint64_t in_flows = average_speed_element . in_flows ;
uint64_t out_flows = average_speed_element . out_flows ;
2014-12-05 16:01:17 +01:00
direction data_direction ;
2014-11-13 09:50:17 +01:00
if ( ! we_do_real_ban ) {
logger < < log4cpp : : Priority : : INFO < < " We do not ban: " < < convert_ip_as_uint_to_string ( client_ip ) < < " because ban disabled completely " ;
return ;
}
2014-12-05 16:13:53 +01:00
// Detect attack direction with simple heuristic
2014-12-05 20:44:24 +01:00
if ( abs ( int ( ( int ) in_pps - ( int ) out_pps ) ) < 1000 ) {
2014-12-05 16:13:53 +01:00
// If difference between pps speed is so small we should do additional investigation using bandwidth speed
if ( in_bps > out_bps ) {
data_direction = INCOMING ;
pps = in_pps ;
} else {
data_direction = OUTGOING ;
pps = out_pps ;
}
2014-07-04 09:30:57 +02:00
} else {
2014-12-05 16:13:53 +01:00
if ( in_pps > out_pps ) {
data_direction = INCOMING ;
pps = in_pps ;
} else {
data_direction = OUTGOING ;
pps = out_pps ;
}
2014-07-04 09:30:57 +02:00
}
2014-12-05 16:48:12 +01:00
current_attack . attack_protocol = detect_attack_protocol ( speed_element , data_direction ) ;
2014-07-04 09:30:57 +02:00
if ( ban_list . count ( client_ip ) > 0 ) {
if ( ban_list [ client_ip ] . attack_direction ! = data_direction ) {
logger < < log4cpp : : Priority : : INFO < < " We expected very strange situation: attack direction for "
< < convert_ip_as_uint_to_string ( client_ip ) < < " was changed " ;
return ;
}
// update attack power
if ( pps > ban_list [ client_ip ] . max_attack_power ) {
ban_list [ client_ip ] . max_attack_power = pps ;
}
2014-06-30 16:37:27 +02:00
return ;
}
prefix_t prefix_for_check_adreess ;
prefix_for_check_adreess . add . sin . s_addr = client_ip ;
prefix_for_check_adreess . family = AF_INET ;
prefix_for_check_adreess . bitlen = 32 ;
2014-12-19 13:48:58 +01:00
bool in_white_list = ( patricia_search_best2 ( whitelist_tree , & prefix_for_check_adreess , 1 ) ! = NULL ) ;
2014-06-30 16:37:27 +02:00
if ( in_white_list ) {
return ;
}
2015-01-27 20:29:51 +01:00
std : : string data_direction_as_string = get_direction_name ( data_direction ) ;
2014-06-28 13:27:57 +02:00
2014-06-30 16:37:27 +02:00
logger . info ( " We run execute_ip_ban code with following params in_pps: %d out_pps: %d in_bps: %d out_bps: %d and we decide it's %s attack " ,
in_pps , out_pps , in_bps , out_bps , data_direction_as_string . c_str ( ) ) ;
2014-06-26 11:30:38 +02:00
2015-01-27 20:29:51 +01:00
std : : string client_ip_as_string = convert_ip_as_uint_to_string ( client_ip ) ;
std : : string pps_as_string = convert_int_to_string ( pps ) ;
2014-06-26 11:30:38 +02:00
2014-11-15 01:10:04 +01:00
// Store ban time
2014-10-21 09:20:18 +02:00
time ( & current_attack . ban_timestamp ) ;
// set ban time in seconds
current_attack . ban_time = standard_ban_time ;
2014-10-20 17:42:14 +02:00
2014-11-15 01:10:04 +01:00
// Pass main information about attack
2014-07-04 09:30:57 +02:00
current_attack . attack_direction = data_direction ;
current_attack . attack_power = pps ;
2014-07-04 12:04:19 +02:00
current_attack . max_attack_power = pps ;
2014-07-04 09:30:57 +02:00
current_attack . in_packets = in_pps ;
current_attack . out_packets = out_pps ;
current_attack . in_bytes = in_bps ;
current_attack . out_bytes = out_bps ;
2014-12-05 15:34:59 +01:00
// pass flow information
current_attack . in_flows = in_flows ;
current_attack . out_flows = out_flows ;
2015-05-07 15:09:43 +02:00
current_attack . fragmented_in_packets = speed_element . fragmented_in_packets ;
2014-12-05 15:34:59 +01:00
current_attack . tcp_in_packets = speed_element . tcp_in_packets ;
2015-05-07 11:51:48 +02:00
current_attack . tcp_syn_in_packets = speed_element . tcp_syn_in_packets ;
2014-12-05 15:34:59 +01:00
current_attack . udp_in_packets = speed_element . udp_in_packets ;
current_attack . icmp_in_packets = speed_element . icmp_in_packets ;
2015-05-07 15:09:43 +02:00
current_attack . fragmented_out_packets = speed_element . fragmented_out_packets ;
2014-12-05 15:34:59 +01:00
current_attack . tcp_out_packets = speed_element . tcp_out_packets ;
2015-05-07 11:51:48 +02:00
current_attack . tcp_syn_out_packets = speed_element . tcp_syn_out_packets ;
2014-12-05 15:34:59 +01:00
current_attack . udp_out_packets = speed_element . udp_out_packets ;
current_attack . icmp_out_packets = speed_element . icmp_out_packets ;
2015-05-07 15:09:43 +02:00
current_attack . fragmented_out_bytes = speed_element . fragmented_out_bytes ;
2014-12-05 15:34:59 +01:00
current_attack . tcp_out_bytes = speed_element . tcp_out_bytes ;
2015-05-07 11:51:48 +02:00
current_attack . tcp_syn_out_bytes = speed_element . tcp_syn_out_bytes ;
2014-12-05 15:34:59 +01:00
current_attack . udp_out_bytes = speed_element . udp_out_bytes ;
current_attack . icmp_out_bytes = speed_element . icmp_out_bytes ;
2015-05-07 15:09:43 +02:00
current_attack . fragmented_in_bytes = speed_element . fragmented_in_bytes ;
2014-12-05 15:34:59 +01:00
current_attack . tcp_in_bytes = speed_element . tcp_in_bytes ;
2015-05-07 11:51:48 +02:00
current_attack . tcp_syn_in_bytes = speed_element . tcp_syn_in_bytes ;
2014-12-05 15:34:59 +01:00
current_attack . udp_in_bytes = speed_element . udp_in_bytes ;
current_attack . icmp_in_bytes = speed_element . icmp_in_bytes ;
2014-12-04 16:48:11 +01:00
// Add average counters
map_element * current_average_speed_element = & SpeedCounterAverage [ client_ip ] ;
current_attack . average_in_packets = current_average_speed_element - > in_packets ;
current_attack . average_in_bytes = current_average_speed_element - > in_bytes ;
current_attack . average_in_flows = current_average_speed_element - > in_flows ;
current_attack . average_out_packets = current_average_speed_element - > out_packets ;
current_attack . average_out_bytes = current_average_speed_element - > out_bytes ;
current_attack . average_out_flows = current_average_speed_element - > out_flows ;
2014-10-21 09:20:18 +02:00
ban_list_mutex . lock ( ) ;
2014-07-04 09:30:57 +02:00
ban_list [ client_ip ] = current_attack ;
2014-10-21 09:20:18 +02:00
ban_list_mutex . unlock ( ) ;
2014-12-19 16:50:00 +01:00
ban_list_details_mutex . lock ( ) ;
2015-01-27 20:29:51 +01:00
ban_list_details [ client_ip ] = std : : vector < simple_packet > ( ) ;
2014-12-19 16:50:00 +01:00
ban_list_details_mutex . unlock ( ) ;
2014-06-30 16:37:27 +02:00
logger < < log4cpp : : Priority : : INFO < < " Attack with direction: " < < data_direction_as_string
< < " IP: " < < client_ip_as_string < < " Power: " < < pps_as_string ;
2014-10-16 14:39:06 +02:00
2014-12-03 15:56:08 +01:00
# ifdef HWFILTER_LOCKING
2014-12-05 15:34:59 +01:00
logger < < log4cpp : : Priority : : INFO < < " We will block traffic to/from this IP with hardware filters " ;
2014-12-03 15:56:08 +01:00
block_all_traffic_with_82599_hardware_filtering ( client_ip_as_string ) ;
# endif
2015-04-13 11:30:20 +02:00
std : : string basic_attack_information = get_attack_description ( client_ip , current_attack ) ;
std : : string full_attack_description = basic_attack_information + flow_attack_details ;
2014-12-05 12:02:46 +01:00
print_attack_details_to_file ( full_attack_description , client_ip_as_string , current_attack ) ;
2014-12-03 15:56:08 +01:00
if ( file_exists ( notify_script_path ) ) {
2015-01-27 20:29:51 +01:00
std : : string script_call_params = notify_script_path + " " + client_ip_as_string + " " + data_direction_as_string + " " + pps_as_string + " attack_details " ;
2014-12-03 15:56:08 +01:00
logger < < log4cpp : : Priority : : INFO < < " Call script for ban client: " < < client_ip_as_string ;
// We should execute external script in separate thread because any lag in this code will be very distructive
2014-12-05 12:02:46 +01:00
boost : : thread exec_thread ( exec_with_stdin_params , script_call_params , full_attack_description ) ;
2014-12-03 15:56:08 +01:00
exec_thread . detach ( ) ;
logger < < log4cpp : : Priority : : INFO < < " Script for ban client is finished: " < < client_ip_as_string ;
2015-04-13 11:30:20 +02:00
}
2015-05-11 21:56:18 +02:00
if ( exabgp_enabled ) {
logger < < log4cpp : : Priority : : INFO < < " Call ExaBGP for ban client started: " < < client_ip_as_string ;
boost : : thread exabgp_thread ( exabgp_ban_manage , " ban " , client_ip_as_string ) ;
exabgp_thread . detach ( ) ;
logger < < log4cpp : : Priority : : INFO < < " Call to ExaBGP for ban client is finished: " < < client_ip_as_string ;
}
2015-04-13 11:30:20 +02:00
# ifdef REDIS
if ( redis_enabled ) {
std : : string redis_key_name = client_ip_as_string + " _information " ;
logger < < log4cpp : : Priority : : INFO < < " Start data save in redis in key: " < < redis_key_name ;
boost : : thread redis_store_thread ( store_data_in_redis , redis_key_name , basic_attack_information ) ;
redis_store_thread . detach ( ) ;
logger < < log4cpp : : Priority : : INFO < < " Finish data save in redis in key: " < < redis_key_name ;
}
// If we have flow dump put in redis too
if ( redis_enabled & & ! flow_attack_details . empty ( ) ) {
std : : string redis_key_name = client_ip_as_string + " _flow_dump " ;
logger < < log4cpp : : Priority : : INFO < < " Start data save in redis in key: " < < redis_key_name ;
boost : : thread redis_store_thread ( store_data_in_redis , redis_key_name , flow_attack_details ) ;
redis_store_thread . detach ( ) ;
logger < < log4cpp : : Priority : : INFO < < " Finish data save in redis in key: " < < redis_key_name ;
}
# endif
2014-12-03 15:56:08 +01:00
}
# ifdef HWFILTER_LOCKING
2015-01-27 20:29:51 +01:00
void block_all_traffic_with_82599_hardware_filtering ( std : : string client_ip_as_string ) {
2014-10-16 14:39:06 +02:00
/* 6 - tcp, 17 - udp, 0 - other (non tcp and non udp) */
2015-01-27 20:29:51 +01:00
std : : vector < int > banned_protocols ;
2014-10-16 14:39:06 +02:00
banned_protocols . push_back ( 17 ) ;
banned_protocols . push_back ( 6 ) ;
banned_protocols . push_back ( 0 ) ;
int rule_number = 10 ;
// Iterate over incoming and outgoing direction
2014-10-16 15:25:46 +02:00
for ( int rule_direction = 0 ; rule_direction < 2 ; rule_direction + + ) {
2014-10-16 14:39:06 +02:00
for ( std : : vector < int > : : iterator banned_protocol = banned_protocols . begin ( ) ;
banned_protocol ! = banned_protocols . end ( ) ; + + banned_protocol ) {
/* On 82599 NIC we can ban traffic using hardware filtering rules */
// Difference between fie tuple and perfect filters:
// http://www.ntop.org/products/pf_ring/hardware-packet-filtering/
hw_filtering_rule rule ;
intel_82599_five_tuple_filter_hw_rule * ft_rule ;
ft_rule = & rule . rule_family . five_tuple_rule ;
memset ( & rule , 0 , sizeof ( rule ) ) ;
rule . rule_family_type = intel_82599_five_tuple_rule ;
rule . rule_id = rule_number + + ;
ft_rule - > queue_id = - 1 ; // drop traffic
ft_rule - > proto = * banned_protocol ;
2015-01-27 20:29:51 +01:00
std : : string hw_filter_rule_direction = " " ;
2014-10-16 14:39:06 +02:00
if ( rule_direction = = 0 ) {
hw_filter_rule_direction = " outgoing " ;
ft_rule - > s_addr = ntohl ( inet_addr ( client_ip_as_string . c_str ( ) ) ) ;
} else {
hw_filter_rule_direction = " incoming " ;
ft_rule - > d_addr = ntohl ( inet_addr ( client_ip_as_string . c_str ( ) ) ) ;
}
if ( pfring_add_hw_rule ( pf_ring_descr , & rule ) ! = 0 ) {
2014-12-19 09:43:22 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Can't add hardware filtering rule for protocol: " < < * banned_protocol < < " in direction: " < < hw_filter_rule_direction ;
2014-10-16 14:39:06 +02:00
}
rule_number + + ;
}
}
2014-12-03 15:56:08 +01:00
}
2014-10-16 14:39:06 +02:00
# endif
2014-10-20 17:42:14 +02:00
/* Thread for cleaning up ban list */
void cleanup_ban_list ( ) {
2014-11-15 01:10:04 +01:00
// Every X seconds we will run ban list cleaner thread
2014-10-20 17:42:14 +02:00
int iteration_sleep_time = 600 ;
2015-03-09 15:35:06 +01:00
// If we use very small ban time we should call ban_cleanup thread more often
if ( iteration_sleep_time > standard_ban_time ) {
iteration_sleep_time = int ( standard_ban_time / 2 ) ;
}
2014-10-21 15:25:13 +02:00
logger < < log4cpp : : Priority : : INFO < < " Run banlist cleanup thread " ;
2014-10-20 17:42:14 +02:00
while ( true ) {
// Sleep for ten minutes
2014-10-21 16:44:20 +02:00
boost : : this_thread : : sleep ( boost : : posix_time : : seconds ( iteration_sleep_time ) ) ;
2014-10-20 17:42:14 +02:00
time_t current_time ;
time ( & current_time ) ;
2015-01-27 20:29:51 +01:00
std : : map < uint32_t , banlist_item > : : iterator itr = ban_list . begin ( ) ;
2014-10-22 01:49:10 +02:00
while ( itr ! = ban_list . end ( ) ) {
uint32_t client_ip = ( * itr ) . first ;
2014-10-20 17:42:14 +02:00
2014-10-22 01:49:10 +02:00
double time_difference = difftime ( current_time , ( ( * itr ) . second ) . ban_timestamp ) ;
int ban_time = ( ( * itr ) . second ) . ban_time ;
2014-10-20 17:42:14 +02:00
2015-04-27 18:01:13 +02:00
// Zero value for ban_time means "no unban feature"
if ( ban_time ! = 0 & & time_difference > ban_time ) {
2014-11-15 01:10:04 +01:00
// Cleanup all data related with this attack
2015-01-27 20:29:51 +01:00
std : : string data_direction_as_string = get_direction_name ( ( * itr ) . second . attack_direction ) ;
std : : string client_ip_as_string = convert_ip_as_uint_to_string ( client_ip ) ;
std : : string pps_as_string = convert_int_to_string ( ( * itr ) . second . attack_power ) ;
2014-10-21 09:20:18 +02:00
logger < < log4cpp : : Priority : : INFO < < " We will unban banned IP: " < < client_ip_as_string < <
" because it ban time " < < ban_time < < " seconds is ended " ;
ban_list_mutex . lock ( ) ;
2015-01-27 20:29:51 +01:00
std : : map < uint32_t , banlist_item > : : iterator itr_to_erase = itr ;
2015-03-15 18:58:40 +01:00
+ + itr ;
2014-10-22 01:49:10 +02:00
ban_list . erase ( itr_to_erase ) ;
2014-10-21 09:20:18 +02:00
ban_list_mutex . unlock ( ) ;
if ( file_exists ( notify_script_path ) ) {
2015-01-27 20:29:51 +01:00
std : : string script_call_params = notify_script_path + " " + client_ip_as_string + " " +
2014-10-21 09:20:18 +02:00
data_direction_as_string + " " + pps_as_string + " unban " ;
logger < < log4cpp : : Priority : : INFO < < " Call script for unban client: " < < client_ip_as_string ;
// We should execute external script in separate thread because any lag in this code will be very distructive
boost : : thread exec_thread ( exec , script_call_params ) ;
exec_thread . detach ( ) ;
logger < < log4cpp : : Priority : : INFO < < " Script for unban client is finished: " < < client_ip_as_string ;
}
2015-04-26 14:46:39 +02:00
if ( exabgp_enabled ) {
logger < < log4cpp : : Priority : : INFO < < " Call ExaBGP for unban client started: " < < client_ip_as_string ;
boost : : thread exabgp_thread ( exabgp_ban_manage , " unban " , client_ip_as_string ) ;
exabgp_thread . detach ( ) ;
logger < < log4cpp : : Priority : : INFO < < " Call to ExaBGP for unban client is finished: " < < client_ip_as_string ;
}
2014-10-22 01:49:10 +02:00
} else {
2015-03-15 18:58:40 +01:00
+ + itr ;
2014-10-20 17:42:14 +02:00
}
}
}
}
2015-01-27 20:29:51 +01:00
std : : string print_ddos_attack_details ( ) {
std : : stringstream output_buffer ;
2014-06-28 23:04:31 +02:00
2015-01-27 20:29:51 +01:00
for ( std : : map < uint32_t , banlist_item > : : iterator ii = ban_list . begin ( ) ; ii ! = ban_list . end ( ) ; + + ii ) {
2014-07-04 12:04:19 +02:00
uint32_t client_ip = ( * ii ) . first ;
2015-01-27 20:29:51 +01:00
std : : string client_ip_as_string = convert_ip_as_uint_to_string ( client_ip ) ;
std : : string max_pps_as_string = convert_int_to_string ( ( ( * ii ) . second ) . max_attack_power ) ;
std : : string attack_direction = get_direction_name ( ( ( * ii ) . second ) . attack_direction ) ;
2014-06-28 23:04:31 +02:00
2015-01-27 20:29:51 +01:00
output_buffer < < client_ip_as_string < < " / " < < max_pps_as_string < < " pps " < < attack_direction < < " at " < < print_time_t_in_fastnetmon_format ( ii - > second . ban_timestamp ) < < std : : endl ;
2014-06-28 23:04:31 +02:00
2014-07-04 12:04:19 +02:00
send_attack_details ( client_ip , ( * ii ) . second ) ;
}
2014-06-28 23:04:31 +02:00
2014-07-04 12:04:19 +02:00
return output_buffer . str ( ) ;
}
2014-06-28 23:04:31 +02:00
2015-01-27 20:29:51 +01:00
std : : string get_attack_description ( uint32_t client_ip , attack_details & current_attack ) {
std : : stringstream attack_description ;
2014-06-28 23:04:31 +02:00
2015-05-07 15:51:10 +02:00
attack_type_t attack_type = detect_attack_type ( current_attack ) ;
std : : string printable_attack_type = get_printable_attack_name ( attack_type ) ;
2014-07-04 12:04:19 +02:00
attack_description
< < " IP: " < < convert_ip_as_uint_to_string ( client_ip ) < < " \n "
2015-05-07 15:51:10 +02:00
< < " Attack type: " < < printable_attack_type < < " \n "
2014-12-05 15:34:59 +01:00
< < " Initial attack power: " < < current_attack . attack_power < < " packets per second \n "
< < " Peak attack power: " < < current_attack . max_attack_power < < " packets per second \n "
< < " Attack direction: " < < get_direction_name ( current_attack . attack_direction ) < < " \n "
2014-12-05 16:48:12 +01:00
< < " Attack protocol: " < < get_printable_protocol_name ( current_attack . attack_protocol ) < < " \n "
2014-12-05 15:34:59 +01:00
< < " Total incoming traffic: " < < convert_speed_to_mbps ( current_attack . in_bytes ) < < " mbps \n "
< < " Total outgoing traffic: " < < convert_speed_to_mbps ( current_attack . out_bytes ) < < " mbps \n "
< < " Total incoming pps: " < < current_attack . in_packets < < " packets per second \n "
< < " Total outgoing pps: " < < current_attack . out_packets < < " packets per second \n "
< < " Total incoming flows: " < < current_attack . in_flows < < " flows per second \n "
< < " Total outgoing flows: " < < current_attack . out_flows < < " flows per second \n " ;
2014-12-04 16:48:11 +01:00
// Add average counters
attack_description
< < " Average incoming traffic: " < < convert_speed_to_mbps ( current_attack . average_in_bytes ) < < " mbps \n "
< < " Average outgoing traffic: " < < convert_speed_to_mbps ( current_attack . average_out_bytes ) < < " mbps \n "
< < " Average incoming pps: " < < current_attack . average_in_packets < < " packets per second \n "
< < " Average outgoing pps: " < < current_attack . average_out_packets < < " packets per second \n "
< < " Average incoming flows: " < < current_attack . average_in_flows < < " flows per second \n "
< < " Average outgoing flows: " < < current_attack . average_out_flows < < " flows per second \n " ;
2015-05-07 11:14:19 +02:00
double average_packet_size_for_incoming_traffic = 0 ;
double average_packet_size_for_outgoing_traffic = 0 ;
if ( current_attack . average_in_packets > 0 ) {
average_packet_size_for_incoming_traffic = ( double ) current_attack . average_in_bytes /
( double ) current_attack . average_in_packets ;
}
if ( current_attack . average_out_packets > 0 ) {
average_packet_size_for_outgoing_traffic = ( double ) current_attack . average_out_bytes /
( double ) current_attack . average_out_packets ;
}
2015-05-07 10:22:37 +02:00
2014-12-05 15:34:59 +01:00
attack_description
2015-05-07 15:09:43 +02:00
< < " Incoming ip fragmented traffic: " < < convert_speed_to_mbps ( current_attack . fragmented_in_bytes ) < < " mbps \n "
< < " Outgoing ip fragmented traffic: " < < convert_speed_to_mbps ( current_attack . fragmented_out_bytes ) < < " mbps \n "
< < " Incoming ip fragmented pps: " < < current_attack . fragmented_in_packets < < " packets per second \n "
< < " Outgoing ip fragmented pps: " < < current_attack . fragmented_out_packets < < " packets per second \n "
2014-12-05 15:34:59 +01:00
< < " Incoming tcp traffic: " < < convert_speed_to_mbps ( current_attack . tcp_in_bytes ) < < " mbps \n "
< < " Outgoing tcp traffic: " < < convert_speed_to_mbps ( current_attack . tcp_out_bytes ) < < " mbps \n "
< < " Incoming tcp pps: " < < current_attack . tcp_in_packets < < " packets per second \n "
< < " Outgoing tcp pps: " < < current_attack . tcp_out_packets < < " packets per second \n "
2015-05-07 11:51:48 +02:00
< < " Incoming syn tcp traffic: " < < convert_speed_to_mbps ( current_attack . tcp_syn_in_bytes ) < < " mbps \n "
< < " Outgoing syn tcp traffic: " < < convert_speed_to_mbps ( current_attack . tcp_syn_out_bytes ) < < " mbps \n "
< < " Incoming syn tcp pps: " < < current_attack . tcp_syn_in_packets < < " packets per second \n "
< < " Outgoing syn tcp pps: " < < current_attack . tcp_syn_out_packets < < " packets per second \n "
2014-12-05 15:34:59 +01:00
< < " Incoming udp traffic: " < < convert_speed_to_mbps ( current_attack . udp_in_bytes ) < < " mbps \n "
< < " Outgoing udp traffic: " < < convert_speed_to_mbps ( current_attack . udp_out_bytes ) < < " mbps \n "
< < " Incoming udp pps: " < < current_attack . udp_in_packets < < " packets per second \n "
< < " Outgoing udp pps: " < < current_attack . udp_out_packets < < " packets per second \n "
2015-05-07 15:09:43 +02:00
2014-12-05 15:34:59 +01:00
< < " Incoming icmp traffic: " < < convert_speed_to_mbps ( current_attack . icmp_in_bytes ) < < " mbps \n "
< < " Outgoing icmp traffic: " < < convert_speed_to_mbps ( current_attack . icmp_out_bytes ) < < " mbps \n "
< < " Incoming icmp pps: " < < current_attack . icmp_in_packets < < " packets per second \n "
< < " Outgoing icmp pps: " < < current_attack . icmp_out_packets < < " packets per second \n " ;
2015-05-07 10:22:37 +02:00
// We do not need very accurate size
attack_description . precision ( 1 ) ;
attack_description
2015-05-07 11:14:19 +02:00
< < " Average packet size for incoming traffic: " < < std : : fixed < < average_packet_size_for_incoming_traffic < < " bytes \n "
< < " Average packet size for outgoing traffic: " < < std : : fixed < < average_packet_size_for_outgoing_traffic < < " bytes \n " ;
2014-12-05 15:34:59 +01:00
2014-12-04 16:48:11 +01:00
return attack_description . str ( ) ;
2014-07-04 12:04:19 +02:00
}
2014-06-28 23:04:31 +02:00
2014-07-04 12:04:19 +02:00
void send_attack_details ( uint32_t client_ip , attack_details current_attack_details ) {
2015-01-27 20:29:51 +01:00
std : : string pps_as_string = convert_int_to_string ( current_attack_details . attack_power ) ;
std : : string attack_direction = get_direction_name ( current_attack_details . attack_direction ) ;
std : : string client_ip_as_string = convert_ip_as_uint_to_string ( client_ip ) ;
2014-07-04 12:04:19 +02:00
2014-11-15 01:10:04 +01:00
// Very strange code but it work in 95% cases
2014-07-04 12:04:19 +02:00
if ( ban_list_details . count ( client_ip ) > 0 & & ban_list_details [ client_ip ] . size ( ) = = ban_details_records_count ) {
2015-01-27 20:29:51 +01:00
std : : stringstream attack_details ;
2014-07-04 12:04:19 +02:00
attack_details < < get_attack_description ( client_ip , current_attack_details ) < < " \n \n " ;
2014-07-04 13:10:23 +02:00
std : : map < unsigned int , unsigned int > protocol_counter ;
2015-01-27 20:29:51 +01:00
for ( std : : vector < simple_packet > : : iterator iii = ban_list_details [ client_ip ] . begin ( ) ; iii ! = ban_list_details [ client_ip ] . end ( ) ; + + iii ) {
2014-07-04 12:04:19 +02:00
attack_details < < print_simple_packet ( * iii ) ;
2014-07-04 13:10:23 +02:00
protocol_counter [ iii - > protocol ] + + ;
}
std : : map < unsigned int , unsigned int > : : iterator max_proto = std : : max_element ( protocol_counter . begin ( ) , protocol_counter . end ( ) , protocol_counter . value_comp ( ) ) ;
attack_details < < " \n " < < " We got more packets ( "
< < max_proto - > second
< < " from "
< < ban_details_records_count
< < " ) for protocol: " < < get_protocol_name_by_number ( max_proto - > first ) < < " \n " ;
2014-07-04 12:04:19 +02:00
logger < < log4cpp : : Priority : : INFO < < " Attack with direction: " < < attack_direction < <
2014-12-05 15:34:59 +01:00
" IP: " < < client_ip_as_string < < " Power: " < < pps_as_string < < " traffic sample collected " ;
2014-12-05 12:02:46 +01:00
print_attack_details_to_file ( attack_details . str ( ) , client_ip_as_string , current_attack_details ) ;
2014-07-04 12:04:19 +02:00
2014-11-15 01:10:04 +01:00
// Pass attack details to script
2014-07-04 12:04:19 +02:00
if ( file_exists ( notify_script_path ) ) {
logger < < log4cpp : : Priority : : INFO < < " Call script for notify about attack details for: " < < client_ip_as_string ;
2015-01-27 20:29:51 +01:00
std : : string script_params = notify_script_path + " " + client_ip_as_string + " " + attack_direction + " " + pps_as_string + " ban " ;
2014-07-04 12:04:19 +02:00
// We should execute external script in separate thread because any lag in this code will be very distructive
boost : : thread exec_with_params_thread ( exec_with_stdin_params , script_params , attack_details . str ( ) ) ;
exec_with_params_thread . detach ( ) ;
logger < < log4cpp : : Priority : : INFO < < " Script for notify about attack details is finished: " < < client_ip_as_string ;
2015-04-13 11:30:20 +02:00
}
# ifdef REDIS
if ( redis_enabled ) {
std : : string redis_key_name = client_ip_as_string + " _packets_dump " ;
logger < < log4cpp : : Priority : : INFO < < " Start data save in redis for key: " < < redis_key_name ;
boost : : thread redis_store_thread ( store_data_in_redis , redis_key_name , attack_details . str ( ) ) ;
redis_store_thread . detach ( ) ;
logger < < log4cpp : : Priority : : INFO < < " Finish data save in redis for key: " < < redis_key_name ;
}
# endif
// TODO: here we have definitely RACE CONDITION!!! FIX IT
2014-11-15 01:10:04 +01:00
// Remove key and prevent collection new data about this attack
2015-04-27 09:49:46 +02:00
ban_list_details_mutex . lock ( ) ;
2014-07-04 12:04:19 +02:00
ban_list_details . erase ( client_ip ) ;
2015-04-27 09:49:46 +02:00
ban_list_details_mutex . unlock ( ) ;
2014-10-19 13:50:17 +02:00
}
2014-06-26 12:10:57 +02:00
}
2014-06-30 15:33:14 +02:00
2014-11-13 16:01:15 +01:00
uint64_t convert_conntrack_hash_struct_to_integer ( packed_conntrack_hash * struct_value ) {
uint64_t unpacked_data = 0 ;
memcpy ( & unpacked_data , struct_value , sizeof ( uint64_t ) ) ;
return unpacked_data ;
}
2014-11-14 22:36:54 +01:00
2014-11-15 00:41:54 +01:00
void convert_integer_to_conntrack_hash_struct ( packed_session * packed_connection_data , packed_conntrack_hash * unpacked_data ) {
memcpy ( unpacked_data , packed_connection_data , sizeof ( uint64_t ) ) ;
}
2015-01-27 20:29:51 +01:00
std : : string print_flow_tracking_for_specified_protocol ( contrack_map_type & protocol_map , std : : string client_ip , direction flow_direction ) {
std : : stringstream buffer ;
2014-11-15 00:41:54 +01:00
// We shoud iterate over all fields
2014-11-29 22:50:31 +01:00
int printed_records = 0 ;
2014-11-15 00:41:54 +01:00
for ( contrack_map_type : : iterator itr = protocol_map . begin ( ) ; itr ! = protocol_map . end ( ) ; + + itr ) {
2014-11-29 22:50:31 +01:00
// We should limit number of records in flow dump because syn flood attacks produce thounsands of lines
if ( printed_records > ban_details_records_count ) {
2015-05-08 10:42:20 +02:00
buffer < < " Flows have cropped due to very long list. \n " ;
2014-11-29 22:50:31 +01:00
break ;
}
2014-11-15 00:41:54 +01:00
uint64_t packed_connection_data = itr - > first ;
packed_conntrack_hash unpacked_key_struct ;
convert_integer_to_conntrack_hash_struct ( & packed_connection_data , & unpacked_key_struct ) ;
2015-01-27 20:29:51 +01:00
std : : string opposite_ip_as_string = convert_ip_as_uint_to_string ( unpacked_key_struct . opposite_ip ) ;
2014-11-15 00:41:54 +01:00
if ( flow_direction = = INCOMING ) {
buffer < < client_ip < < " : " < < unpacked_key_struct . dst_port < < " < " < < opposite_ip_as_string < < " : " < < unpacked_key_struct . src_port < < " " ;
} else if ( flow_direction = = OUTGOING ) {
buffer < < client_ip < < " : " < < unpacked_key_struct . src_port < < " > " < < opposite_ip_as_string < < " : " < < unpacked_key_struct . dst_port < < " " ;
}
buffer < < itr - > second . bytes < < " bytes " < < itr - > second . packets < < " packets " ;
buffer < < " \n " ;
2014-11-29 22:50:31 +01:00
printed_records + + ;
2014-11-15 00:41:54 +01:00
}
return buffer . str ( ) ;
}
2014-12-05 14:09:25 +01:00
/*
Attack types :
- syn flood : one local port , multiple remote hosts ( and maybe multiple remote ports ) and small packet size
*/
/* Iterate over all flow tracking table */
2015-01-27 20:29:51 +01:00
bool process_flow_tracking_table ( conntrack_main_struct & conntrack_element , std : : string client_ip ) {
2014-12-05 14:09:25 +01:00
std : : map < uint32_t , unsigned int > uniq_remote_hosts_which_generate_requests_to_us ;
std : : map < unsigned int , unsigned int > uniq_local_ports_which_target_of_connectiuons_from_inside ;
/* Process incoming TCP connections */
for ( contrack_map_type : : iterator itr = conntrack_element . in_tcp . begin ( ) ; itr ! = conntrack_element . in_tcp . end ( ) ; + + itr ) {
uint64_t packed_connection_data = itr - > first ;
packed_conntrack_hash unpacked_key_struct ;
convert_integer_to_conntrack_hash_struct ( & packed_connection_data , & unpacked_key_struct ) ;
uniq_remote_hosts_which_generate_requests_to_us [ unpacked_key_struct . opposite_ip ] + + ;
uniq_local_ports_which_target_of_connectiuons_from_inside [ unpacked_key_struct . dst_port ] + + ;
// we can calc average packet size
// string opposite_ip_as_string = convert_ip_as_uint_to_string(unpacked_key_struct.opposite_ip);
// unpacked_key_struct.src_port
// unpacked_key_struct.dst_port
// itr->second.packets
// itr->second.bytes
}
2014-12-16 11:56:08 +01:00
return true ;
2014-12-05 14:09:25 +01:00
}
2015-01-27 20:29:51 +01:00
std : : string print_flow_tracking_for_ip ( conntrack_main_struct & conntrack_element , std : : string client_ip ) {
std : : stringstream buffer ;
2014-11-15 00:41:54 +01:00
2015-01-27 20:29:51 +01:00
std : : string in_tcp = print_flow_tracking_for_specified_protocol ( conntrack_element . in_tcp , client_ip , INCOMING ) ;
std : : string in_udp = print_flow_tracking_for_specified_protocol ( conntrack_element . in_udp , client_ip , INCOMING ) ;
2014-11-15 00:41:54 +01:00
2015-05-08 10:42:20 +02:00
unsigned long long total_number_of_incoming_tcp_flows = conntrack_element . in_tcp . size ( ) ;
unsigned long long total_number_of_incoming_udp_flows = conntrack_element . in_udp . size ( ) ;
unsigned long long total_number_of_outgoing_tcp_flows = conntrack_element . out_tcp . size ( ) ;
unsigned long long total_number_of_outgoing_udp_flows = conntrack_element . out_udp . size ( ) ;
2014-11-15 00:41:54 +01:00
bool we_have_incoming_flows = in_tcp . length ( ) > 0 or in_udp . length ( ) > 0 ;
if ( we_have_incoming_flows ) {
buffer < < " Incoming \n \n " ;
if ( in_tcp . length ( ) > 0 ) {
2015-05-08 10:42:20 +02:00
buffer < < " TCP flows: " < < total_number_of_incoming_tcp_flows < < " \n " ;
buffer < < in_tcp < < " \n " ;
2014-11-15 00:41:54 +01:00
}
if ( in_udp . length ( ) > 0 ) {
2015-05-08 10:42:20 +02:00
buffer < < " UDP flows: " < < total_number_of_incoming_udp_flows < < " \n " ;
buffer < < in_udp < < " \n " ;
2014-11-15 00:41:54 +01:00
}
}
2015-01-27 20:29:51 +01:00
std : : string out_tcp = print_flow_tracking_for_specified_protocol ( conntrack_element . out_tcp , client_ip , OUTGOING ) ;
std : : string out_udp = print_flow_tracking_for_specified_protocol ( conntrack_element . out_udp , client_ip , OUTGOING ) ;
2014-11-15 00:41:54 +01:00
bool we_have_outgoing_flows = out_tcp . length ( ) > 0 or out_udp . length ( ) > 0 ;
// print delimiter if we have flows in both directions
if ( we_have_incoming_flows & & we_have_outgoing_flows ) {
buffer < < " \n " ;
}
if ( we_have_outgoing_flows ) {
buffer < < " Outgoing \n \n " ;
if ( out_tcp . length ( ) > 0 ) {
2015-05-08 10:42:20 +02:00
buffer < < " TCP flows: " < < total_number_of_outgoing_tcp_flows < < " \n " ;
buffer < < out_tcp < < " \n " ;
2014-11-15 00:41:54 +01:00
}
if ( out_udp . length ( ) > 0 ) {
2015-05-08 10:42:20 +02:00
buffer < < " UDP flows: " < < total_number_of_outgoing_udp_flows < < " \n " ;
buffer < < out_udp < < " \n " ;
2014-11-15 00:41:54 +01:00
}
}
return buffer . str ( ) ;
}
2014-11-14 22:36:54 +01:00
2015-01-27 20:29:51 +01:00
std : : string print_ban_thresholds ( ) {
std : : stringstream output_buffer ;
2014-12-04 17:07:44 +01:00
2014-12-05 10:46:01 +01:00
output_buffer < < " Configuration params: \n " ;
if ( we_do_real_ban ) {
output_buffer < < " We call ban script: yes \n " ;
} else {
output_buffer < < " We call ban script: no \n " ;
}
2014-12-04 17:07:44 +01:00
output_buffer < < " Packets per second: " ;
if ( enable_ban_for_pps ) {
output_buffer < < ban_threshold_pps ;
} else {
output_buffer < < " disabled " ;
}
output_buffer < < " \n " ;
output_buffer < < " Mbps per second: " ;
if ( enable_ban_for_bandwidth ) {
output_buffer < < ban_threshold_mbps ;
} else {
output_buffer < < " disabled " ;
}
output_buffer < < " \n " ;
output_buffer < < " Flows per second: " ;
if ( enable_ban_for_flows_per_second ) {
output_buffer < < ban_threshold_flows ;
} else {
output_buffer < < " disabled " ;
}
output_buffer < < " \n " ;
return output_buffer . str ( ) ;
}
2015-01-27 20:29:51 +01:00
void print_attack_details_to_file ( std : : string details , std : : string client_ip_as_string , attack_details current_attack ) {
std : : ofstream my_attack_details_file ;
2014-12-05 12:02:46 +01:00
2015-01-27 20:29:51 +01:00
std : : string ban_timestamp_as_string = print_time_t_in_fastnetmon_format ( current_attack . ban_timestamp ) ;
std : : string attack_dump_path = attack_details_folder + " / " + client_ip_as_string + " _ " + ban_timestamp_as_string ;
2014-12-05 12:02:46 +01:00
2015-01-27 20:29:51 +01:00
my_attack_details_file . open ( attack_dump_path . c_str ( ) , std : : ios : : app ) ;
2014-12-05 12:02:46 +01:00
if ( my_attack_details_file . is_open ( ) ) {
my_attack_details_file < < details < < " \n \n " ;
my_attack_details_file . close ( ) ;
} else {
2014-12-19 09:43:22 +01:00
logger < < log4cpp : : Priority : : ERROR < < " Can't print attack details to file " ;
2014-12-05 12:02:46 +01:00
}
}
2015-03-13 16:25:13 +01:00
// Return true when we should ban this IP
2015-03-13 17:29:00 +01:00
bool we_should_ban_this_ip ( map_element * average_speed_element ) {
uint64_t in_pps_average = average_speed_element - > in_packets ;
uint64_t out_pps_average = average_speed_element - > out_packets ;
2015-03-13 16:25:13 +01:00
2015-03-13 17:29:00 +01:00
uint64_t in_bps_average = average_speed_element - > in_bytes ;
uint64_t out_bps_average = average_speed_element - > out_bytes ;
2015-03-13 16:25:13 +01:00
2015-03-13 17:29:00 +01:00
uint64_t in_flows_average = average_speed_element - > in_flows ;
uint64_t out_flows_average = average_speed_element - > out_flows ;
2015-03-13 16:25:13 +01:00
// we detect overspeed by packets
bool attack_detected_by_pps = false ;
bool attack_detected_by_bandwidth = false ;
bool attack_detected_by_flow = false ;
if ( enable_ban_for_pps & & ( in_pps_average > ban_threshold_pps or out_pps_average > ban_threshold_pps ) ) {
attack_detected_by_pps = true ;
}
// we detect overspeed by bandwidth
if ( enable_ban_for_bandwidth & & ( convert_speed_to_mbps ( in_bps_average ) > ban_threshold_mbps or convert_speed_to_mbps ( out_bps_average ) > ban_threshold_mbps ) ) {
attack_detected_by_bandwidth = true ;
}
2014-12-16 12:59:24 +01:00
2015-03-13 16:25:13 +01:00
if ( enable_ban_for_flows_per_second & & ( in_flows_average > ban_threshold_flows or out_flows_average > ban_threshold_flows ) ) {
attack_detected_by_flow = true ;
}
return attack_detected_by_pps or attack_detected_by_bandwidth or attack_detected_by_flow ;
}
2015-05-07 15:51:10 +02:00
attack_type_t detect_attack_type ( attack_details & current_attack ) {
double threshold_value = 0.9 ;
if ( current_attack . attack_direction = = INCOMING ) {
if ( current_attack . tcp_syn_in_packets > threshold_value * current_attack . in_packets ) {
return ATTACK_SYN_FLOOD ;
} else if ( current_attack . icmp_in_packets > threshold_value * current_attack . in_packets ) {
return ATTACK_ICMP_FLOOD ;
} else if ( current_attack . fragmented_in_packets > threshold_value * current_attack . in_packets ) {
2015-05-08 09:36:51 +02:00
return ATTACK_IP_FRAGMENTATION_FLOOD ;
2015-05-07 15:51:10 +02:00
} else if ( current_attack . udp_in_packets > threshold_value * current_attack . in_packets ) {
return ATTACK_UDP_FLOOD ;
}
} else if ( current_attack . attack_direction = = OUTGOING ) {
if ( current_attack . tcp_syn_out_packets > threshold_value * current_attack . out_packets ) {
return ATTACK_SYN_FLOOD ;
} else if ( current_attack . icmp_out_packets > threshold_value * current_attack . out_packets ) {
return ATTACK_ICMP_FLOOD ;
} else if ( current_attack . fragmented_out_packets > threshold_value * current_attack . out_packets ) {
2015-05-08 09:36:51 +02:00
return ATTACK_IP_FRAGMENTATION_FLOOD ;
2015-05-07 15:51:10 +02:00
} else if ( current_attack . udp_out_packets > threshold_value * current_attack . out_packets ) {
return ATTACK_UDP_FLOOD ;
}
}
return ATTACK_UNKNOWN ;
}
std : : string get_printable_attack_name ( attack_type_t attack ) {
if ( attack = = ATTACK_SYN_FLOOD ) {
return " syn_flood " ;
} else if ( attack = = ATTACK_ICMP_FLOOD ) {
return " icmp_flood " ;
} else if ( attack = = ATTACK_UDP_FLOOD ) {
return " udp_flood " ;
2015-05-08 09:36:51 +02:00
} else if ( attack = = ATTACK_IP_FRAGMENTATION_FLOOD ) {
return " ip_fragmentation " ;
2015-05-07 15:51:10 +02:00
} else if ( attack = = ATTACK_UNKNOWN ) {
return " unknown " ;
2015-05-11 17:50:01 +02:00
} else {
return " unknown " ;
2015-05-07 15:51:10 +02:00
}
}