Introduced new parser for traffic.

src/CMakeLists.txt

@@ -236,13 +236,18 @@ if (ENABLE_AFPACKET_SUPPORT)
    target_link_libraries(afpacket_plugin unified_parser)
 endif()
 
+# Library with data types for parsing network structures
 add_library(network_data_structures STATIC network_data_structures.cpp)
 
-# sFLOW plugin
+# Our new parser for parsing traffic up to L4
+add_library(simple_packet_parser_ng STATIC simple_packet_parser_ng.cpp)
+target_link_libraries(simple_packet_parser_ng network_data_structures)
+
+# sFlow plugin
 add_library(sflow_plugin STATIC sflow_plugin/sflow_collector.cpp)
 
-# Link sFlow plugin with new network data structures parser
-target_link_libraries(sflow_plugin network_data_structures)
+# Link sFlow plugin with new traffic parser
+target_link_libraries(sflow_plugin simple_packet_parser_ng)
 
 # netflow plugin
 add_library(netflow_plugin STATIC netflow_plugin/netflow_collector.cpp)

src/afpacket_plugin/afpacket_collector.cpp

@@ -132,7 +132,7 @@ void walk_block(struct block_desc* pbd, const int block_num) {
 
         u_char* data_pointer = (u_char*)((uint8_t*)ppd + ppd->tp_mac);
 
-        simple_packet packet;
+        simple_packet_t packet;
         int parser_result = parse_raw_packet_to_simple_packet((u_char*)data_pointer, ppd->tp_snaplen, packet,
                                                               afpacket_read_packet_length_from_ip_header);
 

src/example_plugin/example_collector.cpp

@@ -34,7 +34,7 @@ void start_example_collection(process_packet_pointer func_ptr) {
     }
 
     // We should fill this structure for passing to FastNetMon
-    simple_packet current_packet;
+    simple_packet_t current_packet;
 
     current_packet.src_ip = 0;
     current_packet.dst_ip = 0;

src/fast_library.cpp

@@ -446,7 +446,7 @@ int clear_bit_value(uint16_t& num, int bit) {
     }
 }
 
-std::string print_simple_packet(simple_packet packet) {
+std::string print_simple_packet(simple_packet_t packet) {
     std::stringstream buffer;
 
     if (packet.ts.tv_sec == 0) {

src/fast_library.h

@@ -51,7 +51,7 @@ uint32_t convert_ip_as_string_to_uint(std::string ip);
 std::string convert_ip_as_uint_to_string(uint32_t ip_as_integer);
 std::string convert_int_to_string(int value);
 std::string print_ipv6_address(struct in6_addr& ipv6_address);
-std::string print_simple_packet(simple_packet packet);
+std::string print_simple_packet(simple_packet_t packet);
 std::string convert_timeval_to_date(struct timeval tv);
 bool convert_hex_as_string_to_uint(std::string hex, uint32_t& value);
 

src/fastnetmon.cpp

@@ -393,7 +393,7 @@ map_for_counters GeoIpCounter;
 
 // In ddos info we store attack power and direction
 std::map<uint32_t, banlist_item> ban_list;
-std::map<uint32_t, std::vector<simple_packet> > ban_list_details;
+std::map<uint32_t, std::vector<simple_packet_t> > ban_list_details;
 
 host_group_map_t host_groups;
 
@@ -472,7 +472,7 @@ void free_up_all_resources();
 std::string print_ddos_attack_details();
 void recalculate_speed();
 std::string print_channel_speed(std::string traffic_type, direction packet_direction);
-void process_packet(simple_packet& current_packet);
+void process_packet(simple_packet_t& current_packet);
 void traffic_draw_program();
 void interruption_signal_handler(int signal_number);
 
@@ -517,7 +517,7 @@ class FastnetmonApiServiceImpl final : public Fastnetmon::Service {
         ban_list_mutex.unlock();
 
         ban_list_details_mutex.lock();
-        ban_list_details[client_ip] = std::vector<simple_packet>();
+        ban_list_details[client_ip] = std::vector<simple_packet_t>();
         ban_list_details_mutex.unlock();
 
         logger << log4cpp::Priority::INFO << "API call ban handlers manually";
@@ -1711,10 +1711,10 @@ bool load_our_networks_list() {
 
 #ifdef IPV6_HASH_COUNTERS
 
-moodycamel::ConcurrentQueue<simple_packet> multi_process_queue_for_ipv6_counters;
+moodycamel::ConcurrentQueue<simple_packet_t> multi_process_queue_for_ipv6_counters;
 
 void ipv6_traffic_processor() {
-    simple_packet packets_from_queue[32];
+    simple_packet_t packets_from_queue[32];
 
     while (true) {
         std::size_t count = 0;
@@ -1758,7 +1758,7 @@ void ipv6_traffic_processor() {
 #endif
 
 /* Process simple unified packet */
-void process_packet(simple_packet& current_packet) {
+void process_packet(simple_packet_t& current_packet) {
     // Packets dump is very useful for bug hunting
     if (DEBUG_DUMP_ALL_PACKETS) {
         logger << log4cpp::Priority::INFO << "Dump: " << print_simple_packet(current_packet);
@@ -3334,7 +3334,7 @@ void execute_ip_ban(uint32_t client_ip, map_element average_speed_element, std::
     ban_list_mutex.unlock();
 
     ban_list_details_mutex.lock();
-    ban_list_details[client_ip] = std::vector<simple_packet>();
+    ban_list_details[client_ip] = std::vector<simple_packet_t>();
     ban_list_details_mutex.unlock();
 
     logger << log4cpp::Priority::INFO << "Attack with direction: " << data_direction_as_string
@@ -3667,11 +3667,11 @@ std::string get_attack_description_in_json(uint32_t client_ip, attack_details& c
     return json_as_text;
 }
 
-std::string generate_simple_packets_dump(std::vector<simple_packet>& ban_list_details) {
+std::string generate_simple_packets_dump(std::vector<simple_packet_t>& ban_list_details) {
     std::stringstream attack_details;
 
     std::map<unsigned int, unsigned int> protocol_counter;
-    for (std::vector<simple_packet>::iterator iii = ban_list_details.begin();
+    for (std::vector<simple_packet_t>::iterator iii = ban_list_details.begin();
          iii != ban_list_details.end(); ++iii) {
         attack_details << print_simple_packet(*iii);
 

src/fastnetmon_simple_packet.h

@@ -7,9 +7,9 @@
 enum direction { INCOMING = 0, OUTGOING, INTERNAL, OTHER };
 
 // simplified packet struct for lightweight save into memory
-class simple_packet {
+class simple_packet_t {
     public:
-    simple_packet()
+    simple_packet_t()
     : sample_ratio(1), src_ip(0), dst_ip(0), source_port(0), destination_port(0), protocol(0),
       length(0), flags(0), number_of_packets(1), ip_fragmented(false), ip_protocol_version(4),
       ttl(0), packet_payload_pointer(NULL), packet_payload_length(0), packet_direction(OTHER) {
@@ -29,13 +29,24 @@ class simple_packet {
     uint16_t source_port;
     uint16_t destination_port;
     unsigned int protocol;
+    
     uint64_t length;
+    uint64_t ip_length = 0; /* IP packet total length. We use it in addition to length because flow spec rule need this length */
+
     uint64_t number_of_packets; /* for netflow */
     uint8_t flags; /* tcp flags */
+
     bool ip_fragmented; /* If IP packet fragmented */
+    bool ip_dont_fragment               = false; /* If IP has don't fragment flag */
+
     struct timeval ts;
     void* packet_payload_pointer;
     int packet_payload_length;
+    uint32_t packet_payload_full_length = 0; // In case of cropped packets we use this
+
+    // vlan tag if we can extract it
+    uint32_t vlan = 0;
+
     // We store packet direction here because direction calculation is very difficult task for cpu
     direction packet_direction;
 };

src/fastnetmon_types.h

@@ -36,7 +36,7 @@ typedef std::vector<subnet_t> subnet_vector_t;
 typedef std::map<subnet_t, std::string> subnet_to_host_group_map_t;
 typedef std::map<std::string, subnet_vector_t> host_group_map_t;
 
-typedef void (*process_packet_pointer)(simple_packet&);
+typedef void (*process_packet_pointer)(simple_packet_t&);
 
 // Enum with available sort by field
 enum sort_type { PACKETS, BYTES, FLOWS };

src/netflow_plugin/netflow_collector.cpp

@@ -72,7 +72,7 @@ void add_peer_template(global_template_storage_t& table_for_add,
 int nf9_rec_to_flow(u_int record_type,
                     u_int record_length,
                     u_int8_t* data,
-                    simple_packet& packet,
+                    simple_packet_t& packet,
                     netflow9_template_records_map& template_records);
 
 struct peer_nf9_template* peer_find_template(global_template_storage_t& table_for_lookup,
@@ -366,7 +366,7 @@ void add_peer_template(global_template_storage_t& table_for_add,
         memcpy(&packet.flow_field, data, record_length); \
         break
 
-int nf9_rec_to_flow(u_int record_type, u_int record_length, u_int8_t* data, simple_packet& packet) {
+int nf9_rec_to_flow(u_int record_type, u_int record_length, u_int8_t* data, simple_packet_t& packet) {
     /* XXX: use a table-based interpreter */
     switch (record_type) {
         V9_FIELD(NF9_IN_BYTES, OCTETS, length);
@@ -447,7 +447,7 @@ int nf9_rec_to_flow(u_int record_type, u_int record_length, u_int8_t* data, simp
     return 0;
 }
 
-int nf10_rec_to_flow(u_int record_type, u_int record_length, u_int8_t* data, simple_packet& packet) {
+int nf10_rec_to_flow(u_int record_type, u_int record_length, u_int8_t* data, simple_packet_t& packet) {
     /* XXX: use a table-based interpreter */
     switch (record_type) {
         V9_FIELD(NF10_IN_BYTES, OCTETS, length);
@@ -486,7 +486,7 @@ void nf10_flowset_to_store(u_int8_t* pkt, size_t len, struct NF10_HEADER* nf10_h
         return;
     }
 
-    simple_packet packet;
+    simple_packet_t packet;
     // We use shifted values and should process only zeroed values
     // because we are working with little and big endian data in same time
     packet.number_of_packets = 0;
@@ -579,7 +579,7 @@ void nf9_flowset_to_store(u_int8_t* pkt, size_t len, struct NF9_HEADER* nf9_hdr,
 
     u_int offset = 0;
 
-    simple_packet packet;
+    simple_packet_t packet;
     // We use shifted values and should process only zeroed values
     // because we are working with little and big endian data in same time
     packet.number_of_packets = 0;
@@ -943,7 +943,7 @@ void process_netflow_packet_v5(u_int8_t* packet, u_int len, std::string client_a
         nf5_flow->if_index_out = fast_ntoh(nf5_flow->if_index_out);
 
         // convert netflow to simple packet form
-        simple_packet current_packet;
+        simple_packet_t current_packet;
 
         current_packet.src_ip = nf5_flow->src_ip;
         current_packet.dst_ip = nf5_flow->dest_ip;

src/netmap_plugin/netmap_collector.cpp

@@ -94,7 +94,7 @@ int receive_packets(struct netmap_ring* ring, int thread_number) {
 
 void consume_pkt(u_char* buffer, int len, int thread_number) {
     // We should fill this structure for passing to FastNetMon
-    simple_packet packet;
+    simple_packet_t packet;
 
     packet.sample_ratio = netmap_sampling_ratio;
 

src/pcap_plugin/pcap_collector.cpp

@@ -108,7 +108,7 @@ void parse_packet(u_char* user, struct pcap_pkthdr* packethdr, const u_char* pac
     // host byte order
     unsigned int packet_length = ntohs(iphdr->ip_len);
 
-    simple_packet current_packet;
+    simple_packet_t current_packet;
 
     // Advance to the transport layer header then parse and display
     // the fields based on the type of hearder: tcp, udp or icmp

src/pcap_reader.cpp

@@ -73,7 +73,7 @@ void init_logging() {
 
 void pcap_parse_packet(const char* flow_type, char* buffer, uint32_t len);
 
-void my_fastnetmon_packet_handler(simple_packet& current_packet) {
+void my_fastnetmon_packet_handler(simple_packet_t& current_packet) {
     std::cout << print_simple_packet(current_packet);
 }
 
@@ -150,7 +150,7 @@ void pcap_parse_packet(char* buffer, uint32_t len, uint32_t snap_len) {
         fastnetmon_print_parsed_pkt(print_buffer, 512, (u_char*)buffer, &raw_packet_header);
         printf("Raw parser: %s", print_buffer);
 
-        simple_packet packet;
+        simple_packet_t packet;
         // TODO: add support for caplen here!
         if (parse_raw_packet_to_simple_packet((u_char*)buffer, len, packet, false)) {
             std::cout << "High level parser: " << print_simple_packet(packet) << std::endl;

src/pfring_plugin/pfring_collector.cpp

@@ -155,7 +155,7 @@ void stop_pfring_collection() {
 
 void parse_packet_pf_ring(const struct pfring_pkthdr* h, const u_char* p, const u_char* user_bytes) {
     // Description of all fields: http://www.ntop.org/pfring_api/structpkt__parsing__info.html
-    simple_packet packet;
+    simple_packet_t packet;
 
     // We pass only one packet to processing
     packet.number_of_packets = 1;

src/sflow_plugin/sflow_collector.cpp

@@ -58,7 +58,7 @@ bool readFlowSample(SFSample* sample, int expanded);
 void readFlowSample_header(SFSample* sample);
 void decode_ipv4_protocol(SFSample* sample);
 void decode_ipv6_protocol(SFSample* sample);
-void print_simple_packet(struct simple_packet& packet);
+void print_simple_packet(simple_packet_t& packet);
 
 process_packet_pointer sflow_process_func_ptr = NULL;
 
@@ -688,7 +688,7 @@ void decodeIPLayer4(SFSample* sample, uint8_t* ptr) {
         return;
     }
 
-    simple_packet current_packet;
+    simple_packet_t current_packet;
 
     if (sample->gotIPV6) {
         current_packet.ip_protocol_version = 6;

src/simple_packet_parser_ng.cpp

@@ -0,0 +1,132 @@
+#include "simple_packet_parser_ng.h"
+#include "all_logcpp_libraries.h"
+#include "network_data_structures.hpp"
+
+#include <cstring>
+
+using namespace network_data_stuctures;
+
+// Our own native function to convert wire packet into simple_packet_t
+// TODO: development is going here, we still need to add number of options here
+// It based on code from parse_ipv4_or_ipv6_packet_up_to_l3
+parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
+                                                        int length_before_sampling,
+                                                        int captured_length,
+                                                        simple_packet_t& packet,
+                                                        bool use_packet_length_from_wire) {
+    // We are using pointer copy because we are changing it
+    uint8_t* local_pointer = pointer;
+
+    // It's very nice for doing checks
+    uint8_t* end_pointer = pointer + captured_length;
+
+    // Return error if it shorter then ethernet headers
+    if (local_pointer + sizeof(ethernet_header_t) > end_pointer) {
+        return parser_code_t::memory_violation;
+    }
+
+    ethernet_header_t* ethernet_header = (ethernet_header_t*)local_pointer;
+    ethernet_header->convert();
+
+    local_pointer += sizeof(ethernet_header_t);
+
+    // Here we store IPv4 or IPv6 l4 protocol numbers
+    uint8_t protocol = 0;
+
+    if (ethernet_header->ethertype == IanaEthertypeVLAN) {
+        // Return error if it shorter then vlan header
+        if (local_pointer + sizeof(ethernet_vlan_header_t) > end_pointer) {
+            return parser_code_t::memory_violation;
+        }
+
+        ethernet_vlan_header_t* ethernet_vlan_header = (ethernet_vlan_header_t*)local_pointer;
+        ethernet_vlan_header->convert();
+
+        packet.vlan = ethernet_vlan_header->vlan_id;
+
+        local_pointer += sizeof(ethernet_vlan_header_t);
+
+        // Change ethernet ethertype to vlan's ethertype
+        ethernet_header->ethertype = ethernet_vlan_header->ethertype;
+    }
+
+    if (ethernet_header->ethertype == IanaEthertypeIPv4) {
+        // Return error if pointer is shorter then IP header
+        if (local_pointer + sizeof(ipv4_header_t) > end_pointer) {
+            return parser_code_t::memory_violation;
+        }
+
+        ipv4_header_t* ipv4_header = (ipv4_header_t*)local_pointer;
+
+        // Populate IP specific options in packet structure before making any conversions, use network representation of
+        // addresses
+        packet.src_ip = ipv4_header->source_ip;
+        packet.dst_ip = ipv4_header->destination_ip;
+
+        packet.ip_protocol_version = 4;
+
+        // Convert all integers in IP header to little endian
+        ipv4_header->convert();
+
+        packet.ttl              = ipv4_header->ttl;
+        packet.ip_length        = ipv4_header->total_length;
+        packet.ip_dont_fragment = ipv4_header->dont_fragment_flag;
+
+        packet.ip_fragmented = ipv4_header->is_fragmented();
+
+        // We keep these variables to maintain backward compatibility with parse_raw_packet_to_simple_packet_full()
+        packet.packet_payload_length      = length_before_sampling;
+        packet.packet_payload_full_length = length_before_sampling;
+
+        // Pointer to payload
+        packet.packet_payload_pointer = (void*)pointer;
+
+        protocol        = ipv4_header->protocol;
+        packet.protocol = protocol;
+
+        if (use_packet_length_from_wire) {
+            packet.length = length_before_sampling;
+        } else {
+            packet.length = ipv4_header->total_length;
+        }
+
+        // Ignore all IP options and shift pointer to L3 payload
+        local_pointer += 4 * ipv4_header->ihl;
+    } else {
+        // TODO: we do not support IPv6 yet
+        return parser_code_t::not_ipv4;
+    }
+
+    if (protocol == IpProtocolNumberTCP) {
+        if (local_pointer + sizeof(tcp_header_t) > end_pointer) {
+            return parser_code_t::memory_violation;
+        }
+
+        tcp_header_t* tcp_header = (tcp_header_t*)local_pointer;
+        tcp_header->convert();
+
+        packet.source_port      = tcp_header->source_port;
+        packet.destination_port = tcp_header->destination_port;
+
+        // TODO: rework this code to use structs with bit fields
+        packet.flags = tcp_header->fin * 0x01 + tcp_header->syn * 0x02 + tcp_header->rst * 0x04 +
+                       tcp_header->psh * 0x08 + tcp_header->ack * 0x10 + tcp_header->urg * 0x20;
+
+    } else if (protocol == IpProtocolNumberUDP) {
+        if (local_pointer + sizeof(udp_header_t) > end_pointer) {
+            return parser_code_t::memory_violation;
+        }
+
+        udp_header_t* udp_header = (udp_header_t*)local_pointer;
+        udp_header->convert();
+
+        packet.source_port      = udp_header->source_port;
+        packet.destination_port = udp_header->destination_port;
+    } else {
+        // We're not interested in other protocol types
+        return parser_code_t::not_ipv4;
+    }
+
+    return parser_code_t::success;
+}
+

src/simple_packet_parser_ng.h

@@ -0,0 +1,8 @@
+#include "fastnetmon_simple_packet.h"
+#include "network_data_structures.hpp"
+
+network_data_stuctures::parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
+                                                                                int length_before_sampling,
+                                                                                int captured_length,
+                                                                                simple_packet_t& packet,
+                                                                                bool use_packet_length_from_wire);

src/snabbswitch_plugin/snabbswitch_collector.cpp

@@ -74,7 +74,7 @@ struct firehose_rdesc {
 } __attribute__((packed));
 
 void firehose_packet(const char* pciaddr, char* data, int length) {
-    simple_packet packet;
+    simple_packet_t packet;
 
     if (!parse_raw_packet_to_simple_packet((u_char*)data, length, packet, false)) {
         total_unparsed_packets++;

src/tests/conntrack_prototype.cpp

@@ -77,7 +77,7 @@ class conntrack_hash_struct_for_simple_packet_t {
 };
 
 // Extract only important for us fields from main simple_packet structure
-bool convert_simple_packet_toconntrack_hash_struct(simple_packet& packet,
+bool convert_simple_packet_toconntrack_hash_struct(simple_packet_t& packet,
                                                    conntrack_hash_struct_for_simple_packet_t& conntrack_struct) {
     conntrack_struct.src_ip = packet.src_ip;
     conntrack_struct.dst_ip = packet.dst_ip;

src/tests/snabb/ndpicallback.cpp

@@ -503,7 +503,7 @@ void firehose_packet(const char* pciaddr, char* data, int length) {
     u_int8_t add_hash = 0;
     fastnetmon_parse_pkt((u_char*)data, &packet_header, 4, timestamp, add_hash);
 
-    simple_packet current_packet;
+    simple_packet_t current_packet;
     parse_raw_packet_to_simple_packet((u_char*)data, length, current_packet);
 
     conntrack_hash_struct_for_simple_packet_t conntrack_structure;

src/unified_parser.cpp

@@ -2,7 +2,7 @@
 
 #include "fastnetmon_packet_parser.h"
 
-bool parse_raw_packet_to_simple_packet(u_char* buffer, int len, simple_packet& packet, bool netmap_read_packet_length_from_ip_header) {
+bool parse_raw_packet_to_simple_packet(u_char* buffer, int len, simple_packet_t& packet, bool netmap_read_packet_length_from_ip_header) {
 
     struct pfring_pkthdr packet_header;
 

src/unified_parser.hpp

@@ -4,4 +4,4 @@
 #include <stdint.h>
 #include <sys/types.h>
 
-bool parse_raw_packet_to_simple_packet(u_char* buffer, int len, simple_packet& packet, bool netmap_read_packet_length_from_ip_header);
+bool parse_raw_packet_to_simple_packet(u_char* buffer, int len, simple_packet_t& packet, bool netmap_read_packet_length_from_ip_header);