Poretd parser to new logic which does not alter buffer content during parsing process

src/fast_library.cpp

@@ -1324,8 +1324,8 @@ bool read_simple_packet(uint8_t* buffer, size_t buffer_length, simple_packet_t&
         packet.ip_fragmented              = root.getIpFragmented();
         packet.ts.tv_sec                  = root.getTsSec();
         packet.ts.tv_usec                 = root.getTsMsec();
-        packet.packet_payload_length      = root.getPacketPayloadLength();
+        packet.captured_payload_length      = root.getPacketPayloadLength();
-        packet.packet_payload_full_length = root.getPacketPayloadFullLength();
+        packet.payload_full_length = root.getPacketPayloadFullLength();
         packet.packet_direction           = (direction_t)root.getPacketDirection();
         packet.source                     = (source_t)root.getSource();
     } catch (kj::Exception e) {
@@ -1361,8 +1361,8 @@ bool write_simple_packet(int fd, simple_packet_t& packet, bool populate_ipv6) {
     capnp_packet.setIpFragmented(packet.ip_fragmented);
     capnp_packet.setTsSec(packet.ts.tv_sec);
     capnp_packet.setTsMsec(packet.ts.tv_usec);
-    capnp_packet.setPacketPayloadLength(packet.packet_payload_length);
+    capnp_packet.setPacketPayloadLength(packet.captured_payload_length);
-    capnp_packet.setPacketPayloadFullLength(packet.packet_payload_full_length);
+    capnp_packet.setPacketPayloadFullLength(packet.payload_full_length);
     capnp_packet.setPacketDirection(packet.packet_direction);
     capnp_packet.setSource(packet.source);
     capnp_packet.setSrcAsn(packet.src_asn);

src/fastnetmon_simple_packet.hpp

@@ -38,6 +38,9 @@ class simple_packet_t {
     in6_addr src_ipv6{};
     in6_addr dst_ipv6{};
 
+    uint8_t source_mac[6]{};
+    uint8_t destination_mac[6]{};
+
     // ASNs
     uint32_t src_asn = 0;
     uint32_t dst_asn = 0;
@@ -79,17 +82,22 @@ class simple_packet_t {
     // If IP has don't fragment flag
     bool ip_dont_fragment = false;
 
+    // Fragment offset in bytes when fragmentation involved
+    uint16_t ip_fragment_offset = 0;
+
     // Time when we actually received this packet, we use quite rough and inaccurate but very fast time source for it
     time_t arrival_time = 0;
 
     // Timestamp of packet as reported by Netflow or IPFIX agent on device, it may be very inaccurate as nobody cares about time on equipment
     struct timeval ts = { 0, 0 };
 
-    void* packet_payload_pointer  = nullptr;
+    void* payload_pointer  = nullptr;
-    int32_t packet_payload_length = 0;
 
-    // In case of cropped packets we use this field
+    // Part of packet we captured from wire. It may not be full length of packet
-    uint32_t packet_payload_full_length = 0;
+    int32_t captured_payload_length = 0;
+
+    // Full length of packet we observed. It may be larger then packet_captured_payload_length in case of cropped mirror or sFlow traffic
+    uint32_t payload_full_length = 0;
 
     // Forwarding status
     forwarding_status_t forwarding_status = forwarding_status_t::unknown;

src/sflow_plugin/sflow_collector.cpp

@@ -421,9 +421,9 @@ bool process_sflow_flow_sample(uint8_t* data_pointer,
             }
 
             // Pass pointer to raw header to FastNetMon processing functions
-            packet.packet_payload_pointer     = header_payload_pointer;
+            packet.payload_pointer     = header_payload_pointer;
-            packet.packet_payload_full_length = sflow_raw_protocol_header.frame_length_before_sampling;
+            packet.payload_full_length = sflow_raw_protocol_header.frame_length_before_sampling;
-            packet.packet_payload_length      = sflow_raw_protocol_header.header_size;
+            packet.captured_payload_length      = sflow_raw_protocol_header.header_size;
 
             packet.sample_ratio = sflow_sample_header_unified_accessor.sampling_rate;
 

src/simple_packet_parser_ng.cpp

@@ -2,6 +2,8 @@
 #include "all_logcpp_libraries.hpp"
 #include "network_data_structures.hpp"
 
+#include <algorithm>
+#include <iterator>
 #include <cstring>
 
 using namespace network_data_stuctures;
@@ -11,18 +13,17 @@ using namespace network_data_stuctures;
 bool decode_mpls = false;
 
 // Our own native function to convert wire packet into simple_packet_t
-// TODO: development is going here, we still need to add number of options here
+parser_code_t parse_raw_packet_to_simple_packet_full_ng(const uint8_t* pointer,
-parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
                                                         int length_before_sampling,
                                                         int captured_length,
                                                         simple_packet_t& packet,
                                                         bool unpack_gre,
                                                         bool read_packet_length_from_ip_header) {
     // We are using pointer copy because we are changing it
-    uint8_t* local_pointer = pointer;
+    const uint8_t* local_pointer = pointer;
 
     // It's very nice for doing checks
-    uint8_t* end_pointer = pointer + captured_length;
+    const uint8_t* end_pointer = pointer + captured_length;
 
     // Return error if it shorter then ethernet headers
     if (local_pointer + sizeof(ethernet_header_t) > end_pointer) {
@@ -30,29 +31,35 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
     }
 
     ethernet_header_t* ethernet_header = (ethernet_header_t*)local_pointer;
-    ethernet_header->convert();
+
+    // Copy Ethernet MAC addresses to packet structure using native C++ approach to avoid touching memory with memcpy
+    std::copy(std::begin(ethernet_header->source_mac), std::end(ethernet_header->source_mac), std::begin(packet.source_mac));
+
+    std::copy(std::begin(ethernet_header->destination_mac), std::end(ethernet_header->destination_mac), std::begin(packet.destination_mac));
 
     local_pointer += sizeof(ethernet_header_t);
 
-    if (ethernet_header->ethertype == IanaEthertypeVLAN) {
+    // Copy ethertype as we may need to change it below
+    uint16_t ethertype = ethernet_header->get_ethertype_host_byte_order();
+
+    if (ethertype == IanaEthertypeVLAN) {
         // Return error if it shorter then vlan header
         if (local_pointer + sizeof(ethernet_vlan_header_t) > end_pointer) {
             return parser_code_t::memory_violation;
         }
 
         ethernet_vlan_header_t* ethernet_vlan_header = (ethernet_vlan_header_t*)local_pointer;
-        ethernet_vlan_header->convert();
 
-        packet.vlan = ethernet_vlan_header->vlan_id;
+        packet.vlan = ethernet_vlan_header->get_vlan_id_host_byte_order();
 
         local_pointer += sizeof(ethernet_vlan_header_t);
 
-        // Change ethernet ethertype to vlan's ethertype
+        // Change ethertype to vlan's ethertype
-        ethernet_header->ethertype = ethernet_vlan_header->ethertype;
+        ethertype = ethernet_vlan_header->get_ethertype_host_byte_order();
     }
 
     if (decode_mpls) {
-        if (ethernet_header->ethertype == IanaEthertypeMPLS_unicast) {
+        if (ethertype == IanaEthertypeMPLS_unicast) {
         REPEAT_MPLS_STRIP:
             if (local_pointer + sizeof(mpls_label_t) > end_pointer) {
                 return parser_code_t::memory_violation;
@@ -75,7 +82,7 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
     // Here we store IPv4 or IPv6 l4 protocol numbers
     uint8_t protocol = 0;
 
-    if (ethernet_header->ethertype == IanaEthertypeIPv4) {
+    if (ethertype == IanaEthertypeIPv4) {
         // Return error if pointer is shorter then IP header
         if (local_pointer + sizeof(ipv4_header_t) > end_pointer) {
             return parser_code_t::memory_violation;
@@ -83,46 +90,54 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
 
         ipv4_header_t* ipv4_header = (ipv4_header_t*)local_pointer;
 
-        // Populate IP specific options in packet structure before making any conversions, use network representation of
+        // Use network representation of addresses
-        // addresses
+        packet.src_ip = ipv4_header->get_source_ip_network_byte_order();
-        packet.src_ip = ipv4_header->source_ip;
+        packet.dst_ip = ipv4_header->get_destination_ip_network_byte_order();
-        packet.dst_ip = ipv4_header->destination_ip;
 
         packet.ip_protocol_version = 4;
 
-        // Convert all integers in IP header to little endian
+        packet.ttl = ipv4_header->get_ttl();
-        ipv4_header->convert();
-
-        packet.ttl = ipv4_header->ttl;
 
         // We need this specific field for Flow Spec mitigation mode
-        packet.ip_length = ipv4_header->total_length;
+        packet.ip_length = ipv4_header->get_total_length_host_byte_order();
 
-        packet.ip_dont_fragment = ipv4_header->dont_fragment_flag;
+        packet.ip_dont_fragment = ipv4_header->get_dont_fragment_flag();
 
         packet.ip_fragmented = ipv4_header->is_fragmented();
 
-        packet.ip_more_fragments = ipv4_header->more_fragments_flag;
+        packet.ip_more_fragments = ipv4_header->get_more_fragments_flag();
+
+        // We must use special function to recover value in a format useable for our consumption
+        // We must not read this field directly
+        packet.ip_fragment_offset = ipv4_header->get_fragment_offset_bytes();
 
         // We keep these variables to maintain backward compatibility with parse_raw_packet_to_simple_packet_full()
-        packet.packet_payload_length      = length_before_sampling;
+        packet.captured_payload_length      = length_before_sampling;
-        packet.packet_payload_full_length = length_before_sampling;
+        packet.payload_full_length = length_before_sampling;
 
         // Pointer to payload
-        packet.packet_payload_pointer = (void*)pointer;
+        packet.payload_pointer = (void*)pointer;
 
-        protocol        = ipv4_header->protocol;
+        protocol        = ipv4_header->get_protocol();
         packet.protocol = protocol;
 
         if (read_packet_length_from_ip_header) {
-            packet.length = ipv4_header->total_length;
+            packet.length = ipv4_header->get_total_length_host_byte_order();
         } else {
             packet.length = length_before_sampling;
         }
 
+        // We need to handle fragmented traffic. In case of IPv4 fragmentation only first packet carries UDP / TCP / other headers
+        // and consequent packets simply lack of this information and we know only protocol for them.
+        // We can consequent packets by non zero fragment_offset
+        if (ipv4_header->get_fragment_offset_bytes() != 0) {
+            // The best we can do it so stop processing here and report success
+            return parser_code_t::success;
+        }
+
         // Ignore all IP options and shift pointer to L3 payload
-        local_pointer += 4 * ipv4_header->ihl;
+        local_pointer += 4 * ipv4_header->get_ihl();
-    } else if (ethernet_header->ethertype == IanaEthertypeIPv6) {
+    } else if (ethertype == IanaEthertypeIPv6) {
         // Return error if pointer is shorter then IP header
         if (local_pointer + sizeof(ipv6_header_t) > end_pointer) {
             return parser_code_t::memory_violation;
@@ -130,31 +145,29 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
 
         ipv6_header_t* ipv6_header = (ipv6_header_t*)local_pointer;
 
-        // Convert all integers in IP header to little endian
+        // TODO: we may use std::copy for it to avoid touching memory with memcpy
-        ipv6_header->convert();
-
         memcpy(&packet.src_ipv6, ipv6_header->source_address, sizeof(packet.src_ipv6));
         memcpy(&packet.dst_ipv6, ipv6_header->destination_address, sizeof(packet.dst_ipv6));
 
         packet.ip_protocol_version = 6;
 
-        packet.ttl = ipv6_header->hop_limit;
+        packet.ttl = ipv6_header->get_hop_limit();
 
         // We need this specific field for Flow Spec mitigation mode
-        packet.ip_length = ipv6_header->payload_length;
+        packet.ip_length = ipv6_header->get_payload_length();
 
         // We keep these variables to maintain backward compatibility with parse_raw_packet_to_simple_packet_full()
-        packet.packet_payload_length      = length_before_sampling;
+        packet.captured_payload_length      = length_before_sampling;
-        packet.packet_payload_full_length = length_before_sampling;
+        packet.payload_full_length = length_before_sampling;
 
         // Pointer to payload
-        packet.packet_payload_pointer = (void*)pointer;
+        packet.payload_pointer = (void*)pointer;
 
-        protocol        = ipv6_header->next_header;
+        protocol        = ipv6_header->get_next_header();
         packet.protocol = protocol;
 
         if (read_packet_length_from_ip_header) {
-            packet.length = ipv6_header->payload_length;
+            packet.length = ipv6_header->get_payload_length();
         } else {
             packet.length = length_before_sampling;
         }
@@ -183,12 +196,12 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
             if (protocol == IpProtocolNumberIPV6_FRAG) {
                 ipv6_extension_header_fragment_t* ipv6_extension_header_fragment = (ipv6_extension_header_fragment_t*)local_pointer;
 
-                ipv6_extension_header_fragment->convert();
-
                 // If we received this header then we assume that packet was fragmented
                 packet.ip_fragmented = true;
 
-                packet.ip_more_fragments = ipv6_extension_header_fragment->more_fragments;
+                packet.ip_more_fragments = ipv6_extension_header_fragment->get_more_fragments();
+
+                packet.ip_fragment_offset = ipv6_extension_header_fragment->get_fragment_offset_bytes();
 
                 // We stop processing here as I believe that it's enough to know that this traffic was fragmented
                 // We do not parse nested protocol in this case at all
@@ -199,7 +212,7 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
 
             return parser_code_t::no_ipv6_options_support;
         }
-    } else if (ethernet_header->ethertype == IanaEthertypeARP) {
+    } else if (ethertype == IanaEthertypeARP) {
         // it's not parser error of course but we need to have visibility about this case
         return parser_code_t::arp;
     } else {
@@ -212,14 +225,13 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
         }
 
         tcp_header_t* tcp_header = (tcp_header_t*)local_pointer;
-        tcp_header->convert();
 
-        packet.source_port      = tcp_header->source_port;
+        packet.source_port      = tcp_header->get_source_port_host_byte_order();
-        packet.destination_port = tcp_header->destination_port;
+        packet.destination_port = tcp_header->get_destination_port_host_byte_order();
 
         // TODO: rework this code to use structs with bit fields
-        packet.flags = tcp_header->fin * 0x01 + tcp_header->syn * 0x02 + tcp_header->rst * 0x04 +
+        packet.flags = tcp_header->get_fin() * 0x01 + tcp_header->get_syn() * 0x02 + tcp_header->get_rst() * 0x04 +
-                       tcp_header->psh * 0x08 + tcp_header->ack * 0x10 + tcp_header->urg * 0x20;
+                       tcp_header->get_psh() * 0x08 + tcp_header->get_ack() * 0x10 + tcp_header->get_urg() * 0x20;
 
     } else if (protocol == IpProtocolNumberUDP) {
         if (local_pointer + sizeof(udp_header_t) > end_pointer) {
@@ -227,32 +239,32 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
         }
 
         udp_header_t* udp_header = (udp_header_t*)local_pointer;
-        udp_header->convert();
 
-        packet.source_port      = udp_header->source_port;
+        packet.source_port      = udp_header->get_source_port_host_byte_order();
-        packet.destination_port = udp_header->destination_port;
+        packet.destination_port = udp_header->get_destination_port_host_byte_order();
     } else if (protocol == IpProtocolNumberGRE) {
         if (!unpack_gre) {
             // We do not decode it automatically but we can report source and destination IPs for it to FNM processing
             return parser_code_t::success;
         }
 
-        if (local_pointer + sizeof(gre_packet_t) > end_pointer) {
+        if (local_pointer + sizeof(gre_header_t) > end_pointer) {
             return parser_code_t::memory_violation;
         }
 
-        gre_packet_t* gre_header = (gre_packet_t*)local_pointer;
+        gre_header_t* gre_header = (gre_header_t*)local_pointer;
-        gre_header->convert();
 
         // Current version of parser does not handle these special codes and we just fail parsing process
         // These flags may extend length of GRE header and current logic is not ready to decode any of them
-        if (gre_header->checksum != 0 || gre_header->reserved != 0 || gre_header->version != 0) {
+        if (gre_header->get_checksum() != 0 || gre_header->get_reserved() != 0 || gre_header->get_version() != 0) {
             return parser_code_t::broken_gre;
         }
 
+        uint16_t gre_nested_protocol = gre_header->get_protocol_type_host_byte_order();
+
         // We will try parsing IPv4 only for now
-        if (gre_header->protocol_type == IanaEthertypeIPv4) {
+        if (gre_nested_protocol == IanaEthertypeIPv4) {
-            local_pointer += sizeof(gre_packet_t);
+            local_pointer += sizeof(gre_header_t);
 
             // This function will override all fields in original packet structure by new fields
             bool read_length_from_ip_header = true;
@@ -265,8 +277,8 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
                                                                remaining_packet_length, packet, read_length_from_ip_header);
 
             return nested_packet_parse_result;
-        } else if (gre_header->protocol_type == IanaEthertypeERSPAN) {
+        } else if (gre_nested_protocol == IanaEthertypeERSPAN) {
-            local_pointer += sizeof(gre_packet_t);
+            local_pointer += sizeof(gre_header_t);
 
             // We need to calculate how much data we have after all parsed fields until end of packet to pass it to function below
             int remaining_packet_length = end_pointer - local_pointer;
@@ -294,16 +306,16 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
 }
 
 // Our own native function to convert IPv4 packet into simple_packet_t
-parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(uint8_t* pointer,
+parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(const uint8_t* pointer,
                                                              int length_before_sampling,
                                                              int captured_length,
                                                              simple_packet_t& packet,
                                                              bool read_packet_length_from_ip_header) {
     // We are using pointer copy because we are changing it
-    uint8_t* local_pointer = pointer;
+    const uint8_t* local_pointer = pointer;
 
     // It's very nice for doing checks
-    uint8_t* end_pointer = pointer + captured_length;
+    const uint8_t* end_pointer = pointer + captured_length;
 
 
     // Here we store IPv4 or IPv6 l4 protocol numbers
@@ -317,40 +329,36 @@ parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(uint8_t* pointer,
 
     ipv4_header_t* ipv4_header = (ipv4_header_t*)local_pointer;
 
-    // Populate IP specific options in packet structure before making any conversions, use network representation of
+    // Use network representation of addresses
-    // addresses
+    packet.src_ip = ipv4_header->get_source_ip_network_byte_order();
-    packet.src_ip = ipv4_header->source_ip;
+    packet.dst_ip = ipv4_header->get_destination_ip_network_byte_order();
-    packet.dst_ip = ipv4_header->destination_ip;
 
     packet.ip_protocol_version = 4;
 
-    // Convert all integers in IP header to little endian
+    packet.ttl              = ipv4_header->get_ttl();
-    ipv4_header->convert();
+    packet.ip_length        = ipv4_header->get_total_length_host_byte_order();
-
+    packet.ip_dont_fragment = ipv4_header->get_dont_fragment_flag();
-    packet.ttl              = ipv4_header->ttl;
-    packet.ip_length        = ipv4_header->total_length;
-    packet.ip_dont_fragment = ipv4_header->dont_fragment_flag;
 
     packet.ip_fragmented = ipv4_header->is_fragmented();
 
     // We keep these variables to maintain backward compatibility with parse_raw_packet_to_simple_packet_full()
-    packet.packet_payload_length      = length_before_sampling;
+    packet.captured_payload_length      = length_before_sampling;
-    packet.packet_payload_full_length = length_before_sampling;
+    packet.payload_full_length = length_before_sampling;
 
     // Pointer to payload
-    packet.packet_payload_pointer = (void*)pointer;
+    packet.payload_pointer = (void*)pointer;
 
-    protocol        = ipv4_header->protocol;
+    protocol        = ipv4_header->get_protocol();
     packet.protocol = protocol;
 
     if (read_packet_length_from_ip_header) {
-        packet.length = ipv4_header->total_length;
+        packet.length = ipv4_header->get_total_length_host_byte_order();
     } else {
         packet.length = length_before_sampling;
     }
 
     // Ignore all IP options and shift pointer to L3 payload
-    local_pointer += 4 * ipv4_header->ihl;
+    local_pointer += 4 * ipv4_header->get_ihl();
 
     if (protocol == IpProtocolNumberTCP) {
         if (local_pointer + sizeof(tcp_header_t) > end_pointer) {
@@ -358,14 +366,13 @@ parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(uint8_t* pointer,
         }
 
         tcp_header_t* tcp_header = (tcp_header_t*)local_pointer;
-        tcp_header->convert();
 
-        packet.source_port      = tcp_header->source_port;
+        packet.source_port      = tcp_header->get_source_port_host_byte_order();
-        packet.destination_port = tcp_header->destination_port;
+        packet.destination_port = tcp_header->get_destination_port_host_byte_order();
 
         // TODO: rework this code to use structs with bit fields
-        packet.flags = tcp_header->fin * 0x01 + tcp_header->syn * 0x02 + tcp_header->rst * 0x04 +
+        packet.flags = tcp_header->get_fin() * 0x01 + tcp_header->get_syn() * 0x02 + tcp_header->get_rst() * 0x04 +
-                       tcp_header->psh * 0x08 + tcp_header->ack * 0x10 + tcp_header->urg * 0x20;
+                       tcp_header->get_psh() * 0x08 + tcp_header->get_ack() * 0x10 + tcp_header->get_urg() * 0x20;
 
     } else if (protocol == IpProtocolNumberUDP) {
         if (local_pointer + sizeof(udp_header_t) > end_pointer) {
@@ -373,10 +380,9 @@ parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(uint8_t* pointer,
         }
 
         udp_header_t* udp_header = (udp_header_t*)local_pointer;
-        udp_header->convert();
 
-        packet.source_port      = udp_header->source_port;
+        packet.source_port      = udp_header->get_source_port_host_byte_order();
-        packet.destination_port = udp_header->destination_port;
+        packet.destination_port = udp_header->get_destination_port_host_byte_order();
     } else {
         // That's fine, it's not some known protocol but we can export basic information retrieved from IP packet
         return parser_code_t::success;

src/simple_packet_parser_ng.hpp

@@ -1,7 +1,7 @@
 #include "fastnetmon_simple_packet.hpp"
 #include "network_data_structures.hpp"
 
-network_data_stuctures::parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
+network_data_stuctures::parser_code_t parse_raw_packet_to_simple_packet_full_ng(const uint8_t* pointer,
                                                                                 int length_before_sampling,
                                                                                 int captured_length,
                                                                                 simple_packet_t& packet,
@@ -9,7 +9,7 @@ network_data_stuctures::parser_code_t parse_raw_packet_to_simple_packet_full_ng(
                                                                                 bool use_packet_length_from_wire);
 
 
-network_data_stuctures::parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(uint8_t* pointer,
+network_data_stuctures::parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(const uint8_t* pointer,
                                                                                      int length_before_sampling,
                                                                                      int captured_length,
                                                                                      simple_packet_t& packet,