wanderer/ fastnetmon-rewritten
1
0
Fork 0
mirror of https://github.com/pavel-odintsov/fastnetmon synced 2024-05-04 02:36:17 +02:00
Code Issues Releases Activity

Poretd parser to new logic which does not alter buffer content during parsing process

This commit is contained in:
Pavel Odintsov 2023-06-07 13:10:36 +01:00
parent e019b46802
commit 0f0ba0d063
5 changed files with 114 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/fast_library.cpp
View File

@ -1324,8 +1324,8 @@ bool read_simple_packet(uint8_t* buffer, size_t buffer_length, simple_packet_t&
packet.ip_fragmented = root.getIpFragmented();
packet.ts.tv_sec = root.getTsSec();
packet.ts.tv_usec = root.getTsMsec();
packet.packet_payload_length = root.getPacketPayloadLength();
packet.packet_payload_full_length = root.getPacketPayloadFullLength();
packet.captured_payload_length = root.getPacketPayloadLength();
packet.payload_full_length = root.getPacketPayloadFullLength();
packet.packet_direction = (direction_t)root.getPacketDirection();
packet.source = (source_t)root.getSource();
} catch (kj::Exception e) {
@ -1361,8 +1361,8 @@ bool write_simple_packet(int fd, simple_packet_t& packet, bool populate_ipv6) {
capnp_packet.setIpFragmented(packet.ip_fragmented);
capnp_packet.setTsSec(packet.ts.tv_sec);
capnp_packet.setTsMsec(packet.ts.tv_usec);
capnp_packet.setPacketPayloadLength(packet.packet_payload_length);
capnp_packet.setPacketPayloadFullLength(packet.packet_payload_full_length);
capnp_packet.setPacketPayloadLength(packet.captured_payload_length);
capnp_packet.setPacketPayloadFullLength(packet.payload_full_length);
capnp_packet.setPacketDirection(packet.packet_direction);
capnp_packet.setSource(packet.source);
capnp_packet.setSrcAsn(packet.src_asn);

16
src/fastnetmon_simple_packet.hpp
View File

@ -38,6 +38,9 @@ class simple_packet_t {
in6_addr src_ipv6{};
in6_addr dst_ipv6{};
uint8_t source_mac[6]{};
uint8_t destination_mac[6]{};
// ASNs
uint32_t src_asn = 0;
uint32_t dst_asn = 0;
@ -79,17 +82,22 @@ class simple_packet_t {
// If IP has don't fragment flag
bool ip_dont_fragment = false;
// Fragment offset in bytes when fragmentation involved
uint16_t ip_fragment_offset = 0;
// Time when we actually received this packet, we use quite rough and inaccurate but very fast time source for it
time_t arrival_time = 0;
// Timestamp of packet as reported by Netflow or IPFIX agent on device, it may be very inaccurate as nobody cares about time on equipment
struct timeval ts = { 0, 0 };
void* packet_payload_pointer = nullptr;
int32_t packet_payload_length = 0;
void* payload_pointer = nullptr;
// In case of cropped packets we use this field
uint32_t packet_payload_full_length = 0;
// Part of packet we captured from wire. It may not be full length of packet
int32_t captured_payload_length = 0;
// Full length of packet we observed. It may be larger then packet_captured_payload_length in case of cropped mirror or sFlow traffic
uint32_t payload_full_length = 0;
// Forwarding status
forwarding_status_t forwarding_status = forwarding_status_t::unknown;

6
src/sflow_plugin/sflow_collector.cpp
View File

@ -421,9 +421,9 @@ bool process_sflow_flow_sample(uint8_t* data_pointer,
}
// Pass pointer to raw header to FastNetMon processing functions
packet.packet_payload_pointer = header_payload_pointer;
packet.packet_payload_full_length = sflow_raw_protocol_header.frame_length_before_sampling;
packet.packet_payload_length = sflow_raw_protocol_header.header_size;
packet.payload_pointer = header_payload_pointer;
packet.payload_full_length = sflow_raw_protocol_header.frame_length_before_sampling;
packet.captured_payload_length = sflow_raw_protocol_header.header_size;
packet.sample_ratio = sflow_sample_header_unified_accessor.sampling_rate;

180
src/simple_packet_parser_ng.cpp
View File

@ -2,6 +2,8 @@
#include "all_logcpp_libraries.hpp"
#include "network_data_structures.hpp"
#include <algorithm>
#include <iterator>
#include <cstring>
using namespace network_data_stuctures;
@ -11,18 +13,17 @@ using namespace network_data_stuctures;
bool decode_mpls = false;
// Our own native function to convert wire packet into simple_packet_t
// TODO: development is going here, we still need to add number of options here
parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
parser_code_t parse_raw_packet_to_simple_packet_full_ng(const uint8_t* pointer,
int length_before_sampling,
int captured_length,
simple_packet_t& packet,
bool unpack_gre,
bool read_packet_length_from_ip_header) {
// We are using pointer copy because we are changing it
uint8_t* local_pointer = pointer;
const uint8_t* local_pointer = pointer;
// It's very nice for doing checks
uint8_t* end_pointer = pointer + captured_length;
const uint8_t* end_pointer = pointer + captured_length;
// Return error if it shorter then ethernet headers
if (local_pointer + sizeof(ethernet_header_t) > end_pointer) {
@ -30,29 +31,35 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
}
ethernet_header_t* ethernet_header = (ethernet_header_t*)local_pointer;
ethernet_header->convert();
// Copy Ethernet MAC addresses to packet structure using native C++ approach to avoid touching memory with memcpy
std::copy(std::begin(ethernet_header->source_mac), std::end(ethernet_header->source_mac), std::begin(packet.source_mac));
std::copy(std::begin(ethernet_header->destination_mac), std::end(ethernet_header->destination_mac), std::begin(packet.destination_mac));
local_pointer += sizeof(ethernet_header_t);
if (ethernet_header->ethertype == IanaEthertypeVLAN) {
// Copy ethertype as we may need to change it below
uint16_t ethertype = ethernet_header->get_ethertype_host_byte_order();
if (ethertype == IanaEthertypeVLAN) {
// Return error if it shorter then vlan header
if (local_pointer + sizeof(ethernet_vlan_header_t) > end_pointer) {
return parser_code_t::memory_violation;
}
ethernet_vlan_header_t* ethernet_vlan_header = (ethernet_vlan_header_t*)local_pointer;
ethernet_vlan_header->convert();
packet.vlan = ethernet_vlan_header->vlan_id;
packet.vlan = ethernet_vlan_header->get_vlan_id_host_byte_order();
local_pointer += sizeof(ethernet_vlan_header_t);
// Change ethernet ethertype to vlan's ethertype
ethernet_header->ethertype = ethernet_vlan_header->ethertype;
// Change ethertype to vlan's ethertype
ethertype = ethernet_vlan_header->get_ethertype_host_byte_order();
}
if (decode_mpls) {
if (ethernet_header->ethertype == IanaEthertypeMPLS_unicast) {
if (ethertype == IanaEthertypeMPLS_unicast) {
REPEAT_MPLS_STRIP:
if (local_pointer + sizeof(mpls_label_t) > end_pointer) {
return parser_code_t::memory_violation;
@ -75,7 +82,7 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
// Here we store IPv4 or IPv6 l4 protocol numbers
uint8_t protocol = 0;
if (ethernet_header->ethertype == IanaEthertypeIPv4) {
if (ethertype == IanaEthertypeIPv4) {
// Return error if pointer is shorter then IP header
if (local_pointer + sizeof(ipv4_header_t) > end_pointer) {
return parser_code_t::memory_violation;
@ -83,46 +90,54 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
ipv4_header_t* ipv4_header = (ipv4_header_t*)local_pointer;
// Populate IP specific options in packet structure before making any conversions, use network representation of
// addresses
packet.src_ip = ipv4_header->source_ip;
packet.dst_ip = ipv4_header->destination_ip;
// Use network representation of addresses
packet.src_ip = ipv4_header->get_source_ip_network_byte_order();
packet.dst_ip = ipv4_header->get_destination_ip_network_byte_order();
packet.ip_protocol_version = 4;
// Convert all integers in IP header to little endian
ipv4_header->convert();
packet.ttl = ipv4_header->ttl;
packet.ttl = ipv4_header->get_ttl();
// We need this specific field for Flow Spec mitigation mode
packet.ip_length = ipv4_header->total_length;
packet.ip_length = ipv4_header->get_total_length_host_byte_order();
packet.ip_dont_fragment = ipv4_header->dont_fragment_flag;
packet.ip_dont_fragment = ipv4_header->get_dont_fragment_flag();
packet.ip_fragmented = ipv4_header->is_fragmented();
packet.ip_more_fragments = ipv4_header->more_fragments_flag;
packet.ip_more_fragments = ipv4_header->get_more_fragments_flag();
// We must use special function to recover value in a format useable for our consumption
// We must not read this field directly
packet.ip_fragment_offset = ipv4_header->get_fragment_offset_bytes();
// We keep these variables to maintain backward compatibility with parse_raw_packet_to_simple_packet_full()
packet.packet_payload_length = length_before_sampling;
packet.packet_payload_full_length = length_before_sampling;
packet.captured_payload_length = length_before_sampling;
packet.payload_full_length = length_before_sampling;
// Pointer to payload
packet.packet_payload_pointer = (void*)pointer;
packet.payload_pointer = (void*)pointer;
protocol = ipv4_header->protocol;
protocol = ipv4_header->get_protocol();
packet.protocol = protocol;
if (read_packet_length_from_ip_header) {
packet.length = ipv4_header->total_length;
packet.length = ipv4_header->get_total_length_host_byte_order();
} else {
packet.length = length_before_sampling;
}
// We need to handle fragmented traffic. In case of IPv4 fragmentation only first packet carries UDP / TCP / other headers
// and consequent packets simply lack of this information and we know only protocol for them.
// We can consequent packets by non zero fragment_offset
if (ipv4_header->get_fragment_offset_bytes() != 0) {
// The best we can do it so stop processing here and report success
return parser_code_t::success;
}
// Ignore all IP options and shift pointer to L3 payload
local_pointer += 4 * ipv4_header->ihl;
} else if (ethernet_header->ethertype == IanaEthertypeIPv6) {
local_pointer += 4 * ipv4_header->get_ihl();
} else if (ethertype == IanaEthertypeIPv6) {
// Return error if pointer is shorter then IP header
if (local_pointer + sizeof(ipv6_header_t) > end_pointer) {
return parser_code_t::memory_violation;
@ -130,31 +145,29 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
ipv6_header_t* ipv6_header = (ipv6_header_t*)local_pointer;
// Convert all integers in IP header to little endian
ipv6_header->convert();
// TODO: we may use std::copy for it to avoid touching memory with memcpy
memcpy(&packet.src_ipv6, ipv6_header->source_address, sizeof(packet.src_ipv6));
memcpy(&packet.dst_ipv6, ipv6_header->destination_address, sizeof(packet.dst_ipv6));
packet.ip_protocol_version = 6;
packet.ttl = ipv6_header->hop_limit;
packet.ttl = ipv6_header->get_hop_limit();
// We need this specific field for Flow Spec mitigation mode
packet.ip_length = ipv6_header->payload_length;
packet.ip_length = ipv6_header->get_payload_length();
// We keep these variables to maintain backward compatibility with parse_raw_packet_to_simple_packet_full()
packet.packet_payload_length = length_before_sampling;
packet.packet_payload_full_length = length_before_sampling;
packet.captured_payload_length = length_before_sampling;
packet.payload_full_length = length_before_sampling;
// Pointer to payload
packet.packet_payload_pointer = (void*)pointer;
packet.payload_pointer = (void*)pointer;
protocol = ipv6_header->next_header;
protocol = ipv6_header->get_next_header();
packet.protocol = protocol;
if (read_packet_length_from_ip_header) {
packet.length = ipv6_header->payload_length;
packet.length = ipv6_header->get_payload_length();
} else {
packet.length = length_before_sampling;
}
@ -183,12 +196,12 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
if (protocol == IpProtocolNumberIPV6_FRAG) {
ipv6_extension_header_fragment_t* ipv6_extension_header_fragment = (ipv6_extension_header_fragment_t*)local_pointer;
ipv6_extension_header_fragment->convert();
// If we received this header then we assume that packet was fragmented
packet.ip_fragmented = true;
packet.ip_more_fragments = ipv6_extension_header_fragment->more_fragments;
packet.ip_more_fragments = ipv6_extension_header_fragment->get_more_fragments();
packet.ip_fragment_offset = ipv6_extension_header_fragment->get_fragment_offset_bytes();
// We stop processing here as I believe that it's enough to know that this traffic was fragmented
// We do not parse nested protocol in this case at all
@ -199,7 +212,7 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
return parser_code_t::no_ipv6_options_support;
}
} else if (ethernet_header->ethertype == IanaEthertypeARP) {
} else if (ethertype == IanaEthertypeARP) {
// it's not parser error of course but we need to have visibility about this case
return parser_code_t::arp;
} else {
@ -212,14 +225,13 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
}
tcp_header_t* tcp_header = (tcp_header_t*)local_pointer;
tcp_header->convert();
packet.source_port = tcp_header->source_port;
packet.destination_port = tcp_header->destination_port;
packet.source_port = tcp_header->get_source_port_host_byte_order();
packet.destination_port = tcp_header->get_destination_port_host_byte_order();
// TODO: rework this code to use structs with bit fields
packet.flags = tcp_header->fin * 0x01 + tcp_header->syn * 0x02 + tcp_header->rst * 0x04 +
tcp_header->psh * 0x08 + tcp_header->ack * 0x10 + tcp_header->urg * 0x20;
packet.flags = tcp_header->get_fin() * 0x01 + tcp_header->get_syn() * 0x02 + tcp_header->get_rst() * 0x04 +
tcp_header->get_psh() * 0x08 + tcp_header->get_ack() * 0x10 + tcp_header->get_urg() * 0x20;
} else if (protocol == IpProtocolNumberUDP) {
if (local_pointer + sizeof(udp_header_t) > end_pointer) {
@ -227,32 +239,32 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
}
udp_header_t* udp_header = (udp_header_t*)local_pointer;
udp_header->convert();
packet.source_port = udp_header->source_port;
packet.destination_port = udp_header->destination_port;
packet.source_port = udp_header->get_source_port_host_byte_order();
packet.destination_port = udp_header->get_destination_port_host_byte_order();
} else if (protocol == IpProtocolNumberGRE) {
if (!unpack_gre) {
// We do not decode it automatically but we can report source and destination IPs for it to FNM processing
return parser_code_t::success;
}
if (local_pointer + sizeof(gre_packet_t) > end_pointer) {
if (local_pointer + sizeof(gre_header_t) > end_pointer) {
return parser_code_t::memory_violation;
}
gre_packet_t* gre_header = (gre_packet_t*)local_pointer;
gre_header->convert();
gre_header_t* gre_header = (gre_header_t*)local_pointer;
// Current version of parser does not handle these special codes and we just fail parsing process
// These flags may extend length of GRE header and current logic is not ready to decode any of them
if (gre_header->checksum != 0 || gre_header->reserved != 0 || gre_header->version != 0) {
if (gre_header->get_checksum() != 0 || gre_header->get_reserved() != 0 || gre_header->get_version() != 0) {
return parser_code_t::broken_gre;
}
uint16_t gre_nested_protocol = gre_header->get_protocol_type_host_byte_order();
// We will try parsing IPv4 only for now
if (gre_header->protocol_type == IanaEthertypeIPv4) {
local_pointer += sizeof(gre_packet_t);
if (gre_nested_protocol == IanaEthertypeIPv4) {
local_pointer += sizeof(gre_header_t);
// This function will override all fields in original packet structure by new fields
bool read_length_from_ip_header = true;
@ -265,8 +277,8 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
remaining_packet_length, packet, read_length_from_ip_header);
return nested_packet_parse_result;
} else if (gre_header->protocol_type == IanaEthertypeERSPAN) {
local_pointer += sizeof(gre_packet_t);
} else if (gre_nested_protocol == IanaEthertypeERSPAN) {
local_pointer += sizeof(gre_header_t);
// We need to calculate how much data we have after all parsed fields until end of packet to pass it to function below
int remaining_packet_length = end_pointer - local_pointer;
@ -294,16 +306,16 @@ parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
}
// Our own native function to convert IPv4 packet into simple_packet_t
parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(uint8_t* pointer,
parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(const uint8_t* pointer,
int length_before_sampling,
int captured_length,
simple_packet_t& packet,
bool read_packet_length_from_ip_header) {
// We are using pointer copy because we are changing it
uint8_t* local_pointer = pointer;
const uint8_t* local_pointer = pointer;
// It's very nice for doing checks
uint8_t* end_pointer = pointer + captured_length;
const uint8_t* end_pointer = pointer + captured_length;
// Here we store IPv4 or IPv6 l4 protocol numbers
@ -317,40 +329,36 @@ parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(uint8_t* pointer,
ipv4_header_t* ipv4_header = (ipv4_header_t*)local_pointer;
// Populate IP specific options in packet structure before making any conversions, use network representation of
// addresses
packet.src_ip = ipv4_header->source_ip;
packet.dst_ip = ipv4_header->destination_ip;
// Use network representation of addresses
packet.src_ip = ipv4_header->get_source_ip_network_byte_order();
packet.dst_ip = ipv4_header->get_destination_ip_network_byte_order();
packet.ip_protocol_version = 4;
// Convert all integers in IP header to little endian
ipv4_header->convert();
packet.ttl = ipv4_header->ttl;
packet.ip_length = ipv4_header->total_length;
packet.ip_dont_fragment = ipv4_header->dont_fragment_flag;
packet.ttl = ipv4_header->get_ttl();
packet.ip_length = ipv4_header->get_total_length_host_byte_order();
packet.ip_dont_fragment = ipv4_header->get_dont_fragment_flag();
packet.ip_fragmented = ipv4_header->is_fragmented();
// We keep these variables to maintain backward compatibility with parse_raw_packet_to_simple_packet_full()
packet.packet_payload_length = length_before_sampling;
packet.packet_payload_full_length = length_before_sampling;
packet.captured_payload_length = length_before_sampling;
packet.payload_full_length = length_before_sampling;
// Pointer to payload
packet.packet_payload_pointer = (void*)pointer;
packet.payload_pointer = (void*)pointer;
protocol = ipv4_header->protocol;
protocol = ipv4_header->get_protocol();
packet.protocol = protocol;
if (read_packet_length_from_ip_header) {
packet.length = ipv4_header->total_length;
packet.length = ipv4_header->get_total_length_host_byte_order();
} else {
packet.length = length_before_sampling;
}
// Ignore all IP options and shift pointer to L3 payload
local_pointer += 4 * ipv4_header->ihl;
local_pointer += 4 * ipv4_header->get_ihl();
if (protocol == IpProtocolNumberTCP) {
if (local_pointer + sizeof(tcp_header_t) > end_pointer) {
@ -358,14 +366,13 @@ parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(uint8_t* pointer,
}
tcp_header_t* tcp_header = (tcp_header_t*)local_pointer;
tcp_header->convert();
packet.source_port = tcp_header->source_port;
packet.destination_port = tcp_header->destination_port;
packet.source_port = tcp_header->get_source_port_host_byte_order();
packet.destination_port = tcp_header->get_destination_port_host_byte_order();
// TODO: rework this code to use structs with bit fields
packet.flags = tcp_header->fin * 0x01 + tcp_header->syn * 0x02 + tcp_header->rst * 0x04 +
tcp_header->psh * 0x08 + tcp_header->ack * 0x10 + tcp_header->urg * 0x20;
packet.flags = tcp_header->get_fin() * 0x01 + tcp_header->get_syn() * 0x02 + tcp_header->get_rst() * 0x04 +
tcp_header->get_psh() * 0x08 + tcp_header->get_ack() * 0x10 + tcp_header->get_urg() * 0x20;
} else if (protocol == IpProtocolNumberUDP) {
if (local_pointer + sizeof(udp_header_t) > end_pointer) {
@ -373,10 +380,9 @@ parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(uint8_t* pointer,
}
udp_header_t* udp_header = (udp_header_t*)local_pointer;
udp_header->convert();
packet.source_port = udp_header->source_port;
packet.destination_port = udp_header->destination_port;
packet.source_port = udp_header->get_source_port_host_byte_order();
packet.destination_port = udp_header->get_destination_port_host_byte_order();
} else {
// That's fine, it's not some known protocol but we can export basic information retrieved from IP packet
return parser_code_t::success;

4
src/simple_packet_parser_ng.hpp
View File

@ -1,7 +1,7 @@
#include "fastnetmon_simple_packet.hpp"
#include "network_data_structures.hpp"
network_data_stuctures::parser_code_t parse_raw_packet_to_simple_packet_full_ng(uint8_t* pointer,
network_data_stuctures::parser_code_t parse_raw_packet_to_simple_packet_full_ng(const uint8_t* pointer,
int length_before_sampling,
int captured_length,
simple_packet_t& packet,
@ -9,7 +9,7 @@ network_data_stuctures::parser_code_t parse_raw_packet_to_simple_packet_full_ng(
bool use_packet_length_from_wire);
network_data_stuctures::parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(uint8_t* pointer,
network_data_stuctures::parser_code_t parse_raw_ipv4_packet_to_simple_packet_full_ng(const uint8_t* pointer,
int length_before_sampling,
int captured_length,
simple_packet_t& packet,