1
0
Fork 0
mirror of https://github.com/pavel-odintsov/fastnetmon synced 2024-06-09 01:06:04 +02:00

Move pcap packet parser to separate library

This commit is contained in:
Pavel Odintsov 2015-07-22 14:16:13 +02:00
parent afd68857ff
commit 2a70662c2c
4 changed files with 42 additions and 413 deletions

View File

@ -87,6 +87,8 @@ endif()
# Our LPM library
add_library(patricia STATIC libpatricia/patricia.c)
add_library(fastnetmon_pcap_format STATIC fastnetmon_pcap_format.cpp)
# Our tools library
add_library(fast_library STATIC fast_library.cpp)
@ -159,6 +161,7 @@ if(Boost_FOUND)
endif()
target_link_libraries(fast_library patricia)
target_link_libraries(fast_library fastnetmon_pcap_format)
# Try to find ncurses librreary
find_package(Curses REQUIRED)
@ -215,6 +218,7 @@ endif()
# Our libs
target_link_libraries(fastnetmon patricia)
target_link_libraries(fastnetmon fastnetmon_pcap_format)
target_link_libraries(fastnetmon ipfix_rfc)
@ -233,6 +237,7 @@ if (BUILD_PLUGIN_RUNNER)
target_link_libraries(fastnetmon_plugin_runner ${CMAKE_THREAD_LIBS_INIT})
target_link_libraries(fastnetmon_plugin_runner patricia)
target_link_libraries(fastnetmon_plugin_runner fastnetmon_pcap_format)
target_link_libraries(fastnetmon_plugin_runner ${LOG4CPP_LIBRARY_PATH})
target_link_libraries(fastnetmon_plugin_runner fast_library)
@ -251,6 +256,9 @@ if (BUILD_PCAP_READER)
target_link_libraries(fastnetmon_pcap_reader fastnetmon_packet_parser)
target_link_libraries(fastnetmon_pcap_reader patricia)
target_link_libraries(fastnetmon_pcap_reader fastnetmon_pcap_format)
target_link_libraries(fastnetmon_pcap_reader fast_library)
target_link_libraries(fastnetmon_pcap_reader ${LOG4CPP_LIBRARY_PATH})
target_link_libraries(fastnetmon_pcap_reader netflow_plugin)

View File

@ -1,6 +1,15 @@
#ifndef FASTNETMON_PCAP_FORMAT_H
#define FASTNETMON_PCAP_FORMAT_H
#include <stdint.h>
#include <unistd.h>
#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
/*
pcap dump format:
global header: struct pcap_file_header
@ -37,4 +46,8 @@ struct fastnetmon_pcap_pkthdr {
uint32_t orig_len; /* actual length of packet */
};
typedef void (*pcap_packet_parser_callback)(char* buffer, uint32_t len);
int pcap_reader(const char* pcap_file_path, pcap_packet_parser_callback pcap_parse_packet_function_ptr);
#endif

View File

@ -45,8 +45,6 @@ log4cpp::Category& logger = log4cpp::Category::getRoot();
/* It's prototype for moc testing of FastNetMon, it's very useful for netflow or direct packet
* parsers debug */
void pcap_parse_packet(const char* flow_type, char* buffer, uint32_t len);
void init_logging() {
log4cpp::PatternLayout* layout = new log4cpp::PatternLayout();
layout->setConversionPattern("%d [%p] %m%n");
@ -59,69 +57,7 @@ void init_logging() {
logger.info("Logger initialized!");
}
int pcap_reader(const char* flow_type, const char* pcap_file_path) {
int filedesc = open(pcap_file_path, O_RDONLY);
if (filedesc <= 0) {
printf("Can't open dump file");
return -1;
}
struct fastnetmon_pcap_file_header pcap_header;
ssize_t file_header_readed_bytes = read(filedesc, &pcap_header, sizeof(struct fastnetmon_pcap_file_header));
if (file_header_readed_bytes != sizeof(struct fastnetmon_pcap_file_header)) {
printf("Can't read pcap file header");
}
// http://www.tcpdump.org/manpages/pcap-savefile.5.html
if (pcap_header.magic == 0xa1b2c3d4 or pcap_header.magic == 0xd4c3b2a1) {
// printf("Magic readed correctly\n");
} else {
printf("Magic in file header broken\n");
return -2;
}
// Buffer for packets
char packet_buffer[pcap_header.snaplen];
unsigned int read_packets = 0;
while (1) {
// printf("Start packet %d processing\n", read_packets);
struct fastnetmon_pcap_pkthdr pcap_packet_header;
ssize_t packet_header_readed_bytes =
read(filedesc, &pcap_packet_header, sizeof(struct fastnetmon_pcap_pkthdr));
if (packet_header_readed_bytes != sizeof(struct fastnetmon_pcap_pkthdr)) {
// We haven't any packets
break;
}
if (pcap_packet_header.incl_len > pcap_header.snaplen) {
printf("Please enlarge packet buffer! We got packet with size: %d but our buffer is %d "
"bytes\n",
pcap_packet_header.incl_len, pcap_header.snaplen);
return -4;
}
ssize_t packet_payload_readed_bytes = read(filedesc, packet_buffer, pcap_packet_header.incl_len);
if (pcap_packet_header.incl_len != packet_payload_readed_bytes) {
printf("I read packet header but can't read packet payload\n");
return -3;
}
// printf("packet payload read\n");
pcap_parse_packet(flow_type, packet_buffer, pcap_packet_header.incl_len);
// printf("Process packet %d\n", read_packets);
read_packets++;
}
printf("I correctly read %d packets from this dump\n", read_packets);
return 0;
}
void pcap_parse_packet(const char* flow_type, char* buffer, uint32_t len);
void my_fastnetmon_packet_handler(simple_packet& current_packet) {
std::cout << print_simple_packet(current_packet);
@ -130,7 +66,9 @@ void my_fastnetmon_packet_handler(simple_packet& current_packet) {
extern process_packet_pointer netflow_process_func_ptr;
extern process_packet_pointer sflow_process_func_ptr;
void pcap_parse_packet(const char* flow_type, char* buffer, uint32_t len) {
char* flow_type = NULL;
void pcap_parse_packet(char* buffer, uint32_t len) {
struct pfring_pkthdr packet_header;
memset(&packet_header, 0, sizeof(packet_header));
packet_header.len = len;
@ -167,6 +105,19 @@ void pcap_parse_packet(const char* flow_type, char* buffer, uint32_t len) {
sample.sourceIP.type = SFLADDRESSTYPE_IP_V4;
read_sflow_datagram(&sample);
} else if (strcmp(flow_type, "raw") == 0) {
// We do not need parsed data here
struct pfring_pkthdr packet_header;
memset(&packet_header, 0, sizeof(packet_header));
packet_header.len = payload_length;
packet_header.caplen = payload_length;
fastnetmon_parse_pkt((u_char*)buffer, &packet_header, 4, 1, 0);
char print_buffer[512];
fastnetmon_print_parsed_pkt(print_buffer, 512, (u_char*)buffer, &packet_header);
printf("%s", print_buffer);
} else {
printf("We do not support this flow type: %s\n", flow_type);
}
@ -176,10 +127,11 @@ int main(int argc, char** argv) {
init_logging();
if (argc != 3) {
printf("Please provide flow type: sflow or netflow and path to pcap dump\n");
printf("Please provide flow type: sflow, netflow or raw and path to pcap dump\n");
exit(1);
}
flow_type = argv[1];
printf("We will process file: %s as %s dump\n", argv[2], argv[1]);
pcap_reader(argv[1], argv[2]);
pcap_reader(argv[2], pcap_parse_packet);
}

View File

@ -1,344 +0,0 @@
// compile with: gcc -shared -o capturecallback.so -fPIC capturecallback.c
#include <stdio.h>
#include <stdlib.h>
#include <stdint.h>
#include <pthread.h>
#include <string.h>
#include <unistd.h>
#include <algorithm>
#include <vector>
#include <unordered_map>
#include <string>
#include "../../fastnetmon_types.h"
#include "../../fastnetmon_packet_parser.h"
#include "libndpi/ndpi_api.h"
// For correct compilation with g++
extern "C" {
void debug_printf(u_int32_t protocol, void *id_struct, ndpi_log_level_t log_level, const char *format, ...) {
va_list va_ap;
struct tm result;
char buf[8192], out_buf[8192];
char theDate[32];
const char *extra_msg = "";
time_t theTime = time(NULL);
va_start (va_ap, format);
/*
if(log_level == NDPI_LOG_ERROR)
extra_msg = "ERROR: ";
else if(log_level == NDPI_LOG_TRACE)
extra_msg = "TRACE: ";
else
extra_msg = "DEBUG: ";
*/
memset(buf, 0, sizeof(buf));
strftime(theDate, 32, "%d/%b/%Y %H:%M:%S", localtime_r(&theTime, &result) );
vsnprintf(buf, sizeof(buf)-1, format, va_ap);
snprintf(out_buf, sizeof(out_buf), "%s %s%s", theDate, extra_msg, buf);
printf("%s", out_buf);
fflush(stdout);
va_end(va_ap);
}
struct ndpi_detection_module_struct* my_ndpi_struct = NULL;
bool init_ndpi() {
u_int32_t detection_tick_resolution = 1000;
my_ndpi_struct = ndpi_init_detection_module(detection_tick_resolution, malloc, free, debug_printf);
if (my_ndpi_struct == NULL) {
printf("Can't init nDPI");
return false;
}
NDPI_PROTOCOL_BITMASK all;
// enable all protocols
NDPI_BITMASK_SET_ALL(all);
ndpi_set_protocol_detection_bitmask2(my_ndpi_struct, &all);
// allocate memory for id and flow tracking
uint32_t size_id_struct = ndpi_detection_get_sizeof_ndpi_id_struct();
uint32_t size_flow_struct = ndpi_detection_get_sizeof_ndpi_flow_struct();
// Load custom protocols
// ndpi_load_protocols_file(ndpi_thread_info[thread_id].ndpi_struct, _protoFilePath);
printf("nDPI started correctly\n");
}
/* Called once before processing packets. */
void firehose_start(); /* optional */
/* Called once after processing packets. */
void firehose_stop(); /* optional */
/*
* Process a packet received from a NIC.
*
* pciaddr: name of PCI device packet is received from
* data: packet payload (ethernet frame)
* length: payload length in bytes
*/
inline void firehose_packet(const char *pciaddr, char *data, int length);
/* Intel 82599 "Legacy" receive descriptor format.
* See Intel 82599 data sheet section 7.1.5.
* http://www.intel.com/content/dam/www/public/us/en/documents/datasheets/82599-10-gbe-controller-datasheet.pdf
*/
struct firehose_rdesc {
uint64_t address;
uint16_t length;
uint16_t cksum;
uint8_t status;
uint8_t errors;
uint16_t vlan;
} __attribute__((packed));
/* Traverse the hardware receive descriptor ring.
* Process each packet that is ready.
* Return the updated ring index.
*/
int firehose_callback_v1(const char *pciaddr,
char **packets,
struct firehose_rdesc *rxring,
int ring_size,
int index) {
while (rxring[index].status & 1) {
int next_index = (index + 1) & (ring_size-1);
__builtin_prefetch(packets[next_index]);
firehose_packet(pciaddr, packets[index], rxring[index].length);
rxring[index].status = 0; /* reset descriptor for reuse */
index = next_index;
}
return index;
}
uint64_t received_packets = 0;
void* speed_printer(void* ptr) {
while (1) {
uint64_t packets_before = received_packets;
sleep(1);
uint64_t packets_after = received_packets;
uint64_t pps = packets_after - packets_before;
printf("We process: %llu pps\n", (long long)pps);
}
}
// We will start speed printer
void firehose_start() {
init_ndpi();
pthread_t thread;
pthread_create(&thread, NULL, speed_printer, NULL);
pthread_detach(thread);
}
// https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash2.cpp
// 64-bit hash for 64-bit platforms
#define BIG_CONSTANT(x) (x##LLU)
uint64_t MurmurHash64A(const void* key, int len, uint64_t seed) {
const uint64_t m = BIG_CONSTANT(0xc6a4a7935bd1e995);
const int r = 47;
uint64_t h = seed ^ (len * m);
const uint64_t* data = (const uint64_t*)key;
const uint64_t* end = data + (len / 8);
while (data != end) {
uint64_t k = *data++;
k *= m;
k ^= k >> r;
k *= m;
h ^= k;
h *= m;
}
const unsigned char* data2 = (const unsigned char*)data;
switch (len & 7) {
case 7:
h ^= uint64_t(data2[6]) << 48;
case 6:
h ^= uint64_t(data2[5]) << 40;
case 5:
h ^= uint64_t(data2[4]) << 32;
case 4:
h ^= uint64_t(data2[3]) << 24;
case 3:
h ^= uint64_t(data2[2]) << 16;
case 2:
h ^= uint64_t(data2[1]) << 8;
case 1:
h ^= uint64_t(data2[0]);
h *= m;
};
h ^= h >> r;
h *= m;
h ^= h >> r;
return h;
}
class conntrack_hash_struct_for_simple_packet_t {
public:
uint32_t src_ip;
uint32_t dst_ip;
uint16_t source_port;
uint16_t destination_port;
unsigned int protocol;
bool operator==(const conntrack_hash_struct_for_simple_packet_t& rhs) {
return memcmp(this, &rhs, sizeof(conntrack_hash_struct_for_simple_packet_t)) == 0;
}
};
// Copy and paste from netmap module
inline bool parse_raw_packet_to_simple_packet(u_char* buffer, int len, simple_packet& packet) {
struct pfring_pkthdr packet_header;
memset(&packet_header, 0, sizeof(packet_header));
packet_header.len = len;
packet_header.caplen = len;
// We do not calculate timestamps because timestamping is very CPU intensive operation:
// https://github.com/ntop/PF_RING/issues/9
u_int8_t timestamp = 0;
u_int8_t add_hash = 0;
fastnetmon_parse_pkt((u_char*)buffer, &packet_header, 4, timestamp, add_hash);
// char print_buffer[512];
// fastnetmon_print_parsed_pkt(print_buffer, 512, (u_char*)buffer, &packet_header);
// logger.info("%s", print_buffer);
if (packet_header.extended_hdr.parsed_pkt.ip_version != 4 && packet_header.extended_hdr.parsed_pkt.ip_version != 6) {
return false;
}
// We need this for deep packet inspection
packet.packet_payload_length = len;
packet.packet_payload_pointer = (void*)buffer;
packet.ip_protocol_version = packet_header.extended_hdr.parsed_pkt.ip_version;
if (packet.ip_protocol_version == 4) {
// IPv4
/* PF_RING stores data in host byte order but we use network byte order */
packet.src_ip = htonl(packet_header.extended_hdr.parsed_pkt.ip_src.v4);
packet.dst_ip = htonl(packet_header.extended_hdr.parsed_pkt.ip_dst.v4);
} else {
// IPv6
memcpy(packet.src_ipv6.s6_addr, packet_header.extended_hdr.parsed_pkt.ip_src.v6.s6_addr, 16);
memcpy(packet.dst_ipv6.s6_addr, packet_header.extended_hdr.parsed_pkt.ip_dst.v6.s6_addr, 16);
}
packet.source_port = packet_header.extended_hdr.parsed_pkt.l4_src_port;
packet.destination_port = packet_header.extended_hdr.parsed_pkt.l4_dst_port;
packet.length = packet_header.len;
packet.protocol = packet_header.extended_hdr.parsed_pkt.l3_proto;
packet.ts = packet_header.ts;
packet.ip_fragmented = packet_header.extended_hdr.parsed_pkt.ip_fragmented;
packet.ttl = packet_header.extended_hdr.parsed_pkt.ip_ttl;
// Copy flags from PF_RING header to our pseudo header
if (packet.protocol == IPPROTO_TCP) {
packet.flags = packet_header.extended_hdr.parsed_pkt.tcp.flags;
} else {
packet.flags = 0;
}
return true;
}
bool convert_simple_packet_toconntrack_hash_struct(simple_packet& packet, conntrack_hash_struct_for_simple_packet_t& conntrack_struct) {
conntrack_struct.src_ip = packet.src_ip;
conntrack_struct.dst_ip = packet.dst_ip;
conntrack_struct.protocol = packet.protocol;
conntrack_struct.source_port = packet.source_port;
conntrack_struct.destination_port = packet.destination_port;
}
typedef std::unordered_map<uint64_t, unsigned int> my_connection_tracking_storage_t;
my_connection_tracking_storage_t my_connection_tracking_storage;
void firehose_packet(const char *pciaddr, char *data, int length) {
// Put packet to the cache
simple_packet current_packet;
parse_raw_packet_to_simple_packet((u_char*)data, length, current_packet);
conntrack_hash_struct_for_simple_packet_t conntrack_structure;
convert_simple_packet_toconntrack_hash_struct(current_packet, conntrack_structure);
unsigned int seed = 13;
uint64_t conntrack_hash = MurmurHash64A(&conntrack_structure, sizeof(conntrack_structure), seed);
// printf("Hash: %llu", conntrack_hash);
my_connection_tracking_storage_t::iterator itr = my_connection_tracking_storage.find(conntrack_hash);
if (itr == my_connection_tracking_storage.end()) {
my_connection_tracking_storage[ conntrack_hash ] = 123;
//printf("Initiate new connection\n");
} else {
//printf("Found this connection\n");
}
/*
struct ndpi_id_struct *src, *dst;
struct ndpi_flow_struct *flow;
uint32_t current_tickt = 0 ;
uint8_t* iph = (uint8_t*)(&data[packet_header.extended_hdr.parsed_pkt.offset.l3_offset]);
unsigned int ipsize = packet_header.len;
ndpi_protocol detected_protocol = ndpi_detection_process_packet(my_ndpi_struct, flow, iph, ipsize, current_tickt, src, dst);
if (detected_protocol.protocol != NDPI_PROTOCOL_UNKNOWN) {
printf("Can't detect protocol");
return;
}
// printf("protocol: %s\n", ndpi_get_proto_name(my_ndpi_struct, flow->detected_protocol));
*/
/*
char print_buffer[512];
fastnetmon_print_parsed_pkt(print_buffer, 512, (u_char*)data, &packet_header);
printf("packet: %s\n", print_buffer);
*/
__sync_fetch_and_add(&received_packets, 1);
//printf("Got packet with %d bytes.\n", length);
}
}