Merge branch 'master' of https://github.com/FastVPSEestiOu/fastnetmon

docs/FINE_TUNING.md

@@ -75,6 +75,26 @@ Ninja use all CPUs for build process:
 4  [||||||||||||||||||||||||||||||||||||||||||||||100.0%]
 ```
 
+Build script for reading Netflow (v5, v9, ipfix) data from pcap dump:
+```bash
+cmake .. -DBUILD_PCAP_READER=ON
+```
+
+Run pcap data:
+```bash
+./fastnetmon_pcap_reader dump.pcap
+```
+
+Build script for running packet capture plugins without analyzer backend:
+```bash
+cmake .. -DBUILD_PLUGIN_RUNNER=ON
+```
+
+Examples for different plugins (plugin name could be netflow, netmap, sflow, pfring, pcap):
+```bash
+./fastnetmon_plugin_runner netflow
+```
+
 Performance tuning:
 - Do not use short prefixes (lesser then /24)
 - Do not use extremely big prefixes (/8, /16) because memory consumption will be very big

src/CMakeLists.txt

@@ -126,6 +126,8 @@ if(Boost_FOUND)
     target_link_libraries(fast_library ${Boost_LIBRARIES})
 endif()
 
+target_link_libraries(fast_library patricia)
+
 # Try to find ncurses librreary
 find_package(Curses REQUIRED)
 
@@ -177,17 +179,40 @@ target_link_libraries(fastnetmon ipfix_rfc)
 # Link to our functions
 target_link_libraries(fastnetmon fast_library)
 
-# Our plugins
-target_link_libraries(fastnetmon sflow_plugin)
-
 if (ENABLE_PFRING_SUPPORT)
 target_link_libraries(fastnetmon pfring_plugin)
 endif()
 
-target_link_libraries(fastnetmon netflow_plugin)
-target_link_libraries(fastnetmon pcap_plugin)
-target_link_libraries(fastnetmon example_plugin)
-target_link_libraries(fastnetmon netmap_plugin)
+target_link_libraries(fastnetmon sflow_plugin netflow_plugin pcap_plugin example_plugin netmap_plugin)
+
+# cmake .. -DBUILD_PLUGIN_RUNNER=ON
+if (BUILD_PLUGIN_RUNNER)
+    add_executable(fastnetmon_plugin_runner plugin_runner.cpp)
+
+    target_link_libraries(fastnetmon_plugin_runner patricia)
+    target_link_libraries(fastnetmon_plugin_runner ${LOG4CPP_LIBRARY_PATH})
+    target_link_libraries(fastnetmon_plugin_runner fast_library)
+
+    # Add all plugins
+    target_link_libraries(fastnetmon_plugin_runner sflow_plugin netflow_plugin pcap_plugin example_plugin netmap_plugin)
+
+    if (ENABLE_PFRING_SUPPORT)
+        target_link_libraries(fastnetmon_plugin_runner ${PFRING_LIBRARIES})
+        target_link_libraries(fastnetmon_plugin_runner pfring_plugin)
+    endif()
+endif()
+
+# cmake .. -DBUILD_PCAP_READER=ON
+if (BUILD_PCAP_READER)
+    add_executable(fastnetmon_pcap_reader pcap_reader.cpp)
+
+    target_link_libraries(fastnetmon_pcap_reader pcap)
+    target_link_libraries(fastnetmon_pcap_reader fastnetmon_packet_parser)
+    target_link_libraries(fastnetmon_pcap_reader patricia)
+    target_link_libraries(fastnetmon_pcap_reader fast_library)
+    target_link_libraries(fastnetmon_pcap_reader ${LOG4CPP_LIBRARY_PATH})
+    target_link_libraries(fastnetmon_pcap_reader netflow_plugin)    
+endif()
 
 install(TARGETS fastnetmon DESTINATION bin)
 install(TARGETS fastnetmon_client DESTINATION bin)

src/build_plugin_tester.sh

@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-
-g++ ipfix_rfc.cpp -c -oipfix_rfc.o
-g++ fast_library.cpp -c -ofast_library.o
-g++ fastnetmon_packet_parser.c -c -o fastnetmon_packet_parser.o
-
-g++ netflow_plugin/netflow_collector.cpp -c -onetflow_collector.o
-g++ sflow_plugin/sflow_collector.cpp -c -osflow_collector.o
-g++ pcap_plugin/pcap_collector.cpp -c  -opcap_collector.o
-g++ pfring_plugin/pfring_collector.cpp -c -opfring_collector.o -I/opt/pf_ring/include 
-g++ netmap_plugin/netmap_collector.cpp -c -onetmap_collector.o -Inetmap_plugin/netmap_includes
-
-g++ plugin_runner.cpp -lnuma -lpcap -llog4cpp ipfix_rfc.o fast_library.o netflow_collector.o sflow_collector.o pcap_collector.o fastnetmon_packet_parser.o netmap_collector.o pfring_collector.o -oplugin_tester -I/opt/pf_ring/include -lpfring -lpthread -L/opt/pf_ring/lib -lboost_regex -lboost_system -lboost_thread
-rm -f netflow_collector.o ipfix_rfc.o sflow_collector.o pcap_collector.o netmap_collector.o fastnetmon_packet_parser.o
-
-

src/fastnetmon_install.pl

@@ -82,9 +82,18 @@ sub install {
 
     if ($distro_type eq 'debian') {
         `apt-get update`;
-        my @debian_packages_for_pfring = ('build-essential', 'bison', 'flex', "linux-headers-$kernel_version",
+        my @debian_packages_for_pfring = ('build-essential', 'bison', 'flex',
             'libnuma-dev', 'wget', 'tar', 'make', 'dpkg-dev', 'dkms', 'debhelper');
+   
+        my $kernel_headers_package_name = "linux-headers-$kernel_version";
+  
+        if ($appliance_name eq 'vyos') { 
+            # VyOS uses another name for package for building kernel modules
+            $kernel_headers_package_name = 'linux-vyatta-kbuild';
+        }
 
+        push @debian_packages_for_pfring, $kernel_headers_package_name;
+    
         # We install one package per apt-get call because installing multiple packages in one time could fail of one
         # pacakge broken
         for my $package (@debian_packages_for_pfring) {
@@ -94,6 +103,12 @@ sub install {
                 print "Package '$package' install failed with code $?\n"
             }  
         }
+
+
+        if ($appliance_name eq 'vyos') {
+            # By default we waven't this symlink and should add it manually
+            `ln -s /usr/src/linux-image/debian/build/build-amd64-none-amd64-vyos/ /lib/modules/$kernel_version/build`;
+        }
     } elsif ($distro_type eq 'centos') {
         my $kernel_package_name = 'kernel-devel';
 

src/netflow_plugin/netflow.h

@@ -89,7 +89,11 @@ struct NF1_FLOW {
 struct NF5_HEADER {
     struct NF_HEADER_COMMON c;
     u_int32_t uptime_ms, time_sec, time_nanosec, flow_sequence;
-    u_int8_t engine_type, engine_id, reserved1, reserved2;
+    u_int8_t engine_type, engine_id;
+    // "First two bits hold the sampling mode; remaining 14 bits hold value of sampling interval"
+    // accoring to https://www.plixer.com/support/netflow_v5.html
+    // http://www.cisco.com/c/en/us/td/docs/net_mgmt/netflow_collection_engine/3-6/user/guide/format.html 
+    u_int16_t sampling_rate;
 } __packed;
 struct NF5_FLOW {
     u_int32_t src_ip, dest_ip, nexthop_ip;

src/netflow_plugin/netflow_collector.cpp

@@ -863,6 +863,10 @@ void process_netflow_packet_v5(u_int8_t* packet, u_int len) {
         return;
     }
 
+    // Yes, but we whould zeroify two bits where sampling type stored
+    //uint16_t netflow5_sampling_ratio = fast_ntoh(nf5_hdr->sampling_rate);
+    uint16_t netflow5_sampling_ratio = sampling_rate;
+
     for (u_int i = 0; i < nflows; i++) {
         size_t offset = NF5_PACKET_SIZE(i);
         struct NF5_FLOW* nf5_flow = (struct NF5_FLOW*)(packet + offset);
@@ -890,7 +894,9 @@ void process_netflow_packet_v5(u_int8_t* packet, u_int len) {
         current_packet.length = fast_ntoh(nf5_flow->flow_octets);
         current_packet.number_of_packets = fast_ntoh(nf5_flow->flow_packets);
 
-        current_packet.sample_ratio = sampling_rate;
+        // TODO: use sampling data from packet, disable customization here
+        // Wireshark dump approves this idea
+        current_packet.sample_ratio = netflow5_sampling_ratio;
 
         current_packet.source_port = fast_ntoh(nf5_flow->src_port);
         current_packet.destination_port = fast_ntoh(nf5_flow->dest_port);

src/netflow_plugin/netflow_collector.h

@@ -6,7 +6,7 @@
 #include "../fastnetmon_types.h"
 
 // For testing
-void process_netflow_packet(u_int8_t* packet, u_int len);
+void process_netflow_packet(u_int8_t* packet, u_int len, std::string client_addres_in_string_format);
 void start_netflow_collection(process_packet_pointer func_ptr);
 
 #endif

src/netmap_plugin/netmap_collector.cpp

@@ -28,6 +28,14 @@
 #include <net/netmap_user.h>
 #include <boost/thread.hpp>
 
+#if defined(__FreeBSD__)
+// On FreeBSD function pthread_attr_setaffinity_np declared here
+#include <pthread_np.h>
+
+// Also we have different type name for cpu set's store
+typedef cpuset_t cpu_set_t;
+#endif
+
 #include "../fastnetmon_packet_parser.h"
 
 // For pooling operations

src/pcap_reader.cpp

@@ -15,10 +15,10 @@
 #include <netinet/in.h>
 #include <arpa/inet.h>
 
-#include "../netflow_plugin/netflow_collector.h"
-#include "../fastnetmon_packet_parser.h"
-#include "../fastnetmon_types.h"
-#include "../fast_library.h"
+#include "netflow_plugin/netflow_collector.h"
+#include "fastnetmon_packet_parser.h"
+#include "fastnetmon_types.h"
+#include "fast_library.h"
 
 #include "log4cpp/Category.hh"
 #include "log4cpp/Appender.hh"
@@ -160,13 +160,24 @@ void pcap_parse_packet(char* buffer, uint32_t len) {
     }
 
     unsigned int payload_length = packet_header.len - packet_header.extended_hdr.parsed_pkt.offset.payload_offset;
-    process_netflow_packet((u_int8_t*)payload_ptr, payload_length);
+    std::string fake_peer_ip = "10.0.1.2";
+    process_netflow_packet((u_int8_t*)payload_ptr, payload_length, fake_peer_ip);
 }
 
-int main() {
+int main(int argc, char** argv) {
     init_logging();
+
+    if (argc != 2) {
+        printf("Please provide path to pcap dump\n");
+        exit(1);
+    }
+
+    printf("We will process file: %s\n", argv[1]);
+    pcap_reader(argv[1]);
+
+
     // pcap_reader("/root/netflowexample2_netflow9_cisco_sampling_issue.pcap");
-    pcap_reader("/root/flow_dump_ipfix_issue_with_fixed_to_2055.pcap");
+    //pcap_reader("/root/flow_dump_ipfix_issue_with_fixed_to_2055.pcap");
     // pcap_reader("/root/ipfix_example_ipt_netflow_syn_flood.pcap");
     // pcap_reader("/Users/pavel-odintsov/Dropbox/ipfix_example_ipt_netflow_syn_flood.pcap");
 }

src/plugin_runner.cpp

@@ -15,7 +15,11 @@
 #include "netflow_plugin/netflow_collector.h"
 #include "sflow_plugin/sflow_collector.h"
 #include "pcap_plugin/pcap_collector.h"
+
+#ifdef PF_RING
 #include "pfring_plugin/pfring_collector.h"
+#endif
+
 #include "netmap_plugin/netmap_collector.h"
 
 // log4cpp logging facility
@@ -75,8 +79,12 @@ int main(int argc, char* argv[]) {
         std::cout << "Starting pcap" << std::endl;
         start_pcap_collection(process_packet);
     } else if (strstr(argv[1], "pfring") != NULL) {
+#ifdef PF_RING
         std::cout << "Starting pf_ring" << std::endl;
         start_pfring_collection(process_packet);
+#else
+        std::cout << "PF_RING support disabled here" << std::endl; 
+#endif
     } else if (strstr(argv[1], "netmap") != NULL) {
         std::cout << "Starting netmap" << std::endl;
         start_netmap_collection(process_packet);

src/tests/build_pcap_reader.sh

@@ -1,9 +0,0 @@
-#!/usr/bin/env bash
-
-clang++ ../ipfix_rfc.cpp -c -oipfix_rfc.o
-clang++ ../fast_library.cpp -c -ofast_library.o
-clang++ ../netflow_plugin/netflow_collector.cpp -c -onetflow_collector.o -I/opt/local/include
-clang ../fastnetmon_packet_parser.c -c -ofastnetmon_packet_parser.o
-clang++ pcap_reader.cpp fastnetmon_packet_parser.o ipfix_rfc.o fast_library.o netflow_collector.o -I/opt/local/include -L/opt/local/lib -o pcap_reader -llog4cpp -lboost_system -lboost_regex
-
-rm -f fastnetmon_packet_parser.o