More robust packet matching.

Packet matches now stored in seperate log file to be verified/modified manually.

data/match-flows.sh

@@ -1,13 +1,20 @@
 #!/bin/sh
 
+readonly MATCH_FOUND=1
 readonly BASE_DIR="$(dirname $0)"
-readonly MATCHER="$BASE_DIR/../packet-matcher/packet-matcher"
+readonly MATCHER="$BASE_DIR/../src/pcap-matcher/pcap-matcher"
+readonly OUTFILE="$BASE_DIR/keylog-matchings.txt"
 
-for k in "$BASE_DIR"/keylogs/*/*.log; do
-    outdir="$BASE_DIR/timings/$(basename $(dirname $k))"
-    outfile="$outdir/$(basename $k)"
-    mkdir -p "$outdir" > /dev/null
-    if ! "$MATCHER" "$k" flows/*.pcap > "$outfile"; then
-        rm "$outfile"
+for k in "$BASE_DIR"/keylogs/*/*/*.log; do
+    printf "%s " "$k"
+    #seq 2 15 | while read i; do
+    echo 12 | while read i; do
+        if "$MATCHER" -t $i -l 250 "$k" flows/*.pcap; then
+            return $MATCH_FOUND
+        fi
+    done 2> /dev/null
+    if [ $? -ne $MATCH_FOUND ]; then
+        echo "No flow file found for $k" 1>&2
+        printf "\n" # Create empty line for log to represent no match found.
     fi
-done
+done > "$OUTFILE"

data/match-packets.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+readonly BASE_DIR="$(dirname $0)"
+readonly MATCHER="$BASE_DIR/../src/packet-matcher/packet-matcher"
+readonly MATCHFILE="$BASE_DIR/keylog-matchings.txt"
+
+grep "$1" "$MATCHFILE" |
+    grep -v ' $' |
+    while read f; do
+        "$MATCHER" $f
+    done

src/packet-matcher/getPackets.c

@@ -13,14 +13,17 @@
 #define PROG_NAME "a.out"
 #endif
 
+// Seconds
+#define KEY_EPS 1
+
+#define SSH_SETUP_PACKET_COUNT 4
+
 #define FLAG_SHORT_HELP "-h"
 #define FLAG_LONG_HELP  "--help"
 
 #define FLAG_SHORT_DIFF "-d"
 #define FLAG_LONG_DIFF  "--diff"
 
-#define FILTER_KEY_PKT "tcp[13] & 8 == 8 and dst port 22"
-
 /* Format of keylog lines in log file.
  * 4 = the fd SSH reads user input from (don't know why it isn't 0 but it isn't)
  * 16384 = How many bytes SSH reads as a time
@@ -38,7 +41,7 @@ typedef struct {
   FILE* keylog;
   char* keylogName;
   struct timeval keylogStart;
-  List* timecaps;
+  TimeCap* timecap;
   struct timeval synEpsiolon;
 } Args;
 
@@ -85,20 +88,13 @@ void check_error(int cond, char* str) {
   }
 }
 
-void check_error_pcap(int cond, pcap_t* pcap, char* str) {
-  if (cond) {
-    pcap_perror(pcap, str);
-    exit(1);
-  }
-}
-
 void print_usage(int exitCode) {
-  printf("Usage: %s [options] keylog pcap [pcap ...]\n"
-         "Pair a keylog to a SSH TCP stream.\n\n"
+  printf("Usage: %s [options] keylog pcap\n"
+         "Pair each key in a keylog to a packet.\n\n"
          "Options:\n"
          "%-20s Show this help message and exit\n"
-         "%-20s The allowable difference in time (in seconds) between keylog \n"
-         "%-20s start and flow start\n",
+         "%-20s The allowable difference in time (in seconds) between client\n"
+         "%-20s and server.\n",
          PROG_NAME,
          "  "FLAG_SHORT_HELP", "FLAG_LONG_HELP,
          "  "FLAG_SHORT_DIFF", "FLAG_LONG_DIFF, "");
@@ -128,20 +124,6 @@ void parse_flag(char** argv, Args* res, size_t* i) {
   }
 }
 
-/* Assumes each pcap has at least one packet and each pcap doesn't overlap.
- */
-static void add_timecap(List* pcaps, char* filename) {
-  char* errBuf  = alloca(PCAP_ERRBUF_SIZE + strlen(filename) +
-                         sizeof(NO_PACKET_ERR));
-  TimeCap* cap = new_timecap(filename, errBuf);
-  if (cap == NULL) {
-    fprintf(stderr, "%s: %s\n", filename, errBuf);
-  }
-  else {
-    list_add_head(pcaps, cap);
-  }
-}
-
 static void read_line(FILE* keylog) {
   int c;
   while ((c = fgetc(keylog)) != '\n' && c != EOF) {
@@ -160,14 +142,20 @@ static struct timeval get_keylog_start(FILE* keylog) {
 }
 
 static Args get_args(int argc, char** argv) {
-  Args res = {NULL, NULL, {0, 0}, new_list(NULL), DEFAULT_SYN_E};
+  Args res = {NULL, NULL, {0, 0}, NULL, DEFAULT_SYN_E};
   size_t i = 1;
+  char* errBuff;
   while (argv[i]) {
     if (argv[i][0] == '-') {
       parse_flag(argv, &res, &i);
     }
     else if (res.keylog) {
-      add_timecap(res.timecaps, argv[i]);
+      errBuff = alloca(PCAP_ERRBUF_SIZE + strlen(argv[i]) +
+                         sizeof(NO_PACKET_ERR));
+      res.timecap = new_timecap(argv[i], errBuff);
+      if (res.timecap == NULL) {
+        fprintf(stderr, "%s: %s\n", argv[i], errBuff);
+      }
     }
     else {
       res.keylog = fopen(argv[i], "r");
@@ -177,14 +165,15 @@ static Args get_args(int argc, char** argv) {
     }
     ++i;
   }
-  if (res.keylog == NULL || res.timecaps == NULL) {
+  if (res.keylog == NULL || res.timecap == NULL) {
     print_usage(1);
   }
   return res;
 }
 
-void timeval_subtract(struct timeval *result, struct timeval *x, struct timeval *y)
-{
+void timeval_subtract(struct timeval *result,
+                      struct timeval *x,
+                      struct timeval *y) {
   *result = *x;
   if (result->tv_usec < y->tv_usec) {
     result->tv_sec -= 1;
@@ -196,12 +185,10 @@ void timeval_subtract(struct timeval *result, struct timeval *x, struct timeval
   result->tv_usec = abs(result->tv_usec);
 }
 
-int is_match(void* tc, void* a) {
+int timercmp_eps(struct timeval* a, struct timeval* b, struct timeval eps) {
   struct timeval diff;
-  TimeCap* cap = (TimeCap*) tc;
-  Args* args = (Args*) a;
-  timeval_subtract(&diff, &args->keylogStart, &cap->time);
-  return timercmp(&diff, &args->synEpsiolon, <);
+  timeval_subtract(&diff, a, b);
+  return timercmp(&diff, &eps, <);
 }
 
 void print_time(struct timeval ts) {
@@ -230,17 +217,21 @@ void print_keystroke(Keystroke key) {
     printf(" %hu\n", key.packetId);
 }
 
+void skip_n_packets(pcap_t* pcap, size_t n) {
+  struct pcap_pkthdr header;
+  while (n-- > 0) {
+    pcap_next(pcap, &header);
+  }
+}
+
 void match_packets_to_keys(TimeCap* cap, FILE* keylog) {
-  struct bpf_program filter;
   struct pcap_pkthdr header;
   Keystroke key = get_next_key(keylog);
   Packet* data;
-  int err = pcap_compile(cap->pcap, &filter, FILTER_KEY_PKT, true,
-                         PCAP_NETMASK_UNKNOWN);
-  check_error_pcap(err == -1, cap->pcap, NULL);
-  pcap_setfilter(cap->pcap, &filter);
+  skip_n_packets(cap->pcap, SSH_SETUP_PACKET_COUNT);
   while ((data = (Packet*) pcap_next(cap->pcap, &header)) != NULL) {
-    if (timercmp(&header.ts, &key.time, >)) {
+    if (timercmp(&header.ts, &key.time, >) ||
+        timercmp_eps(&header.ts, &key.time, double_to_timeval(KEY_EPS))) {
       key.packetTime = header.ts;
       key.packetId = data->id;
       print_keystroke(key);
@@ -248,26 +239,17 @@ void match_packets_to_keys(TimeCap* cap, FILE* keylog) {
       key = get_next_key(keylog);
     }
   }
-  pcap_freecode(&filter);
 }
 
 void cleanup(Args args) {
-  delete_list(args.timecaps, delete_timecap);
+  delete_timecap(args.timecap);
   fclose(args.keylog);
 }
 
 int main(int argc, char** argv) {
   Args args = get_args(argc, argv);
-  TimeCap* cap = (TimeCap*) list_find(args.timecaps, &args, is_match);
   int exitStatus = 0;
-  if (cap != NULL) {
-    match_packets_to_keys(cap, args.keylog);
-  }
-  else {
-    fprintf(stderr, "Failed to find matching flow file for %s.\n",
-            args.keylogName);
-    exitStatus = 1;
-  }
+  match_packets_to_keys(args.timecap, args.keylog);
   cleanup(args);
   return exitStatus;
 }

src/pcap-matcher/main.c

@@ -1,5 +1,6 @@
 #include<stdio.h>
 #include<stdlib.h>
+#include<stdbool.h>
 #include<string.h>
 #include<pcap/pcap.h>
 #include<stdbool.h>
@@ -27,6 +28,9 @@
 #define FLAG_SHORT_LENG "-l"
 #define FLAG_LONG_LENG  "--len-diff"
 
+#define FLAG_SHORT_VERB "-v"
+#define FLAG_LONG_VERB  "--verbose"
+
 /* Format of keylog lines in log file.
  * 4 = the fd SSH reads user input from (don't know why it isn't 0 but it isn't)
  * 16384 = How many bytes SSH reads as a time
@@ -49,6 +53,8 @@ typedef struct {
   List* timecaps;
   struct timeval synEpsiolon;
   size_t keyEpsilon;
+  bool verbose;
+  size_t line_count;
 } Args;
 
 void check_error(int cond, char* str) {
@@ -63,12 +69,14 @@ void print_usage(int exitCode) {
          "Pair a keylog to a SSH TCP stream.\n\n"
          "Options:\n"
          "%-20s Show this help message and exit\n"
+         "%-20s Print reasons for rejecting pcap files\n"
          "%-20s The allowable difference in time (in seconds) between keylog \n"
          "%-20s start and flow start\n"
          "%-20s The allowable difference in estimated keys between keylog and\n"
          "%-20s flow \n",
          PROG_NAME,
          "  "FLAG_SHORT_HELP", "FLAG_LONG_HELP,
+         "  "FLAG_SHORT_VERB", "FLAG_LONG_VERB,
          "  "FLAG_SHORT_DIFF", "FLAG_LONG_DIFF, "",
          "  "FLAG_SHORT_LENG", "FLAG_LONG_LENG, "");
   exit(exitCode);
@@ -87,6 +95,10 @@ void parse_flag(char** argv, Args* res, size_t* i) {
   if (strcmp(arg, FLAG_SHORT_HELP) == 0 || strcmp(arg, FLAG_LONG_HELP) == 0) {
     print_usage(0);
   }
+  else if (strcmp(arg, FLAG_SHORT_VERB) == 0 ||
+           strcmp(arg, FLAG_LONG_VERB) == 0) {
+    res->verbose = true;
+  }
   else if (strcmp(arg, FLAG_SHORT_DIFF) == 0 ||
            strcmp(arg, FLAG_LONG_DIFF) == 0) {
     if (sscanf(argv[++*i], " %lf", &d) == 1 && d > 0) {
@@ -136,6 +148,14 @@ static struct timeval get_keylog_start(FILE* keylog) {
   return double_to_timeval(seconds);
 }
 
+size_t count_lines(FILE* file) {
+  size_t res = 1;
+  while (EOF != read_line(file)) {
+    ++res;
+  }
+  return res;
+}
+
 static Args get_args(int argc, char** argv) {
   Args res = {NULL, NULL, {0, 0}, new_list(NULL), DEFAULT_SYN_E, DEFAULT_TIM_E};
   size_t i = 1;
@@ -151,6 +171,7 @@ static Args get_args(int argc, char** argv) {
       res.keylogName = argv[i];
       check_error(res.keylog == NULL, argv[i]);
       res.keylogStart = get_keylog_start(res.keylog);
+      res.line_count = count_lines(res.keylog);
     }
     ++i;
   }
@@ -180,24 +201,27 @@ int timercmp_eps(struct timeval* a, struct timeval* b, struct timeval eps) {
   return timercmp(&diff, &eps, <);
 }
 
-size_t count_lines(FILE* file) {
-  size_t res = 1;
-  while (EOF != read_line(file)) {
-    ++res;
-  }
-  return res;
-}
+#define LOG(ARGS, FMT, ...) do {                \
+  if ((ARGS)->verbose) {                        \
+    fprintf(stderr, (FMT), ##__VA_ARGS__);      \
+  }                                             \
+} while (0)
 
 int is_match(void* tc, void* a) {
   TimeCap* cap = (TimeCap*) tc;
   Args* args = (Args*) a;
-  size_t line_count;
+  int res;
   if (timercmp_eps(&args->keylogStart, &cap->time, args->synEpsiolon)) {
-    line_count = count_lines(args->keylog);
-    return line_count <= cap->pack_count + args->keyEpsilon &&
-      line_count >= cap->pack_count - args->keyEpsilon;
+    res = args->line_count <= cap->pack_count + args->keyEpsilon &&
+      args->line_count >= cap->pack_count - args->keyEpsilon;
+    if (!res) {
+      LOG(args, "PCAP %s had incompatible line numbers.\n", cap->name);
+      LOG(args, "%zu keys, %zu packets.\n", args->line_count, cap->pack_count);
+    }
+    return res;
   }
   else {
+    LOG(args, "PCAP %s had incompatible timestamp.\n", cap->name);
     return 0;
   }
 }
@@ -217,6 +241,7 @@ int main(int argc, char** argv) {
   else {
     fprintf(stderr, "Failed to find matching flow file for %s.\n",
             args.keylogName);
+    exitStatus = 1;
   }
   cleanup(args);
   return exitStatus;