From f2411235adaae91da9777b4fcc911b5afd88db2d Mon Sep 17 00:00:00 2001 From: John W Higgins Date: Wed, 26 Jun 2019 11:00:46 -0700 Subject: [PATCH 1/5] Add http_version to both request and response --- src/main.c | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main.c b/src/main.c index b0c56b7..36cab41 100644 --- a/src/main.c +++ b/src/main.c @@ -577,13 +577,23 @@ har_debug_callback(CURL * easy, har_headers_from_text(headers, s, size); json_object_set_new(req, "_headersText", json_string(g_strdup(s))); + } + + /* requestline */ + const char * end = g_strstr_len(data, size, "\r\n"); + strncpy(s, data, (end - data)); + s[(end - data)] = '\0'; + if (global_verbose) { /* save requestLine */ - const char * end = g_strstr_len(data, size, "\r\n"); - strncpy(s, data, (end - data)); - s[(end - data)] = '\0'; json_object_set_new(req, "_requestLine", json_string(g_strdup(s))); } + + const char * http_version = strrchr(s, ' '); + if (http_version) { + json_object_set_new(req, "httpVersion", json_string(g_strdup(http_version+1))); + } + request_header_index += 1; break; @@ -598,6 +608,7 @@ har_debug_callback(CURL * easy, json_object_set_new(resp, "_statusLine", json_string(g_strdup(s))); } char ** ss = g_strsplit(s, " ", 3); + json_object_set_new(resp, "httpVersion", json_string(g_strdup(ss[0]))); json_object_set_new(resp, "statusText", json_string(g_strdup(ss[2]))); g_free(s); } From 0b008e3af2861038ae768af16563f2fe9640c818 Mon Sep 17 00:00:00 2001 From: John W Higgins Date: Wed, 26 Jun 2019 11:24:04 -0700 Subject: [PATCH 2/5] Add serverIPAddress --- src/main.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main.c b/src/main.c index 36cab41..a507c71 100644 --- a/src/main.c +++ b/src/main.c @@ -911,6 +911,13 @@ har_entry_from_curl_easy_getinfo(json_t * obj, CURL * easy, json_object_set_new(resp, "redirectURL", json_string(g_strdup(redirect_url))); } + const char * ip; + + curl_easy_getinfo(easy, CURLINFO_PRIMARY_IP, &ip); + if (ip) { + json_object_set_new(entry, "serverIPAddress", json_string(g_strdup(ip))); + } + /* finish up with write callback */ har_response_headers_from_byte_array(resp, harheadout); From 167d4210d09d9231d47e27a0ca594a6edba08dbe Mon Sep 17 00:00:00 2001 From: John W Higgins Date: Wed, 26 Jun 2019 15:10:02 -0700 Subject: [PATCH 3/5] Adding cookies --- src/main.c | 107 +++++++++++++++++++++++++++++++++++-- tests/request-cookies.json | 11 ++++ 2 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 tests/request-cookies.json diff --git a/src/main.c b/src/main.c index a507c71..537948a 100644 --- a/src/main.c +++ b/src/main.c @@ -135,13 +135,14 @@ har_headers_to_curl_slist(json_t * headers) } void -har_headers_from_text(json_t * headers, const char * s, size_t s_len) +har_headers_from_text(json_t * headers, json_t * cookies, const char * s, size_t s_len) { int ix; gchar ** parts; gchar * line; gchar ** lines; json_t * header; + json_t * cookie; const char * name; const char * value; if (!s) { @@ -169,6 +170,59 @@ har_headers_from_text(json_t * headers, const char * s, size_t s_len) json_object_set_new(header, "name", json_string(name)); json_object_set_new(header, "value", json_string(value)); json_array_append_new(headers, header); + + if (!g_ascii_strcasecmp(name, "cookie")) { + gchar ** ss = g_strsplit(value, ";", -1); + + gchar ** nv; + int loc = -1; + + while (ss[++loc]) { + cookie = json_object(); + + nv = g_strsplit(g_strstrip(ss[loc]), "=", 2); + + json_object_set_new(cookie, "name", json_string(nv[0])); + json_object_set_new(cookie, "value", json_string(nv[1])); + + g_strfreev(nv); + + json_array_append_new(cookies, cookie); + } + } else if (!g_ascii_strcasecmp(name, "set-cookie")) { + cookie = json_object(); + + gchar ** ss = g_strsplit(value, ";", -1); + gchar ** nv = g_strsplit(ss[0], "=", 2); + + json_object_set_new(cookie, "name", json_string(g_strdup(nv[0]))); + json_object_set_new(cookie, "value", json_string(g_strdup(nv[1]))); + + g_strfreev(nv); + + int loc = 0; + while (ss[++loc]) { + nv = g_strsplit(g_strstrip(ss[loc]), "=", 2); + + if (!g_ascii_strcasecmp(nv[0], "expires")) { + json_object_set_new(cookie, "expires", json_string(g_strstrip(g_strdup(nv[1])))); + } else if (!g_ascii_strcasecmp(nv[0], "path")) { + json_object_set_new(cookie, "path", json_string(g_strstrip(g_strdup(nv[1])))); + } else if (!g_ascii_strcasecmp(nv[0], "domain")) { + json_object_set_new(cookie, "domain", json_string(g_strstrip(g_strdup(nv[1])))); + } else if (!g_ascii_strcasecmp(nv[0], "max-age")) { + json_object_set_new(cookie, "max-age", json_string(g_strstrip(g_strdup(nv[1])))); + } else if (!g_ascii_strcasecmp(nv[0], "secure")) { + json_object_set_new(cookie, "secure", json_true()); + } else if (!g_ascii_strcasecmp(nv[0], "httponly")) { + json_object_set_new(cookie, "httpOnly", json_true()); + } + + g_strfreev(nv); + } + + json_array_append_new(cookies, cookie); + } } } @@ -387,7 +441,13 @@ har_response_headers_from_byte_array(json_t * resp, GByteArray * bytes) headers = json_object_get(resp, "headers"); } - har_headers_from_text(headers, s, s_len); + json_t * cookies = json_object_get(resp, "cookies"); + if (!cookies || !json_is_array(headers)) { + json_object_set_new(resp, "cookies", json_array()); + cookies = json_object_get(resp, "cookies"); + } + + har_headers_from_text(headers, cookies, s, s_len); json_array_foreach(headers, ix, header) { const char * name = json_string_value(json_object_get(header, "name")); @@ -574,7 +634,25 @@ har_debug_callback(CURL * easy, json_object_set_new(req, "headersSize", json_integer(size)); if (global_verbose) { - har_headers_from_text(headers, s, size); + json_t * cookie_jar = json_object_get(req, "_cookie_jar"); + if (cookie_jar) { + json_t * send_cookies = json_object_get(req, "_send_cookies"); + if (!send_cookies) { + json_object_set_new(req, "_send_cookies", json_true()); + } + json_t * store_cookies = json_object_get(req, "_store_cookies"); + if (!store_cookies) { + json_object_set_new(req, "_store_cookies", json_true()); + } + } + + json_t * cookies = json_object_get(req, "cookies"); + if (!cookies || !json_is_array(cookies)) { + json_object_set_new(req, "cookies", json_array()); + cookies = json_object_get(req, "cookies"); + } + + har_headers_from_text(headers, cookies, s, size); json_object_set_new(req, "_headersText", json_string(g_strdup(s))); } @@ -815,6 +893,7 @@ har_entry_to_curl_easy_setopt(json_t * obj, CURL * easy, json_t * entry = obj; json_t * req = json_object_get(entry, "request"); json_t * resp = json_object_get(entry, "response"); + json_t * opts = json_object_get(entry, "_harcurl_options"); json_t * part; struct curl_httppost * formpost; @@ -886,6 +965,28 @@ har_entry_to_curl_easy_setopt(json_t * obj, CURL * easy, curl_easy_setopt(easy, CURLOPT_WRITEDATA, harbodyout); curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, &har_write_callback); + if (opts) { + /* install cookie jar */ + json_t * cookie_jar = json_object_get(opts, "_cookie_jar"); + if (cookie_jar && json_is_string(cookie_jar)) { + const char * filename = json_string_value(cookie_jar); + + json_t * send_cookies = json_object_get(opts, "_send_cookies"); + if (!send_cookies || json_is_true(send_cookies)) { + curl_easy_setopt(easy, CURLOPT_COOKIEFILE, filename); + } + + json_t * store_cookies = json_object_get(opts, "_store_cookies"); + if (!store_cookies || json_is_true(send_cookies)) { + curl_easy_setopt(easy, CURLOPT_COOKIEJAR, filename); + } + } + + if (!global_verbose) { + json_object_del(entry, "_harcurl_options"); + } + } + //json_decref(req); //json_decref(entry); return HAR_OK; diff --git a/tests/request-cookies.json b/tests/request-cookies.json new file mode 100644 index 0000000..3ae1670 --- /dev/null +++ b/tests/request-cookies.json @@ -0,0 +1,11 @@ +{ + "_harcurl_options": { + "_cookie_jar": "/tmp/cookies", + "_send_cookies": true, + "_store_cookies": false + }, + "request": { + "method": "GET", + "url": "http://httpbin.org/get" + } +} From 6cb6cabad190dee26ffa0f28ea038be17f59196f Mon Sep 17 00:00:00 2001 From: John W Higgins Date: Wed, 26 Jun 2019 16:57:20 -0700 Subject: [PATCH 4/5] Add timings --- src/main.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/src/main.c b/src/main.c index 537948a..a771db1 100644 --- a/src/main.c +++ b/src/main.c @@ -1019,6 +1019,57 @@ har_entry_from_curl_easy_getinfo(json_t * obj, CURL * easy, json_object_set_new(entry, "serverIPAddress", json_string(g_strdup(ip))); } + /* timings */ + + json_t * timings = json_object(); + double time; + double time_previous; + + curl_easy_getinfo(easy, CURLINFO_NAMELOOKUP_TIME, &time); + if (time) { + json_object_set_new(timings, "dns", json_real(time * 1000)); + time_previous = time; + } + + curl_easy_getinfo(easy, CURLINFO_CONNECT_TIME, &time); + if (time) { + json_object_set_new(timings, "connect", json_real((time - time_previous) * 1000)); + time_previous = time; + } + + curl_easy_getinfo(easy, CURLINFO_REDIRECT_TIME, &time); + if (time) { + json_object_set_new(timings, "redirect", json_real((time - time_previous) * 1000)); + time_previous = time; + } + + curl_easy_getinfo(easy, CURLINFO_APPCONNECT_TIME, &time); + if (time) { + json_object_set_new(timings, "ssl", json_real((time - time_previous) * 1000)); + time_previous = time; + } + + curl_easy_getinfo(easy, CURLINFO_PRETRANSFER_TIME, &time); + if (time) { + json_object_set_new(timings, "send", json_real((time - time_previous) * 1000)); + time_previous = time; + } + + curl_easy_getinfo(easy, CURLINFO_STARTTRANSFER_TIME, &time); + if (time) { + json_object_set_new(timings, "wait", json_real((time - time_previous) * 1000)); + time_previous = time; + } + + curl_easy_getinfo(easy, CURLINFO_TOTAL_TIME, &time); + if (time) { + json_object_set_new(entry, "time", json_real(time * 1000)); + json_object_set_new(timings, "receive", json_real((time - time_previous) * 1000)); + time_previous = time; + } + + json_object_set_new(entry, "timings", timings); + /* finish up with write callback */ har_response_headers_from_byte_array(resp, harheadout); @@ -1146,7 +1197,7 @@ main(int argc, char *argv[]) json_object_set_new(entry, "_stoppedDateTime", json_string(g_strdup(g_time_val_to_iso8601 (&ended)))); } json_object_set_new(entry, "startedDateTime", json_string(g_strdup(g_time_val_to_iso8601 (&started)))); - json_object_set_new(entry, "time", json_real((1.0e3)*(double)(ended.tv_sec - started.tv_sec) + (1.0e-3)*(double)(ended.tv_usec - started.tv_usec))); + flags = JSON_SORT_KEYS | JSON_INDENT(2); status = json_dumpf(entry, stdout, flags); if (status) { From 5eb418d0984bfeb907f24603d8407a309bd81b15 Mon Sep 17 00:00:00 2001 From: John W Higgins Date: Wed, 26 Jun 2019 17:00:37 -0700 Subject: [PATCH 5/5] Redirects --- src/main.c | 11 +++++++++++ tests/request-redirects.json | 9 +++++++++ 2 files changed, 20 insertions(+) create mode 100644 tests/request-redirects.json diff --git a/src/main.c b/src/main.c index a771db1..53a64f0 100644 --- a/src/main.c +++ b/src/main.c @@ -966,6 +966,17 @@ har_entry_to_curl_easy_setopt(json_t * obj, CURL * easy, curl_easy_setopt(easy, CURLOPT_WRITEFUNCTION, &har_write_callback); if (opts) { + /* follow redirects */ + part = json_object_get(opts, "_follow_redirects"); + if (!part) { + part = json_false(); + json_object_set_new(opts, "_follow_redirects", part); + } + + if (json_is_true(part)) { + curl_easy_setopt(easy, CURLOPT_FOLLOWLOCATION, 1); + } + /* install cookie jar */ json_t * cookie_jar = json_object_get(opts, "_cookie_jar"); if (cookie_jar && json_is_string(cookie_jar)) { diff --git a/tests/request-redirects.json b/tests/request-redirects.json new file mode 100644 index 0000000..7b9cfce --- /dev/null +++ b/tests/request-redirects.json @@ -0,0 +1,9 @@ +{ + "_harcurl_options": { + "_follow_redirects": true + }, + "request": { + "method": "GET", + "url": "http://httpbin.org/get" + } +}