request parser speed improvements and test updates

lib/http/request.cpp

@@ -1,6 +1,8 @@
 #include "request.hpp"
 #include "../log/log.hpp"
 #include "constants.hpp"
+#include <map>
+#include <cstring>
 #include <stdio.h>
 
 namespace anthracite::http {
@@ -12,7 +14,7 @@ void request::parse_header(std::string& raw_line) {
     std::string header_name = raw_line.substr(0,delim_pos);
     std::string header_val = raw_line.substr(value_pos);
 
-    _headers[header_name] = header(header_name, header_val);
+    _headers[header_name] = header_val;
 }
 
 void request::parse_query_param(std::string& raw_param) {
@@ -22,27 +24,32 @@ void request::parse_query_param(std::string& raw_param) {
     std::string query_name = raw_param.substr(0,delim_pos);
     std::string query_val = raw_param.substr(value_pos);
 
-    _query_params[query_name] = query_param(query_name, query_val);
+    _query_params[query_name] = query_val;
 }
 
-void request::parse_path(std::string& raw_path) {
+void request::parse_path(char* raw_path) {
-    std::stringstream ss(raw_path);
+    char* saveptr = nullptr;
-    std::string tok;
+    char* tok = strtok_r(raw_path, "?", &saveptr);
 
-    if (getline(ss, tok, '?')){
+    if (tok){
         _path = tok;
     }
 
-    while(getline(ss, tok, '&')) {
+    tok = strtok_r(nullptr, "?", &saveptr);
-        parse_query_param(tok);
+    while(tok) {
+        std::string rtok(tok);
+        parse_query_param(rtok);
+        tok = strtok_r(nullptr, "?", &saveptr);
     }
 }
 
-void request::parse_request_line(std::string& raw_line) {
+void request::parse_request_line(char* raw_line) {
         request_line_parser_state state = METHOD;
-        std::stringstream ss(raw_line);
+
-        std::string tok;
+        char* saveptr = nullptr;
-        while(getline(ss, tok, ' ')){
+        char* tok = strtok_r(raw_line, " \r", &saveptr);
+
+        while(tok){
             switch(state) {
                 case METHOD: {
                     auto search = method_map.find(tok);
@@ -57,6 +64,7 @@ void request::parse_request_line(std::string& raw_line) {
                 };
                 
                 case PATH: {
+                    std::string str_tok(tok);
                     parse_path(tok);
                     state = VERSION;
                     break;
@@ -72,6 +80,7 @@ void request::parse_request_line(std::string& raw_line) {
                     return;
                 };
             }
+            tok = strtok_r(nullptr, " \r", &saveptr);
         }
 }
 
@@ -82,22 +91,25 @@ request::request(std::string& raw_data, const std::string& client_ip)
 
     parser_state state = REQUEST_LINE;
 
-    std::stringstream line_stream(raw_data);
+    char* saveptr = nullptr;
-    std::string line;
+    char* tok = strtok_r(raw_data.data(), "\r\n", &saveptr);
 
-    while(getline(line_stream, line, '\n') && state != BODY_CONTENT){
+    while(tok && state != BODY_CONTENT){
-        line.pop_back(); // HTTP requests do newline as \r\n, this removes the \r 
         switch(state) {
             case REQUEST_LINE: {
-                                   parse_request_line(line); 
+                                   parse_request_line(tok); 
                                    state = HEADERS;
+                                    tok = strtok_r(nullptr, "\n", &saveptr);
                                    break;
                                };
             case HEADERS: {
-                if (line.length() == 0) {
+                if (tok[0] == '\r') {
                     state = BODY_CONTENT;
                 } else {
-                    parse_header(line); 
+                    std::string rtok(tok);
+                    rtok.pop_back();
+                    parse_header(rtok); 
+                tok = strtok_r(nullptr, "\n", &saveptr);
                 }
                 break;
             };
@@ -105,9 +117,13 @@ request::request(std::string& raw_data, const std::string& client_ip)
         }
     }
 
-    if (getline(line_stream, line, '\0')) {
+    tok = strtok_r(nullptr, "", &saveptr);
-        _body_content = line;
+    if (tok) {
+        _body_content = std::string(tok);
     }
+    //if (getline(line_stream, line, '\0')) {
+    //    _body_content = line;
+    //}
 }
 
 std::string request::path() { return _path; }
@@ -131,7 +147,7 @@ bool request::close_connection()
     const auto& header = _headers.find("Connection");
     const bool found = header != _headers.end();
 
-    if (found && header->second.value() == "keep-alive") {
+    if (found && header->second == "keep-alive") {
         return false;
     }
 
@@ -141,19 +157,36 @@ bool request::close_connection()
 std::string request::to_string()
 {
     std::string response = "";
-    response += reverse_method_map.find(_method)->second + " " + _path + "?";
+    response += reverse_method_map.find(_method)->second + " " + _path;
 
-    for (auto qp : _query_params) {
+    if (_query_params.size() > 0) {
-        response += qp.second.to_string() + "&";
+        response += "?";
+    }
+
+    auto qp_map = std::map(_query_params.begin(), _query_params.end());
+    auto qp = qp_map.begin();
+    while (qp != qp_map.end()) {
+        response += qp->first + "=" + qp->second;
+        if (++qp != qp_map.end()) {
+            response += "&";
+        }
     }
 
     response += " " + reverse_version_map.find(_http_version)->second + "\r\n";
 
-    for (auto header : _headers) {
+    if (_headers.size() == 0) {
-        response += header.second.to_string();
+        response += "\r\n";
+    }
+
+    auto hd_map = std::map(_headers.begin(), _headers.end());
+    auto hd = hd_map.begin();
+    while (hd != hd_map.end()) {
+        response += hd->first + ": " + hd->second + "\r\n";
+        if (++hd == hd_map.end()) {
+            response += "\r\n";
+        }
     }
 
-    response += "\r\n";
     response += _body_content;
 
     return response;

lib/http/request.hpp

@@ -24,12 +24,12 @@ private:
     std::string _path;
     std::string _client_ipaddr;
     std::string _body_content;
-    std::unordered_map<std::string, header> _headers; // kinda goofy, whatever
+    std::unordered_map<std::string, std::string> _headers; 
-    std::unordered_map<std::string, query_param> _query_params; // kinda goofy, whatever
+    std::unordered_map<std::string, std::string> _query_params;
-                                                                //
+
-    void parse_request_line(std::string& raw_line);
+    void parse_request_line(char* raw_line);
     void parse_header(std::string& raw_line);
-    void parse_path(std::string& raw_path);
+    void parse_path(char* raw_path);
     void parse_query_param(std::string& raw_param);
 
 public:

tests/speed_tests.cpp

@@ -3,26 +3,64 @@
 #include <chrono>
 #include "../lib/http/request.hpp"
 
+#ifdef SPEEDTEST_COMPARE_BOOST
+#include <boost/beast.hpp>
+#endif
+
+
+using std::chrono::high_resolution_clock;
+using std::chrono::duration_cast;
+using std::chrono::duration;
+using std::chrono::milliseconds;
+
+constexpr uint32_t num_requests = 10000000;
 
 TEST(speed_tests, request_parse) {
-    using std::chrono::high_resolution_clock;
-    using std::chrono::duration_cast;
-    using std::chrono::duration;
-    using std::chrono::milliseconds;
-
     std::ifstream t("./test_files/test_request.http");
     std::stringstream buffer;
     buffer << t.rdbuf();
     std::string raw_req = buffer.str();
 
-    auto t1 = high_resolution_clock::now();
+    auto start = high_resolution_clock::now();
-    for(int i = 0; i < 1000000; ++i) {
+
+    for(int i = 0; i < num_requests; ++i) {
         volatile anthracite::http::request req (raw_req, "0.0.0.0");
     }
-    auto t2 = high_resolution_clock::now();
 
-    /* Getting number of milliseconds as an integer. */
+    auto end = high_resolution_clock::now();
-    auto ms_int = duration_cast<milliseconds>(t2 - t1);
 
-    std::cout << "Parsed 1 Million requests in " << ms_int << "ms" << std::endl;
+    auto ms_int = duration_cast<milliseconds>(end-start);
+
+    double m_rps = ((1000.0 / ms_int.count()) * num_requests) / 1000000;
+
+    std::cout << "Parsed " << (num_requests/1000000) << " Million requests in " << ms_int << " ms";
+    std::cout << " at " << m_rps << " Million RPS " << std::endl;
+
+    ASSERT_LT(ms_int.count(), 2000);
 }
+
+#ifdef SPEEDTEST_COMPARE_BOOST
+TEST(speed_tests, boost) {
+    std::ifstream t("./test_files/test_request.http");
+    std::stringstream buffer;
+    buffer << t.rdbuf();
+    std::string raw_req = buffer.str();
+
+    auto start = high_resolution_clock::now();
+
+    for(int i = 0; i < num_requests; ++i) {
+        boost::system::error_code ec;
+        boost::beast::http::request_parser<boost::beast::http::string_body> p;
+        p.put(boost::asio::buffer(raw_req), ec);
+        boost::beast::http::request<boost::beast::http::string_body> r = p.get();
+    }
+
+    auto end = high_resolution_clock::now();
+    auto ms_int = duration_cast<milliseconds>(end-start);
+
+    double m_rps = ((1000.0 / ms_int.count()) * num_requests) / 1000000;
+
+    std::cout << "Parsed " << (num_requests/1000000) << " Million requests in " << ms_int << " ms";
+    std::cout << " at " << m_rps << " Million RPS " << std::endl;
+}
+#endif

tests/test_files/test_request.http

@@ -1,13 +1,15 @@
 GET /foo/bar?test=a&test2=b HTTP/1.1
-Host: example.org
-User-Agent: Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; fr; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8
 Accept: */*
-Accept-Language: fr,fr-fr;q=0.8,en-us;q=0.5,en;q=0.3
-Accept-Encoding: gzip,deflate
 Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7
-Keep-Alive: 115
+Accept-Encoding: gzip,deflate
+Accept-Language: fr,fr-fr;q=0.8,en-us;q=0.5,en;q=0.3
 Connection: keep-alive
 Content-Type: application/x-www-form-urlencoded
-X-Requested-With: XMLHttpRequest
-Referer: http://example.org/test
 Cookie: foo=bar; lorem=ipsum;
+Host: example.org
+Keep-Alive: 115
+Referer: http://example.org/test
+User-Agent: Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10.6; fr; rv:1.9.2.8) Gecko/20100722 Firefox/3.6.8
+X-Requested-With: XMLHttpRequest
+
+{ "test": "content" }

tests/unit_tests.cpp

@@ -0,0 +1,17 @@
+#include <gtest/gtest.h>
+#include <fstream>
+#include "../lib/http/request.hpp"
+#include <boost/beast.hpp>
+
+TEST(unit_tests, single_request_parse) {
+    std::ifstream t("./test_files/test_request.http");
+    std::stringstream buffer;
+    buffer << t.rdbuf();
+
+    std::string raw_req = buffer.str();
+    std::string expected = buffer.str();
+
+    anthracite::http::request req (raw_req, "0.0.0.0");
+
+    ASSERT_EQ(expected, req.to_string());
+}