polished up event loop changes

CHANGELOG.md

@@ -1,6 +1,7 @@
 # 0.3.0 
 - SSL support via OpenSSL
-- Switched from thread per connection to event driven threading model
+- Added "Thread Manager" class to allow for multiple (or custom) threading models (process per thread, event loop)
+- Defaul threading model is now event loop
 - Rewrote request parser for readability and speed
 - Added improved logging with different log levels
 - Separated anthracite into libanthracite and anthracite-bin to allow for other projects to implement anthracite (example in ./src/api_main.cpp)

CMakeLists.txt

@@ -39,9 +39,6 @@ target_link_libraries(anthracite-bin anthracite)
 add_dependencies(anthracite-bin build-supplemental)
 add_dependencies(anthracite-bin anthracite)
 
-add_executable(anthracite-api-bin src/api_main.cpp) 
-target_link_libraries(anthracite-api-bin anthracite)
-
 include(FetchContent)
 FetchContent_Declare(
   googletest

README.md

@@ -1,5 +1,6 @@
 # Anthracite
-A simple web server written in C++. Supports HTTP 1.0 & 1.1.
+
+Anthracite is an extensible, low-dependency, fast web server.
 
 ## Developing
 
@@ -16,7 +17,7 @@ Create a `build/` directory, run `cmake ..`, and then `make` to build.
 - [x] HTTP/1.1
 - [x] Enhance logging
 - [x] Create library that can be used to implement custom backends (i.e. webapi, fileserver, etc) 
-- [ ] Faster parsing 
+- [x] Faster parsing 
 - [ ] HTTP/2 
 - [ ] Improve benchmarking infrastructure
 - [ ] Fix glaring security issues 

lib/anthracite.cpp

@@ -1,44 +0,0 @@
-#include "./anthracite.hpp"
-#include "./log/log.hpp"
-#include <iostream>
-#include <netinet/in.h>
-#include <sys/socket.h>
-#include <unistd.h>
-#include "./config/config.hpp"
-#include "./thread_mgr/event_loop.hpp"
-#include <signal.h>
-
-using namespace anthracite;
-
-void log_request_and_response(http::request& req, std::unique_ptr<http::response>& resp, uint32_t micros);
-
-constexpr int default_port = 80;
-constexpr int max_worker_threads = 128;
-
-thread_mgr::event_loop* elp = nullptr;
-
-
-extern "C" void signalHandler(int signum) {
-    log::warn << "Caught signal #" << signum << ", exiting Anthracite" << std::endl;
-    elp->stop();
-}
-
-
-int anthracite_main(backends::backend& be, config::config& config)
-{
-    signal(SIGTERM, signalHandler);
-    signal(SIGSEGV, signalHandler);
-    signal(SIGINT, signalHandler);
-    signal(SIGABRT, signalHandler);
-
-    log::logger.initialize(log::LOG_LEVEL_INFO);
-    thread_mgr::event_loop el(be, config);
-    elp = &el;
-    el.start();
-    return 0;
-}
-
-void log_request_and_response(http::request& req, std::unique_ptr<http::response>& resp, uint32_t micros)
-{
-    log::info << "[" << resp->status_code() << " " + http::status_map.find(resp->status_code())->second + "] " + req.client_ip() + " " + http::reverse_method_map.find(req.get_method())->second + " " + req.path() << " in " << micros << " usecs" << std::endl;
-}

lib/anthracite.hpp

@@ -1,6 +0,0 @@
-#include "backends/backend.hpp"
-#include "config/config.hpp"
-
-using namespace anthracite;
-
-int anthracite_main(backends::backend& be, config::config& cfg);

lib/config/config.hpp

@@ -23,12 +23,13 @@ namespace anthracite::config {
     };
 
     class config {
-        uint16_t _worker_threads;
+        int _worker_threads;
+        int _max_clients;
         std::optional<http_config> _http_config;
         std::optional<https_config> _https_config;
 
     public:
-        config(uint16_t worker_threads) : _worker_threads(worker_threads) {
+        config(int worker_threads, int max_clients) : _worker_threads(worker_threads), _max_clients(max_clients) {
         }
 
         void add_http_config(http_config config) {
@@ -39,10 +40,14 @@ namespace anthracite::config {
             _https_config = config;
         }
 
-        uint16_t worker_threads() {
+        int worker_threads() {
             return _worker_threads;
         }
         
+        int max_clients() {
+            return _max_clients;
+        }
+
         std::optional<http_config>& http_cfg() {
             return _http_config;
         }

lib/http/request.cpp

@@ -1,13 +1,14 @@
 #include "request.hpp"
 #include "../log/log.hpp"
 #include "constants.hpp"
-#include <map>
 #include <cstring>
+#include <map>
 #include <stdio.h>
 
 namespace anthracite::http {
 
-void request::parse_header(std::string& raw_line) {
+void request::parse_header(std::string& raw_line)
+{
     auto delim_pos = raw_line.find_first_of(':');
     auto value_pos = raw_line.find_first_not_of(' ', delim_pos + 1);
 
@@ -17,7 +18,8 @@ void request::parse_header(std::string& raw_line) {
     _headers[header_name] = header_val;
 }
 
-void request::parse_query_param(std::string& raw_param) {
+void request::parse_query_param(std::string& raw_param)
+{
     auto delim_pos = raw_param.find_first_of('=');
     auto value_pos = delim_pos + 1;
 
@@ -27,7 +29,8 @@ void request::parse_query_param(std::string& raw_param) {
     _query_params[query_name] = query_val;
 }
 
-void request::parse_path(char* raw_path) {
+void request::parse_path(char* raw_path)
+{
     char* saveptr = nullptr;
     char* tok = strtok_r(raw_path, "?", &saveptr);
 
@@ -43,7 +46,8 @@ void request::parse_path(char* raw_path) {
     }
 }
 
-void request::parse_request_line(char* raw_line) {
+void request::parse_request_line(char* raw_line)
+{
     request_line_parser_state state = METHOD;
 
     char* saveptr = nullptr;
@@ -113,7 +117,8 @@ request::request(std::string& raw_data, const std::string& client_ip)
             }
             break;
         };
-            case BODY_CONTENT: break;
+        case BODY_CONTENT:
+            break;
         }
     }
 
@@ -147,11 +152,11 @@ bool request::close_connection()
     const auto& header = _headers.find("Connection");
     const bool found = header != _headers.end();
 
-    if (found && header->second == "keep-alive") {
+    if (found && header->second == "close") {
-        return false;
+        return true;
     }
 
-    return true;
+    return false;
 }
 
 std::string request::to_string()

lib/log/log.cpp

@@ -10,6 +10,11 @@ void Logger::initialize(enum LOG_LEVEL level)
     _level = level;
 }
 
+void Logger::log_request_and_response(http::request& req, std::unique_ptr<http::response>& resp, uint32_t micros)
+{
+    log::info << "[" << resp->status_code() << " " + http::status_map.find(resp->status_code())->second + "] " + req.client_ip() + " " + http::reverse_method_map.find(req.get_method())->second + " " + req.path() << " in " << micros << " usecs" << std::endl;
+}
+
 LogBuf::LogBuf(std::ostream& output_stream, const std::string& tag, enum LOG_LEVEL level)
     : _output_stream(output_stream)
     , _tag(tag)

lib/log/log.hpp

@@ -3,6 +3,10 @@
 #include <iostream>
 #include <ostream>
 #include <sstream>
+#include <inttypes.h>
+#include <memory>
+#include "../http/request.hpp"
+#include "../http/response.hpp"
 
 namespace anthracite::log {
     enum LOG_LEVEL {
@@ -21,9 +25,9 @@ namespace anthracite::log {
         
         Logger();
         void initialize(enum LOG_LEVEL level);
+        void log_request_and_response(http::request& req, std::unique_ptr<http::response>& resp, uint32_t micros);
     };
     
-
     class LogBuf : public std::stringbuf
     {
             std::string _tag;
@@ -50,4 +54,6 @@ namespace anthracite::log {
 
     static class LogBuf debugBuf{std::cout, "DEBG", LOG_LEVEL_DEBUG};
     static std::ostream debug(&debugBuf);
+
+
 };

lib/socket/openssl_socket.cpp

@@ -1,18 +1,18 @@
-#include <exception>
-#include <openssl/ssl.h>
-#include <openssl/err.h>
 #include "./openssl_socket.hpp"
+#include "../log/log.hpp"
 #include <arpa/inet.h>
 #include <array>
+#include <exception>
+#include <iostream>
 #include <malloc.h>
 #include <netinet/in.h>
+#include <openssl/err.h>
+#include <openssl/ssl.h>
 #include <string>
 #include <sys/socket.h>
 #include <sys/time.h>
 #include <unistd.h>
 #include <vector>
-#include <iostream>
-#include "../log/log.hpp"
 
 namespace anthracite::socket {
 
@@ -70,8 +70,6 @@ bool openssl_socket::wait_for_conn()
     } else {
         return false;
     }
-
- 
 }
 
 void openssl_socket::close_conn()

lib/socket/socket.hpp

@@ -32,6 +32,8 @@ public:
     virtual void close_conn();
     virtual void send_message(std::string& msg);
     virtual std::string recv_message(int buffer_size);
+
+    int csock() { return client_socket; }
 };
 
 };

lib/thread_mgr/event_loop.cpp

@@ -1,36 +1,45 @@
 #include "./event_loop.hpp"
 #include "../log/log.hpp"
 #include "../socket/openssl_socket.hpp"
-#include <mutex>
+#include "assert.h"
+#include "sys/epoll.h"
 #include <chrono>
+#include <mutex>
 #include <syncstream>
 
-using std::chrono::high_resolution_clock;
-using std::chrono::duration_cast;
 using std::chrono::duration;
+using std::chrono::duration_cast;
+using std::chrono::high_resolution_clock;
 using std::chrono::milliseconds;
 
 namespace anthracite::thread_mgr {
-    event_loop::event::event(socket::anthracite_socket* socket) :
+event_loop::event::event(socket::anthracite_socket* socket, std::chrono::time_point<std::chrono::high_resolution_clock> timestamp)
-                    _socket(socket) {}
+    : _socket(socket)
+    , _ts(timestamp)
+{
+}
 
-    socket::anthracite_socket* event_loop::event::socket() {
+socket::anthracite_socket* event_loop::event::socket()
+{
     return _socket;
 }
 
-    event_loop::event_loop(backends::backend& backend, config::config& config) : thread_mgr(backend, config), _error_backend("./www") {}
+std::chrono::time_point<std::chrono::high_resolution_clock>& event_loop::event::timestamp()
+{
+    return _ts;
+}
 
-    bool event_loop::event_handler(event& event) {
+event_loop::event_loop(backends::backend& backend, config::config& config)
+    : thread_mgr(backend, config)
+    , _error_backend("./www")
+{
+}
+
+bool event_loop::event_handler(event& event)
+{
     std::string raw_request = event.socket()->recv_message(http::HEADER_BYTES);
 
-        // We're doing the start here even though it would ideally be done 
-        // before the first line since if we leave the connection open for 
-        // HTTP 1.1, we can spend a bit of time waiting
-        auto start = high_resolution_clock::now();
-        
     if (raw_request == "") {
-            event.socket()->close_conn();
-            delete event.socket();
         return false;
     }
 
@@ -41,20 +50,19 @@ namespace anthracite::thread_mgr {
     event.socket()->send_message(resp->content());
 
     auto end = high_resolution_clock::now();
-        auto ms_int = duration_cast<std::chrono::microseconds>(end-start);
+    auto ms_int = duration_cast<std::chrono::microseconds>(end - event.timestamp());
-        //log_request_and_response(req, resp , ms_int.count());
+    log::logger.log_request_and_response(req, resp, ms_int.count());
 
     resp.reset();
     if (req.close_connection()) {
-            event.socket()->close_conn();
-            delete event.socket();
         return false;
     }
 
     return true;
 }
 
-    void event_loop::worker_thread_loop(int threadno) {
+void event_loop::worker_thread_loop(int threadno)
+{
     unsigned char buf[sizeof(class event)];
 
     std::osyncstream(log::info) << "Starting worker thread " << threadno << std::endl;
@@ -62,11 +70,14 @@ namespace anthracite::thread_mgr {
         // Get event from queue
         std::unique_lock lock(_event_mtx);
 
-
         event* ev = nullptr;
 
         if (_events.size() > 0) {
-                ev = new (buf) event(_events.back());
+            if (!lock.owns_lock()) {
+                lock.lock();
+            }
+            assert(lock.owns_lock());
+            ev = new (buf) event(_events.front());
             _events.pop();
             lock.unlock();
         } else {
@@ -76,29 +87,50 @@ namespace anthracite::thread_mgr {
                 break;
             }
 
-                ev = new (buf) event(_events.back());
+            assert(lock.owns_lock());
+            ev = new (buf) event(_events.front());
             _events.pop();
             lock.unlock();
         }
 
-
+        if (event_handler(*ev)) {
-            // process
+            struct epoll_event event;
-            bool requeue = event_handler(*ev);
+            event.events = EPOLLIN;
-
+            event.data.ptr = ev->socket();
-            // if necessary, requeue
+            epoll_ctl(_epoll_fd, EPOLL_CTL_ADD, ev->socket()->csock(), &event);
-            if (requeue) {
+        } else {
-                {
+            ev->socket()->close_conn();
-                    std::lock_guard<std::mutex> lg(_event_mtx);
+            delete ev->socket();
-                    _events.push(*ev);
-                }
-                _event_cv.notify_one();
         }
     }
 
     std::osyncstream(log::info) << "Stopping worker thread " << threadno << std::endl;
 }
 
-    void event_loop::listener_thread_loop(config::http_config& http_config) {
+void event_loop::eventer_thread_loop()
+{
+    struct epoll_event* events = new struct epoll_event[_config.max_clients()];
+    std::osyncstream(log::info) << "epoll() thread started" << std::endl;
+    while (_run) {
+        int ready_fds = epoll_wait(_epoll_fd, events, _config.max_clients(), 1000);
+
+        if (ready_fds > 0) {
+            std::lock_guard<std::mutex> lg(_event_mtx);
+            for (int i = 0; i < ready_fds; i++) {
+                socket::anthracite_socket* sockptr = reinterpret_cast<socket::anthracite_socket*>(events[i].data.ptr);
+                struct epoll_event ev;
+                epoll_ctl(_epoll_fd, EPOLL_CTL_DEL, sockptr->csock(), &events[i]);
+                _events.push(event(sockptr, std::chrono::high_resolution_clock::now()));
+            }
+            _event_cv.notify_one();
+        }
+    }
+    delete[] events;
+    std::osyncstream(log::info) << "epoll() thread exited" << std::endl;
+}
+
+void event_loop::listener_thread_loop(config::http_config& http_config)
+{
     socket::anthracite_socket* socket;
 
     config::http_config* http_ptr = &http_config;
@@ -125,9 +157,10 @@ namespace anthracite::thread_mgr {
                 client_sock = new socket::anthracite_socket(*socket);
             }
 
-                std::lock_guard<std::mutex> lg(_event_mtx);
+            struct epoll_event event;
-                _events.push(event(client_sock));
+            event.events = EPOLLIN;
-                _event_cv.notify_one();
+            event.data.ptr = client_sock;
+            epoll_ctl(_epoll_fd, EPOLL_CTL_ADD, client_sock->csock(), &event);
         }
     }
 
@@ -136,9 +169,11 @@ namespace anthracite::thread_mgr {
     delete socket;
 }
 
-    void event_loop::start() {
+void event_loop::start()
+{
     log::info << "Starting event_loop Thread Manager" << std::endl;
 
+    _epoll_fd = epoll_create(1);
     _run = true;
 
     std::vector<std::thread> listener_threads;
@@ -159,6 +194,11 @@ namespace anthracite::thread_mgr {
         listener_threads.push_back(std::move(thread));
     }
 
+    {
+        auto thread = std::thread(&event_loop::eventer_thread_loop, this);
+        listener_threads.push_back(std::move(thread));
+    }
+
     for (std::thread& t : worker_threads) {
         t.join();
     }
@@ -168,7 +208,8 @@ namespace anthracite::thread_mgr {
     }
 }
 
-    void event_loop::stop() {
+void event_loop::stop()
+{
     _run = false;
     std::lock_guard<std::mutex> lg(_event_mtx);
     _event_cv.notify_all();

lib/thread_mgr/event_loop.hpp

@@ -1,6 +1,7 @@
 #include "./thread_mgr.hpp"
 #include "../socket/socket.hpp"
 #include "../backends/file_backend.hpp"
+#include <chrono>
 #include <condition_variable>
 #include <mutex>
 #include <queue>
@@ -9,11 +10,14 @@ namespace anthracite::thread_mgr {
     class event_loop : public virtual thread_mgr {
             class event {
                 socket::anthracite_socket* _socket;
+                std::chrono::time_point<std::chrono::high_resolution_clock> _ts; 
                 public:
-                    event(socket::anthracite_socket* socket);
+                    event(socket::anthracite_socket* socket, std::chrono::time_point<std::chrono::high_resolution_clock> timestamp);
                     socket::anthracite_socket* socket();
+                    std::chrono::time_point<std::chrono::high_resolution_clock>& timestamp();    
             };
 
+            int _epoll_fd;
             std::mutex _event_mtx;
             std::condition_variable _event_cv;
             std::queue<event> _events;
@@ -21,6 +25,7 @@ namespace anthracite::thread_mgr {
 
             void worker_thread_loop(int threadno);
             void listener_thread_loop(config::http_config& http_config);
+            void eventer_thread_loop();
             bool event_handler(event& ev);
 
         public:

src/api_main.cpp

@@ -1,112 +0,0 @@
-#include "../lib/anthracite.hpp"
-#include "../lib/backends/backend.hpp"
-#include "../lib/http/constants.hpp"
-#include <iostream>
-#include <memory>
-#include <optional>
-#include <sstream>
-#include <unordered_map>
-#include <vector>
-#include <span>
-
-using namespace anthracite;
-
-using CallbackType = std::unique_ptr<http::response> (*)(http::request&);
-class api_backend : public backends::backend {
-
-    class RouteNode {
-    public:
-        std::optional<CallbackType> callback;
-
-        RouteNode()
-            : callback(std::nullopt)
-        {
-        }
-        std::unordered_map<std::string, RouteNode> routes;
-    };
-
-    RouteNode root;
-
-    std::unique_ptr<http::response> default_route(http::request& req)
-    {
-        std::unique_ptr<http::response> resp = std::make_unique<http::response>();
-
-        resp->add_body("Not Found");
-        resp->add_header(http::header("Content-Type", "application/json"));
-        resp->add_status(http::status_codes::NOT_FOUND);
-
-        return resp;
-    }
-
-    std::unique_ptr<http::response> find_handler(http::request& req)
-    {
-        std::string filename = req.path().substr(1);
-        std::vector<std::string> result;
-        std::stringstream ss(filename);
-        std::string item;
-
-        RouteNode* cur = &root;
-        while (getline(ss, item, '/')) {
-            if (cur->routes.find(item) == cur->routes.end()) {
-                if (cur->routes.find("*") == cur->routes.end()) {
-                    break;
-                } else {
-                    cur = &cur->routes["*"];
-                }
-            } else {
-                cur = &cur->routes[item];
-            }
-        }
-
-        if (cur->callback.has_value()) {
-            return cur->callback.value()(req);
-        } else {
-            return default_route(req);
-        }
-    }
-
-    std::unique_ptr<http::response> handle_request(http::request& req) override
-    {
-        return find_handler(req);
-    }
-
-public:
-    api_backend()
-    {
-        root.routes = std::unordered_map<std::string, RouteNode>();
-    }
-
-    void register_endpoint(std::string pathspec, CallbackType callback)
-    {
-        std::vector<std::string> result;
-        std::stringstream ss(pathspec);
-        std::string item;
-
-        RouteNode* cur = &root;
-        while (getline(ss, item, '/')) {
-            cur->routes[item] = RouteNode {};
-            cur = &cur->routes[item];
-        }
-
-        cur->callback = callback;
-    }
-};
-
-std::unique_ptr<http::response> handle_request(http::request& req)
-{
-    std::unique_ptr<http::response> resp = std::make_unique<http::response>();
-
-    resp->add_body(R"({"user": "endpoint"}")");
-    resp->add_header(http::header("Content-Type", "application/json"));
-    resp->add_status(http::status_codes::OK);
-
-    return resp;
-}
-
-int main(int argc, char** argv)
-{
-    auto args = std::span(argv, size_t(argc));
-    api_backend ab;
-    ab.register_endpoint("users/*", handle_request);
-    //anthracite_main(argc, argv, ab);
-}

src/file_main.cpp

@@ -1,15 +1,35 @@
-#include "../lib/anthracite.hpp"
 #include "../lib/backends/file_backend.hpp"
 #include "../lib/config/config.hpp"
+#include "../lib/log/log.hpp"
+#include "../lib/thread_mgr/event_loop.hpp"
+#include "signal.h"
+#include "string.h"
+#include <memory>
 
-using namespace anthracite;
+std::shared_ptr<anthracite::thread_mgr::thread_mgr> server = nullptr;
+
+extern "C" void signalHandler(int signum)
+{
+    anthracite::log::warn << "Caught signal SIG" << sigabbrev_np(signum) << ", exiting Anthracite" << std::endl;
+    if (server != nullptr) {
+        server->stop();
+    }
+}
 
 int main(int argc, char** argv)
 {
-    backends::file_backend fb("./www");
+    anthracite::log::logger.initialize(anthracite::log::LOG_LEVEL_INFO);
-    config::config cfg(5);
+    anthracite::log::info << "Starting Anthracite, a higher performance web server" << std::endl;
-    cfg.add_http_config(config::http_config(8080));
+    signal(SIGINT, signalHandler);
-    cfg.add_https_config(config::https_config(8081, "", ""));
 
-    anthracite_main(fb, cfg);
+    anthracite::backends::file_backend fb("./www");
+    anthracite::config::config cfg(5, 10000);
+    cfg.add_http_config(anthracite::config::http_config(8080));
+    // cfg.add_https_config(config::https_config(8081, "", ""));
+
+    server = std::make_shared<anthracite::thread_mgr::event_loop>(fb, cfg);
+
+    server->start();
+
+    anthracite::log::info << "Stopping Anthracite, a higher performance web server" << std::endl;
 }

tests/speed_tests.cpp

@@ -1,21 +1,21 @@
-#include <gtest/gtest.h>
-#include <fstream>
-#include <chrono>
 #include "../lib/http/request.hpp"
+#include <chrono>
+#include <fstream>
+#include <gtest/gtest.h>
 
 #ifdef SPEEDTEST_COMPARE_BOOST
 #include <boost/beast.hpp>
 #endif
 
-
-using std::chrono::high_resolution_clock;
-using std::chrono::duration_cast;
 using std::chrono::duration;
+using std::chrono::duration_cast;
+using std::chrono::high_resolution_clock;
 using std::chrono::milliseconds;
 
 constexpr uint32_t num_requests = 10000000;
 
-TEST(speed_tests, request_parse) {
+TEST(speed_tests, request_parse)
+{
     std::ifstream t("./test_files/test_request.http");
     std::stringstream buffer;
     buffer << t.rdbuf();
@@ -40,7 +40,8 @@ TEST(speed_tests, request_parse) {
 }
 
 #ifdef SPEEDTEST_COMPARE_BOOST
-TEST(speed_tests, boost) {
+TEST(speed_tests, boost)
+{
     std::ifstream t("./test_files/test_request.http");
     std::stringstream buffer;
     buffer << t.rdbuf();

tests/unit_tests.cpp

@@ -1,9 +1,10 @@
-#include <gtest/gtest.h>
-#include <fstream>
 #include "../lib/http/request.hpp"
 #include <boost/beast.hpp>
+#include <fstream>
+#include <gtest/gtest.h>
 
-TEST(unit_tests, single_request_parse) {
+TEST(unit_tests, single_request_parse)
+{
     std::ifstream t("./test_files/test_request.http");
     std::stringstream buffer;
     buffer << t.rdbuf();