From 7b63f846d7e420a6c0bb0f8e5c682d221f30bc32 Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Wed, 5 Feb 2025 13:31:20 -0500 Subject: [PATCH 01/15] start hacky tls self-roll --- lib/anthracite.cpp | 25 +--- lib/socket/socket.hpp | 14 ++- lib/socket/tls_socket.cpp | 106 +++++++++++++++++ lib/socket/tls_socket.hpp | 235 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 355 insertions(+), 25 deletions(-) create mode 100644 lib/socket/tls_socket.cpp create mode 100644 lib/socket/tls_socket.hpp diff --git a/lib/anthracite.cpp b/lib/anthracite.cpp index 51a646e..20c2373 100644 --- a/lib/anthracite.cpp +++ b/lib/anthracite.cpp @@ -1,6 +1,7 @@ #include "./anthracite.hpp" #include "./log/log.hpp" #include "./socket/socket.hpp" +#include "./socket/tls_socket.hpp" #include "backends/file_backend.hpp" #include #include @@ -24,7 +25,7 @@ using std::chrono::duration_cast; using std::chrono::duration; using std::chrono::milliseconds; -void handle_client(socket::anthracite_socket s, backends::backend& b, backends::file_backend& fb, std::mutex& thread_wait_mutex, std::condition_variable& thread_wait_condvar, int& active_threads) +void handle_client(socket::tls_socket s, backends::backend& b, backends::file_backend& fb, std::mutex& thread_wait_mutex, std::condition_variable& thread_wait_condvar, int& active_threads) { while (true) { std::string raw_request = s.recv_message(http::HEADER_BYTES); @@ -38,20 +39,7 @@ void handle_client(socket::anthracite_socket s, backends::backend& b, backends:: break; } - http::request req(raw_request, s.get_client_ip()); - std::unique_ptr resp = req.is_supported_version() ? b.handle_request(req) : fb.handle_error(http::status_codes::HTTP_VERSION_NOT_SUPPORTED); - std::string header = resp->header_to_string(); - s.send_message(header); - s.send_message(resp->content()); - - auto end = high_resolution_clock::now(); - auto ms_int = duration_cast(end-start); - log_request_and_response(req, resp , ms_int.count()); - - resp.reset(); - if (req.close_connection()) { - break; - } + continue; } s.close_conn(); { @@ -63,17 +51,16 @@ void handle_client(socket::anthracite_socket s, backends::backend& b, backends:: int anthracite_main(int argc, char** argv, backends::backend& be) { - log::logger.initialize(log::LOG_LEVEL_INFO); + log::logger.initialize(log::LOG_LEVEL_DEBUG); auto args = std::span(argv, size_t(argc)); int port_number = default_port; if (argc > 1) { port_number = atoi(args[1]); } - log::verbose << "Initializing Anthracite" << std::endl; - socket::anthracite_socket s(port_number); + + socket::tls_socket s(port_number); backends::file_backend fb(argc > 2 ? args[2] : "./www"); - log::verbose << "Initialization Complete" << std::endl; log::info << "Listening for HTTP connections on port " << port_number << std::endl; int active_threads = 0; diff --git a/lib/socket/socket.hpp b/lib/socket/socket.hpp index 7a57de7..dc924cc 100644 --- a/lib/socket/socket.hpp +++ b/lib/socket/socket.hpp @@ -1,3 +1,5 @@ +#pragma once + #include #include #include @@ -10,8 +12,8 @@ namespace anthracite::socket { class anthracite_socket { +protected: static const int MAX_QUEUE_LENGTH = 100; -private: int server_socket; int client_socket {}; std::string client_ip; @@ -22,11 +24,11 @@ private: public: anthracite_socket(int port, int max_queue = MAX_QUEUE_LENGTH); - void wait_for_conn(); - const std::string& get_client_ip(); - void close_conn(); - void send_message(std::string& msg); - std::string recv_message(int buffer_size); + virtual void wait_for_conn(); + virtual const std::string& get_client_ip(); + virtual void close_conn(); + virtual void send_message(std::string& msg); + virtual std::string recv_message(int buffer_size); }; }; diff --git a/lib/socket/tls_socket.cpp b/lib/socket/tls_socket.cpp new file mode 100644 index 0000000..9ae13e2 --- /dev/null +++ b/lib/socket/tls_socket.cpp @@ -0,0 +1,106 @@ +#include "./tls_socket.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../log/log.hpp" + + +namespace anthracite::socket { + +tls_socket::tls_socket(int port, int max_queue) : anthracite_socket(port, max_queue), _handshakeDone(false) { +} + +void tls_socket::wait_for_conn() +{ + client_ip = ""; + client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); + std::array ip_str { 0 }; + inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN); + client_ip = std::string(ip_str.data()); +} + +void tls_socket::close_conn() +{ + close(client_socket); + client_socket = -1; +} + +void tls_socket::send_message(std::string& msg) +{ + if (client_socket == -1) { + return; + } + send(client_socket, &msg[0], msg.length(), 0); +} + +void tls_socket::perform_handshake() { + struct tls_msg_hdr hdr{}; + ssize_t result = recv(client_socket, &hdr, sizeof(hdr), 0); + + if (result < 1) { + return; + } + + log::info << "MsgType " << unsigned(hdr.msg_type); + log::info << " MsgLen " << hdr.length << std::endl; + + char hhdr[4]; + result = recv(client_socket, &hhdr, sizeof(hhdr), 0); + + if (result < 1) { + return; + } + + uint16_t msg_size = ClientHello::deserialize_uint16(hhdr + 2); + + log::debug << "TLS ClientHello Size: " << msg_size << std::endl; + + + char* client_hello_data = (char*) malloc(msg_size); + + result = recv(client_socket, client_hello_data, msg_size, 0); + + std::cout << result << " Bytes rxd" << std::endl; + + ClientHello hello_msg(client_hello_data, result); + + char *ptr; + ServerHello hello_retmsg(hello_msg.session_id); + int size = hello_retmsg.get_buf(&ptr); + log::debug << "Sending message of length " << size << std::endl; + send(client_socket, ptr , size, 0); + for(;;){} + _handshakeDone = true; +} + +std::string tls_socket::recv_message(int buffer_size) +{ + if (client_socket == -1) { + return ""; + } + + setsockopt(client_socket, SOL_SOCKET, SO_RCVTIMEO, &timeout_tv, sizeof timeout_tv); + + if (!_handshakeDone) { + perform_handshake(); + return ""; + } + + std::vector response(buffer_size + 1); + ssize_t result = recv(client_socket, response.data(), buffer_size + 1, 0); + + if (result < 1) { + return ""; + } + + response[buffer_size] = '\0'; + return { response.data() }; +} + +}; diff --git a/lib/socket/tls_socket.hpp b/lib/socket/tls_socket.hpp new file mode 100644 index 0000000..3eb09c7 --- /dev/null +++ b/lib/socket/tls_socket.hpp @@ -0,0 +1,235 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include "./socket.hpp" +#include +#include +#include +#include "../log/log.hpp" + +namespace anthracite::socket { + +constexpr uint32_t TLS_MSGHDR_RXSIZE = 4; + + +struct __attribute__((packed)) tls_version { + uint8_t major; + uint8_t minor; +}; + +struct __attribute__((packed)) tls_msg_hdr { + uint8_t msg_type; + tls_version version; + uint16_t length; +}; + +struct tls_extension { + uint16_t extension_type; + std::vector data; +}; + +class ServerHello { + public: + struct tls_version server_version; + std::array random_bytes; + std::array _session_id; + uint16_t cipher; + uint8_t compression; + + ServerHello(std::array session_id) { + srand(time(nullptr)); + server_version.major = 3; + server_version.minor = 3; + + for (int i = 0; i < 32; i++) { + random_bytes[i] = rand() % 256; + } + + _session_id = session_id; + + // TLS_RSA_WITH_NULL_MD5 + cipher = 1; + + // None + compression = 0; + } + + int get_buf(char** bufptr) { + constexpr int msgsize = 2 + 32 + 1 + 32 + 2 + 1 + 7; + constexpr int mmsgsize = msgsize + 4; + constexpr int bufsize = mmsgsize + 5; + + *bufptr = (char*) malloc(bufsize); + + char* buf = *bufptr; + + buf[0] = 0x16; + buf[1] = 3; + buf[2] = 1; + buf[3] = (mmsgsize >> 8) & 0xFF; + buf[4] = (mmsgsize) & 0xFF; + + + buf[5] = 0x02; + buf[6] = (msgsize >> 16) & 0xFF; + buf[7] = (msgsize >> 8) & 0xFF; + buf[8] = (msgsize) & 0xFF; + + buf[9] = server_version.major; + buf[10] = server_version.minor; + + for(int i = 0; i < 32; i++) { + buf[i+11] = random_bytes[i]; + } + + buf[43] = 32; + + for(int i = 0; i < 32; i++) { + buf[i+44] = _session_id[i]; + } + + // Cipher + buf[76] = 00; + buf[77] = 0x33; + + // Compression + buf[78] = 00; + + // Extensions Length + buf[79] = 00; + buf[80] = 01; + + // Renegotiation + buf[81] = 0xFF; + buf[82] = 01; + + // Disabled + buf[83] = 00; + buf[84] = 01; + buf[85] = 00; + + + return bufsize; + } + + + + +}; + + +class ClientHello { + public: + struct tls_version client_version; + std::array random_bytes; + std::array session_id; + std::vector cipher_suites; + std::vector compression_methods; + std::vector extensions; + + static uint32_t deserialize_uint32(char *buffer) + { + uint32_t value = 0; + + value |= buffer[0] << 24; + value |= buffer[1] << 16; + value |= buffer[2] << 8; + value |= buffer[3]; + return value; + } + + static uint32_t deserialize_uint24(char *buffer) + { + uint32_t value = 0; + + value |= buffer[0] << 16; + value |= buffer[1] << 8; + value |= buffer[2]; + return value; + } + + static uint16_t deserialize_uint16(char *buffer) + { + uint32_t value = 0; + + value |= buffer[0] << 8; + value |= buffer[1]; + + return value; + } + + + // TODO: Note that the security of this funciton is terrible and absolutely + // can cause nasty things to happen with a malformed message + ClientHello(char* buffer, ssize_t size) { + int bufptr = 0; + // Get version data + client_version.major = (uint8_t) buffer[bufptr++]; + client_version.minor = (uint8_t) buffer[bufptr++]; + log::debug << "TLS Version : maj " << unsigned(client_version.major) << " min " << unsigned(client_version.minor) << std::endl; + + log::debug << "TLS Random Data: "; + for(int i = 0; i < 32; i++) { + random_bytes[i] = buffer[bufptr++]; + log::debug << std::hex << unsigned(random_bytes[i]) << std::hex << " "; + } + log::debug << std::endl; + + // Get session id + int session_id_length = (uint8_t) buffer[bufptr++]; + log::debug << "TLS SesId Data : "; + for(int i = 0; i < session_id_length; i++) { + session_id[i] = buffer[bufptr++]; + log::debug << std::hex << unsigned(session_id[i]) << " "; + } + log::debug << std::dec; + log::debug << std::endl; + + // Get cipher suites + uint16_t cipher_suites_length = deserialize_uint16(&buffer[bufptr]); + bufptr += 2; + + log::debug << cipher_suites_length << " cipher suites supported" << std::endl; + + for(uint16_t i = 0; i < cipher_suites_length; i++) { + cipher_suites.push_back(deserialize_uint16(&buffer[bufptr])); + bufptr += 2; + } + + // Get compression methods + uint16_t compression_methods_length = buffer[bufptr++]; + + log::debug << compression_methods_length << " compression methods supported" << std::endl; + + for(uint16_t i = 0; i < compression_methods_length; i++) { + cipher_suites.push_back(buffer[bufptr]); + bufptr ++; + } + } +}; + +class tls_socket : anthracite_socket { + + + +private: + bool _handshakeDone; + + void perform_handshake(); + +public: + tls_socket(int port, int max_queue = MAX_QUEUE_LENGTH); + void wait_for_conn() override; + void close_conn() override; + void send_message(std::string& msg) override; + std::string recv_message(int buffer_size) override; +}; + +}; From da9f2f2d519434f024c80446c6cfee2bfefadc7a Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Mon, 17 Feb 2025 18:46:33 -0500 Subject: [PATCH 02/15] broken ssl --- CMakeLists.txt | 4 +++ lib/anthracite.cpp | 71 ++++++++++++++++++++++++++++++------------- lib/socket/socket.hpp | 18 ++++++----- src/file_main.cpp | 3 +- 4 files changed, 66 insertions(+), 30 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 949bb34..2449bbd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -7,6 +7,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_CXX_FLAGS_RELEASE "-O3") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) +find_package(OpenSSL REQUIRED) + add_custom_target(build-version COMMAND cd ../build_supp && ./version.sh DEPENDS build_supp/version.txt @@ -29,6 +31,8 @@ add_custom_target(run FILE(GLOB LIB_SOURCES lib/*.cpp lib/**/*.cpp build_supp/version.cpp) add_library(anthracite ${LIB_SOURCES}) add_dependencies(anthracite build-version) +target_link_libraries(anthracite OpenSSL::SSL OpenSSL::Crypto) +target_include_directories(anthracite PUBLIC ${OPENSSL_INCLUDE_DIR}) add_executable(anthracite-bin src/file_main.cpp) target_link_libraries(anthracite-bin anthracite) diff --git a/lib/anthracite.cpp b/lib/anthracite.cpp index 51a646e..d10cacb 100644 --- a/lib/anthracite.cpp +++ b/lib/anthracite.cpp @@ -11,6 +11,8 @@ #include #include #include +#include "./socket/openssl_socket.hpp" +#include using namespace anthracite; @@ -24,10 +26,10 @@ using std::chrono::duration_cast; using std::chrono::duration; using std::chrono::milliseconds; -void handle_client(socket::anthracite_socket s, backends::backend& b, backends::file_backend& fb, std::mutex& thread_wait_mutex, std::condition_variable& thread_wait_condvar, int& active_threads) +void handle_client(socket::anthracite_socket* s, backends::backend& b, backends::file_backend& fb, std::mutex& thread_wait_mutex, std::condition_variable& thread_wait_condvar, int& active_threads) { while (true) { - std::string raw_request = s.recv_message(http::HEADER_BYTES); + std::string raw_request = s->recv_message(http::HEADER_BYTES); // We're doing the start here even though it would ideally be done // before the first line since if we leave the connection open for @@ -38,11 +40,11 @@ void handle_client(socket::anthracite_socket s, backends::backend& b, backends:: break; } - http::request req(raw_request, s.get_client_ip()); + http::request req(raw_request, s->get_client_ip()); std::unique_ptr resp = req.is_supported_version() ? b.handle_request(req) : fb.handle_error(http::status_codes::HTTP_VERSION_NOT_SUPPORTED); std::string header = resp->header_to_string(); - s.send_message(header); - s.send_message(resp->content()); + s->send_message(header); + s->send_message(resp->content()); auto end = high_resolution_clock::now(); auto ms_int = duration_cast(end-start); @@ -53,7 +55,8 @@ void handle_client(socket::anthracite_socket s, backends::backend& b, backends:: break; } } - s.close_conn(); + s->close_conn(); + delete s; { std::lock_guard lock(thread_wait_mutex); active_threads--; @@ -61,34 +64,60 @@ void handle_client(socket::anthracite_socket s, backends::backend& b, backends:: thread_wait_condvar.notify_one(); } -int anthracite_main(int argc, char** argv, backends::backend& be) -{ - log::logger.initialize(log::LOG_LEVEL_INFO); - auto args = std::span(argv, size_t(argc)); - int port_number = default_port; +int listen_loop(int port_number, backends::backend& be, bool tls) { + socket::anthracite_socket* socket; - if (argc > 1) { - port_number = atoi(args[1]); + if (tls){ + socket = new socket::openssl_socket(port_number); + } else { + socket = new socket::anthracite_socket(port_number); } - log::verbose << "Initializing Anthracite" << std::endl; - socket::anthracite_socket s(port_number); - backends::file_backend fb(argc > 2 ? args[2] : "./www"); - log::verbose << "Initialization Complete" << std::endl; - log::info << "Listening for HTTP connections on port " << port_number << std::endl; + backends::file_backend fb("./www"); + log::info << "Listening for " << (tls ? "HTTPS" : "HTTP" ) << " connections on port " << port_number << std::endl; + int active_threads = 0; std::mutex thread_wait_mutex; std::condition_variable thread_wait_condvar; while (true) { - s.wait_for_conn(); + socket->wait_for_conn(); std::unique_lock lock(thread_wait_mutex); thread_wait_condvar.wait(lock, [active_threads] { return active_threads < max_worker_threads; }); active_threads++; - std::thread(handle_client, s, std::ref(be), std::ref(fb), std::ref(thread_wait_mutex), std::ref(thread_wait_condvar), std::ref(active_threads)).detach(); + + socket::anthracite_socket* client_sock; + + if (tls){ + client_sock = new socket::openssl_socket(*dynamic_cast(socket)); + } else { + client_sock = new socket::anthracite_socket(*socket); + } + + std::thread(handle_client, socket, std::ref(be), std::ref(fb), std::ref(thread_wait_mutex), std::ref(thread_wait_condvar), std::ref(active_threads)).detach(); } - exit(0); + delete socket; +} + +int anthracite_main(int argc, char** argv, backends::backend& be) +{ + log::logger.initialize(log::LOG_LEVEL_INFO); + auto args = std::span(argv, size_t(argc)); + + std::vector threads; + + if (argc > 1) { + auto thread = std::thread(listen_loop, atoi(argv[1]), std::ref(be), false); + thread.detach(); + threads.push_back(std::move(thread)); + } if (argc > 2) { + auto thread = std::thread(listen_loop, atoi(argv[2]), std::ref(be), true); + thread.detach(); + threads.push_back(std::move(thread)); + } + std::promise().get_future().wait(); + return 0; } void log_request_and_response(http::request& req, std::unique_ptr& resp, uint32_t micros) diff --git a/lib/socket/socket.hpp b/lib/socket/socket.hpp index 7a57de7..648c404 100644 --- a/lib/socket/socket.hpp +++ b/lib/socket/socket.hpp @@ -1,3 +1,5 @@ +#pragma once + #include #include #include @@ -10,23 +12,25 @@ namespace anthracite::socket { class anthracite_socket { - static const int MAX_QUEUE_LENGTH = 100; -private: + +protected: int server_socket; int client_socket {}; std::string client_ip; struct sockaddr_in client_addr {}; socklen_t client_addr_len {}; static const struct timeval timeout_tv; + static const int MAX_QUEUE_LENGTH = 100; public: anthracite_socket(int port, int max_queue = MAX_QUEUE_LENGTH); - void wait_for_conn(); - const std::string& get_client_ip(); - void close_conn(); - void send_message(std::string& msg); - std::string recv_message(int buffer_size); + virtual const std::string& get_client_ip() final; + + virtual void wait_for_conn(); + virtual void close_conn(); + virtual void send_message(std::string& msg); + virtual std::string recv_message(int buffer_size); }; }; diff --git a/src/file_main.cpp b/src/file_main.cpp index c81b1e9..2af7664 100644 --- a/src/file_main.cpp +++ b/src/file_main.cpp @@ -5,7 +5,6 @@ using namespace anthracite; int main(int argc, char** argv) { - auto args = std::span(argv, size_t(argc)); - backends::file_backend fb(argc > 2 ? args[2] : "./www"); + backends::file_backend fb("./www"); anthracite_main(argc, argv, fb); } From a63d9d1e650de64cc8c7a3a419416a6f29d70492 Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Mon, 17 Feb 2025 18:46:52 -0500 Subject: [PATCH 03/15] broken ssl --- lib/socket/openssl_socket.cpp | 100 ++++++++++++++++++++++++++++++++++ lib/socket/openssl_socket.hpp | 22 ++++++++ 2 files changed, 122 insertions(+) create mode 100644 lib/socket/openssl_socket.cpp create mode 100644 lib/socket/openssl_socket.hpp diff --git a/lib/socket/openssl_socket.cpp b/lib/socket/openssl_socket.cpp new file mode 100644 index 0000000..6127a00 --- /dev/null +++ b/lib/socket/openssl_socket.cpp @@ -0,0 +1,100 @@ +#include +#include +#include +#include "./openssl_socket.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "../log/log.hpp" + +namespace anthracite::socket { + +SSL_CTX* openssl_socket::_context = nullptr; + +openssl_socket::openssl_socket(int port, int max_queue) + : anthracite_socket(port, max_queue) +{ + const SSL_METHOD *method = TLS_server_method(); + + if (_context == nullptr) { + _context = SSL_CTX_new(method); + } + + if (!_context) { + log::err << "Unable to initialize SSL" << std::endl; + throw std::exception(); + } + + if (SSL_CTX_use_certificate_file(_context, "cert.pem", SSL_FILETYPE_PEM) <= 0) { + log::err << "Unable to open cert.pem" << std::endl; + throw std::exception(); + } + + if (SSL_CTX_use_PrivateKey_file(_context, "key.pem", SSL_FILETYPE_PEM) <= 0 ) { + log::err << "Unable to open key.pem" << std::endl; + throw std::exception(); + } +} + +openssl_socket::~openssl_socket() = default; + +void openssl_socket::wait_for_conn() +{ + client_ip = ""; + client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); + std::array ip_str { 0 }; + inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN); + client_ip = std::string(ip_str.data()); + + _ssl = SSL_new(_context); + SSL_set_fd(_ssl, client_socket); + if (SSL_accept(_ssl) <= 0) { + log::warn << "Unable to open SSL connection with client" << std::endl; + client_ip = ""; + close(client_socket); + client_socket = -1; + } +} + +void openssl_socket::close_conn() +{ + SSL_shutdown(_ssl); + SSL_free(_ssl); + close(client_socket); + client_socket = -1; +} + +void openssl_socket::send_message(std::string& msg) +{ + if (client_socket == -1) { + return; + } + SSL_write(_ssl, &msg[0], msg.length()); +} + +std::string openssl_socket::recv_message(int buffer_size) +{ + if (client_socket == -1) { + return ""; + } + + setsockopt(client_socket, SOL_SOCKET, SO_RCVTIMEO, &timeout_tv, sizeof timeout_tv); + std::vector response(buffer_size + 1); + ssize_t result = SSL_read(_ssl, response.data(), buffer_size+1); + + if (result < 1) { + return ""; + } + + response[buffer_size] = '\0'; + return { response.data() }; +} + +}; diff --git a/lib/socket/openssl_socket.hpp b/lib/socket/openssl_socket.hpp new file mode 100644 index 0000000..2d79c9c --- /dev/null +++ b/lib/socket/openssl_socket.hpp @@ -0,0 +1,22 @@ +#pragma once + +#include "./socket.hpp" +#include +#include + +namespace anthracite::socket { +class openssl_socket : public anthracite_socket { + private: + static SSL_CTX* _context; + SSL* _ssl; + + public: + openssl_socket(int port, int max_queue = MAX_QUEUE_LENGTH); + ~openssl_socket(); + + void wait_for_conn() override; + void close_conn() override; + void send_message(std::string& msg) override; + std::string recv_message(int buffer_size) override; +}; +}; From f1195d1f04061a17f8f68c4aafdf9099be35269d Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Wed, 19 Feb 2025 23:54:15 -0500 Subject: [PATCH 04/15] add shell.nix --- .envrc | 1 + shell.nix | 11 +++++++++++ src/api_main.cpp | 1 + 3 files changed, 13 insertions(+) create mode 100644 .envrc create mode 100644 shell.nix diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..1d953f4 --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use nix diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000..e9a5a69 --- /dev/null +++ b/shell.nix @@ -0,0 +1,11 @@ +{ pkgs ? import {} }: + pkgs.mkShell { + nativeBuildInputs = [ pkgs.pkg-config pkgs.openssl pkgs.libgcc pkgs.boost pkgs.cmake ]; + + shellHook = '' + export OPENSSL_DIR="${pkgs.openssl.dev}" + export PKG_CONFIG_PATH="${pkgs.openssl.dev}/lib/pkgconfig" + export OPENSSL_NO_VENDOR=1 +export OPENSSL_LIB_DIR="${pkgs.lib.getLib pkgs.openssl}/lib" + ''; +} diff --git a/src/api_main.cpp b/src/api_main.cpp index 4a42858..e7b5366 100644 --- a/src/api_main.cpp +++ b/src/api_main.cpp @@ -7,6 +7,7 @@ #include #include #include +#include using namespace anthracite; From f09b261b6241f66ea364e928ae20a5f19dcbc4a3 Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Thu, 20 Feb 2025 13:48:30 -0500 Subject: [PATCH 05/15] ssl fix --- lib/anthracite.cpp | 5 +++-- shell.nix | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/anthracite.cpp b/lib/anthracite.cpp index d10cacb..f1cf31c 100644 --- a/lib/anthracite.cpp +++ b/lib/anthracite.cpp @@ -89,12 +89,13 @@ int listen_loop(int port_number, backends::backend& be, bool tls) { socket::anthracite_socket* client_sock; if (tls){ - client_sock = new socket::openssl_socket(*dynamic_cast(socket)); + socket::openssl_socket* ssl_sock = dynamic_cast(socket); + client_sock = new socket::openssl_socket(*ssl_sock); } else { client_sock = new socket::anthracite_socket(*socket); } - std::thread(handle_client, socket, std::ref(be), std::ref(fb), std::ref(thread_wait_mutex), std::ref(thread_wait_condvar), std::ref(active_threads)).detach(); + std::thread(handle_client, client_sock, std::ref(be), std::ref(fb), std::ref(thread_wait_mutex), std::ref(thread_wait_condvar), std::ref(active_threads)).detach(); } delete socket; diff --git a/shell.nix b/shell.nix index e9a5a69..201b9ef 100644 --- a/shell.nix +++ b/shell.nix @@ -6,6 +6,6 @@ export OPENSSL_DIR="${pkgs.openssl.dev}" export PKG_CONFIG_PATH="${pkgs.openssl.dev}/lib/pkgconfig" export OPENSSL_NO_VENDOR=1 -export OPENSSL_LIB_DIR="${pkgs.lib.getLib pkgs.openssl}/lib" + export OPENSSL_LIB_DIR="${pkgs.lib.getLib pkgs.openssl}/lib" ''; } From ca05aa1e5ab95c9e8295a30d755ae2c46d848f44 Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Fri, 21 Feb 2025 14:09:01 -0500 Subject: [PATCH 06/15] event loop with threadmgr --- CHANGELOG.md | 2 + lib/anthracite.cpp | 115 +++++----------------------------- lib/anthracite.hpp | 4 +- lib/socket/openssl_socket.cpp | 37 +++++++---- lib/socket/openssl_socket.hpp | 2 +- lib/socket/socket.cpp | 19 ++++-- lib/socket/socket.hpp | 3 +- src/api_main.cpp | 2 +- src/file_main.cpp | 7 ++- 9 files changed, 69 insertions(+), 122 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f50b57..b26e941 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,6 @@ # 0.3.0 +- SSL support via OpenSSL +- Switched from thread per connection to event driven threading model - Rewrote request parser for readability and speed - Added improved logging with different log levels - Separated anthracite into libanthracite and anthracite-bin to allow for other projects to implement anthracite (example in ./src/api_main.cpp) diff --git a/lib/anthracite.cpp b/lib/anthracite.cpp index f1cf31c..450deaf 100644 --- a/lib/anthracite.cpp +++ b/lib/anthracite.cpp @@ -1,18 +1,12 @@ #include "./anthracite.hpp" #include "./log/log.hpp" -#include "./socket/socket.hpp" -#include "backends/file_backend.hpp" -#include -#include #include -#include #include -#include #include -#include #include -#include "./socket/openssl_socket.hpp" -#include +#include "./config/config.hpp" +#include "./thread_mgr/event_loop.hpp" +#include using namespace anthracite; @@ -21,103 +15,26 @@ void log_request_and_response(http::request& req, std::unique_ptrrecv_message(http::HEADER_BYTES); - - // We're doing the start here even though it would ideally be done - // before the first line since if we leave the connection open for - // HTTP 1.1, we can spend a bit of time waiting - auto start = high_resolution_clock::now(); - if (raw_request == "") { - break; - } - - http::request req(raw_request, s->get_client_ip()); - std::unique_ptr resp = req.is_supported_version() ? b.handle_request(req) : fb.handle_error(http::status_codes::HTTP_VERSION_NOT_SUPPORTED); - std::string header = resp->header_to_string(); - s->send_message(header); - s->send_message(resp->content()); - - auto end = high_resolution_clock::now(); - auto ms_int = duration_cast(end-start); - log_request_and_response(req, resp , ms_int.count()); - - resp.reset(); - if (req.close_connection()) { - break; - } - } - s->close_conn(); - delete s; - { - std::lock_guard lock(thread_wait_mutex); - active_threads--; - } - thread_wait_condvar.notify_one(); +extern "C" void signalHandler(int signum) { + log::warn << "Caught signal #" << signum << ", exiting Anthracite" << std::endl; + elp->stop(); } -int listen_loop(int port_number, backends::backend& be, bool tls) { - socket::anthracite_socket* socket; - if (tls){ - socket = new socket::openssl_socket(port_number); - } else { - socket = new socket::anthracite_socket(port_number); - } - - backends::file_backend fb("./www"); - log::info << "Listening for " << (tls ? "HTTPS" : "HTTP" ) << " connections on port " << port_number << std::endl; - - int active_threads = 0; - std::mutex thread_wait_mutex; - std::condition_variable thread_wait_condvar; - - while (true) { - socket->wait_for_conn(); - std::unique_lock lock(thread_wait_mutex); - thread_wait_condvar.wait(lock, [active_threads] { return active_threads < max_worker_threads; }); - active_threads++; - - socket::anthracite_socket* client_sock; - - if (tls){ - socket::openssl_socket* ssl_sock = dynamic_cast(socket); - client_sock = new socket::openssl_socket(*ssl_sock); - } else { - client_sock = new socket::anthracite_socket(*socket); - } - - std::thread(handle_client, client_sock, std::ref(be), std::ref(fb), std::ref(thread_wait_mutex), std::ref(thread_wait_condvar), std::ref(active_threads)).detach(); - } - - delete socket; -} - -int anthracite_main(int argc, char** argv, backends::backend& be) +int anthracite_main(backends::backend& be, config::config& config) { + signal(SIGTERM, signalHandler); + signal(SIGSEGV, signalHandler); + signal(SIGINT, signalHandler); + signal(SIGABRT, signalHandler); + log::logger.initialize(log::LOG_LEVEL_INFO); - auto args = std::span(argv, size_t(argc)); - - std::vector threads; - - if (argc > 1) { - auto thread = std::thread(listen_loop, atoi(argv[1]), std::ref(be), false); - thread.detach(); - threads.push_back(std::move(thread)); - } if (argc > 2) { - auto thread = std::thread(listen_loop, atoi(argv[2]), std::ref(be), true); - thread.detach(); - threads.push_back(std::move(thread)); - } - std::promise().get_future().wait(); + thread_mgr::event_loop el(be, config); + elp = ⪙ + el.start(); return 0; } diff --git a/lib/anthracite.hpp b/lib/anthracite.hpp index a4f6d2c..f342344 100644 --- a/lib/anthracite.hpp +++ b/lib/anthracite.hpp @@ -1,4 +1,6 @@ #include "backends/backend.hpp" +#include "config/config.hpp" using namespace anthracite; -int anthracite_main(int argc, char** argv, backends::backend& be); + +int anthracite_main(backends::backend& be, config::config& cfg); diff --git a/lib/socket/openssl_socket.cpp b/lib/socket/openssl_socket.cpp index 6127a00..eddf655 100644 --- a/lib/socket/openssl_socket.cpp +++ b/lib/socket/openssl_socket.cpp @@ -45,22 +45,33 @@ openssl_socket::openssl_socket(int port, int max_queue) openssl_socket::~openssl_socket() = default; -void openssl_socket::wait_for_conn() +bool openssl_socket::wait_for_conn() { client_ip = ""; - client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); - std::array ip_str { 0 }; - inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN); - client_ip = std::string(ip_str.data()); - - _ssl = SSL_new(_context); - SSL_set_fd(_ssl, client_socket); - if (SSL_accept(_ssl) <= 0) { - log::warn << "Unable to open SSL connection with client" << std::endl; - client_ip = ""; - close(client_socket); - client_socket = -1; + struct timeval tv = {.tv_sec = 1, .tv_usec = 0}; + fd_set read_fd; + FD_ZERO(&read_fd); + FD_SET(server_socket, &read_fd); + if (select(server_socket+1, &read_fd, NULL, NULL, &wait_timeout)) { + client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); + std::array ip_str { 0 }; + inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN); + client_ip = std::string(ip_str.data()); + _ssl = SSL_new(_context); + SSL_set_fd(_ssl, client_socket); + if (SSL_accept(_ssl) <= 0) { + log::warn << "Unable to open SSL connection with client" << std::endl; + client_ip = ""; + close(client_socket); + client_socket = -1; + return false; + } + return true; + } else { + return false; } + + } void openssl_socket::close_conn() diff --git a/lib/socket/openssl_socket.hpp b/lib/socket/openssl_socket.hpp index 2d79c9c..4d4df6f 100644 --- a/lib/socket/openssl_socket.hpp +++ b/lib/socket/openssl_socket.hpp @@ -14,7 +14,7 @@ class openssl_socket : public anthracite_socket { openssl_socket(int port, int max_queue = MAX_QUEUE_LENGTH); ~openssl_socket(); - void wait_for_conn() override; + bool wait_for_conn() override; void close_conn() override; void send_message(std::string& msg) override; std::string recv_message(int buffer_size) override; diff --git a/lib/socket/socket.cpp b/lib/socket/socket.cpp index 720f668..5a54ac1 100644 --- a/lib/socket/socket.cpp +++ b/lib/socket/socket.cpp @@ -29,13 +29,22 @@ anthracite_socket::anthracite_socket(int port, int max_queue) listen(server_socket, max_queue); } -void anthracite_socket::wait_for_conn() +bool anthracite_socket::wait_for_conn() { client_ip = ""; - client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); - std::array ip_str { 0 }; - inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN); - client_ip = std::string(ip_str.data()); + struct timeval tv = {.tv_sec = 1, .tv_usec = 0}; + fd_set read_fd; + FD_ZERO(&read_fd); + FD_SET(server_socket, &read_fd); + if (select(server_socket+1, &read_fd, NULL, NULL, &wait_timeout)) { + client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); + std::array ip_str { 0 }; + inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN); + client_ip = std::string(ip_str.data()); + return true; + } else { + return false; + } } const std::string& anthracite_socket::get_client_ip() diff --git a/lib/socket/socket.hpp b/lib/socket/socket.hpp index 648c404..5df0fde 100644 --- a/lib/socket/socket.hpp +++ b/lib/socket/socket.hpp @@ -14,6 +14,7 @@ namespace anthracite::socket { class anthracite_socket { protected: + struct timeval wait_timeout = { .tv_sec = 1, .tv_usec = 0}; int server_socket; int client_socket {}; std::string client_ip; @@ -27,7 +28,7 @@ public: virtual const std::string& get_client_ip() final; - virtual void wait_for_conn(); + virtual bool wait_for_conn(); virtual void close_conn(); virtual void send_message(std::string& msg); virtual std::string recv_message(int buffer_size); diff --git a/src/api_main.cpp b/src/api_main.cpp index e7b5366..54d7b3b 100644 --- a/src/api_main.cpp +++ b/src/api_main.cpp @@ -108,5 +108,5 @@ int main(int argc, char** argv) auto args = std::span(argv, size_t(argc)); api_backend ab; ab.register_endpoint("users/*", handle_request); - anthracite_main(argc, argv, ab); + //anthracite_main(argc, argv, ab); } diff --git a/src/file_main.cpp b/src/file_main.cpp index 2af7664..7a723a9 100644 --- a/src/file_main.cpp +++ b/src/file_main.cpp @@ -1,10 +1,15 @@ #include "../lib/anthracite.hpp" #include "../lib/backends/file_backend.hpp" +#include "../lib/config/config.hpp" using namespace anthracite; int main(int argc, char** argv) { backends::file_backend fb("./www"); - anthracite_main(argc, argv, fb); + config::config cfg(5); + cfg.add_http_config(config::http_config(8080)); + cfg.add_https_config(config::https_config(8081, "", "")); + + anthracite_main(fb, cfg); } From 058c3950953a7480bbd7c1ad10ce2f8763ee6985 Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Fri, 21 Feb 2025 14:09:16 -0500 Subject: [PATCH 07/15] event loop with threadmgr --- .direnv/bin/nix-direnv-reload | 19 + .direnv/nix-profile-24.11-mma8n3yfap91nw44 | 1 + .direnv/nix-profile-24.11-mma8n3yfap91nw44.rc | 2182 +++++++++++++++++ lib/config/config.hpp | 54 + lib/thread_mgr/event_loop.cpp | 176 ++ lib/thread_mgr/event_loop.hpp | 31 + lib/thread_mgr/thread_mgr.hpp | 18 + 7 files changed, 2481 insertions(+) create mode 100755 .direnv/bin/nix-direnv-reload create mode 120000 .direnv/nix-profile-24.11-mma8n3yfap91nw44 create mode 100644 .direnv/nix-profile-24.11-mma8n3yfap91nw44.rc create mode 100644 lib/config/config.hpp create mode 100644 lib/thread_mgr/event_loop.cpp create mode 100644 lib/thread_mgr/event_loop.hpp create mode 100644 lib/thread_mgr/thread_mgr.hpp diff --git a/.direnv/bin/nix-direnv-reload b/.direnv/bin/nix-direnv-reload new file mode 100755 index 0000000..3bd21d1 --- /dev/null +++ b/.direnv/bin/nix-direnv-reload @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -e +if [[ ! -d "/home/nickorlow/programming/personal/anthracite" ]]; then + echo "Cannot find source directory; Did you move it?" + echo "(Looking for "/home/nickorlow/programming/personal/anthracite")" + echo 'Cannot force reload with this script - use "direnv reload" manually and then try again' + exit 1 +fi + +# rebuild the cache forcefully +_nix_direnv_force_reload=1 direnv exec "/home/nickorlow/programming/personal/anthracite" true + +# Update the mtime for .envrc. +# This will cause direnv to reload again - but without re-building. +touch "/home/nickorlow/programming/personal/anthracite/.envrc" + +# Also update the timestamp of whatever profile_rc we have. +# This makes sure that we know we are up to date. +touch -r "/home/nickorlow/programming/personal/anthracite/.envrc" "/home/nickorlow/programming/personal/anthracite/.direnv"/*.rc diff --git a/.direnv/nix-profile-24.11-mma8n3yfap91nw44 b/.direnv/nix-profile-24.11-mma8n3yfap91nw44 new file mode 120000 index 0000000..34f60a5 --- /dev/null +++ b/.direnv/nix-profile-24.11-mma8n3yfap91nw44 @@ -0,0 +1 @@ +/nix/store/w578af1s1zz4s6s3q2mhr5m2x7jq0cpi-nix-shell-env \ No newline at end of file diff --git a/.direnv/nix-profile-24.11-mma8n3yfap91nw44.rc b/.direnv/nix-profile-24.11-mma8n3yfap91nw44.rc new file mode 100644 index 0000000..2ac7bb9 --- /dev/null +++ b/.direnv/nix-profile-24.11-mma8n3yfap91nw44.rc @@ -0,0 +1,2182 @@ +unset shellHook +PATH=${PATH:-} +nix_saved_PATH="$PATH" +XDG_DATA_DIRS=${XDG_DATA_DIRS:-} +nix_saved_XDG_DATA_DIRS="$XDG_DATA_DIRS" +AR='ar' +export AR +AS='as' +export AS +BASH='/nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37/bin/bash' +CC='gcc' +export CC +CMAKE_INCLUDE_PATH='/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/include:/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/include' +export CMAKE_INCLUDE_PATH +CMAKE_LIBRARY_PATH='/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2/lib:/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc/lib:/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0/lib' +export CMAKE_LIBRARY_PATH +CONFIG_SHELL='/nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37/bin/bash' +export CONFIG_SHELL +CXX='g++' +export CXX +HOSTTYPE='x86_64' +HOST_PATH='/nix/store/6wgd8c9vq93mqxzc7jhkl86mv6qbc360-coreutils-9.5/bin:/nix/store/r99d2m4swgmrv9jvm4l9di40hvanq1aq-findutils-4.10.0/bin:/nix/store/3sln66ij8pg114apkd8p6nr04y37q5z2-diffutils-3.10/bin:/nix/store/yq39xdwm4z0fhx7dsm8mlpgvcz3vbfg3-gnused-4.9/bin:/nix/store/vniy1y5n8g28c55y7788npwc4h09fh7c-gnugrep-3.11/bin:/nix/store/scgfwh3z1s3l2vhvyjsgfgx5ql552sls-gawk-5.3.1/bin:/nix/store/0wqn2k3v5xzrc9rwinijdyr2ywwl82x4-gnutar-1.35/bin:/nix/store/5y240z436gf3rwmkwbhn1a17pqw509w4-gzip-1.13/bin:/nix/store/1yyryxp7mh7zsciapi8f1n0mnxkigmf8-bzip2-1.0.8-bin/bin:/nix/store/hbzw8k8ygv6bfzvsvnd4gb8qmm8xjbvn-gnumake-4.4.1/bin:/nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37/bin:/nix/store/rfrjws98w6scfx7m63grb0m6sg925ahd-patch-2.7.6/bin:/nix/store/4i4mjaf7z6gddspar487grxk5k1j4dcd-xz-5.6.3-bin/bin:/nix/store/9wbpsj6ksd16x1qdqs29xli1dpz3fnl0-file-5.45/bin' +export HOST_PATH +IFS=' +' +IN_NIX_SHELL='impure' +export IN_NIX_SHELL +LD='ld' +export LD +LINENO='76' +MACHTYPE='x86_64-pc-linux-gnu' +NIXPKGS_CMAKE_PREFIX_PATH='/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2:/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev:/nix/store/rc5j45brxiyl55fgd3adbcc43pdfr29v-openssl-3.3.2-bin:/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2:/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc:/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev:/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0:/nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5:/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0:/nix/store/ap724yhgv28mpsi1mmqcwypj4rrfhqmg-update-autotools-gnu-config-scripts-hook:/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0:/nix/store/qlzvmgr8w9prdlyys7irqf86p7bndf5b-binutils-wrapper-2.43.1' +export NIXPKGS_CMAKE_PREFIX_PATH +NIX_BINTOOLS='/nix/store/qlzvmgr8w9prdlyys7irqf86p7bndf5b-binutils-wrapper-2.43.1' +export NIX_BINTOOLS +NIX_BINTOOLS_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu='1' +export NIX_BINTOOLS_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu +NIX_BUILD_CORES='12' +export NIX_BUILD_CORES +NIX_CC='/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0' +export NIX_CC +NIX_CC_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu='1' +export NIX_CC_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu +NIX_CFLAGS_COMPILE=' -frandom-seed=w578af1s1z -isystem /nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/include -isystem /nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/include -isystem /nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/include -isystem /nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/include' +export NIX_CFLAGS_COMPILE +NIX_ENFORCE_NO_NATIVE='1' +export NIX_ENFORCE_NO_NATIVE +NIX_HARDENING_ENABLE='bindnow format fortify fortify3 pic relro stackprotector strictoverflow zerocallusedregs' +export NIX_HARDENING_ENABLE +NIX_LDFLAGS='-rpath /home/nickorlow/programming/personal/anthracite/outputs/out/lib -L/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2/lib -L/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc/lib -L/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0/lib -L/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2/lib -L/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc/lib -L/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0/lib' +export NIX_LDFLAGS +NIX_NO_SELF_RPATH='1' +NIX_PKG_CONFIG_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu='1' +export NIX_PKG_CONFIG_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu +NIX_STORE='/nix/store' +export NIX_STORE +NM='nm' +export NM +OBJCOPY='objcopy' +export OBJCOPY +OBJDUMP='objdump' +export OBJDUMP +OLDPWD='' +export OLDPWD +OPTERR='1' +OSTYPE='linux-gnu' +PATH='/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2/bin:/nix/store/rc5j45brxiyl55fgd3adbcc43pdfr29v-openssl-3.3.2-bin/bin:/nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5/bin:/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0/bin:/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0/bin:/nix/store/62qjb50708fdhb4f2y7zxyqr1afir4fk-gcc-13.3.0/bin:/nix/store/29mb4q8b5306f4gk2wh38h0c1akb0n97-glibc-2.40-36-bin/bin:/nix/store/6wgd8c9vq93mqxzc7jhkl86mv6qbc360-coreutils-9.5/bin:/nix/store/qlzvmgr8w9prdlyys7irqf86p7bndf5b-binutils-wrapper-2.43.1/bin:/nix/store/vk4mlknqk9yjbqa68a7rvpfxfdw3rad7-binutils-2.43.1/bin:/nix/store/6wgd8c9vq93mqxzc7jhkl86mv6qbc360-coreutils-9.5/bin:/nix/store/r99d2m4swgmrv9jvm4l9di40hvanq1aq-findutils-4.10.0/bin:/nix/store/3sln66ij8pg114apkd8p6nr04y37q5z2-diffutils-3.10/bin:/nix/store/yq39xdwm4z0fhx7dsm8mlpgvcz3vbfg3-gnused-4.9/bin:/nix/store/vniy1y5n8g28c55y7788npwc4h09fh7c-gnugrep-3.11/bin:/nix/store/scgfwh3z1s3l2vhvyjsgfgx5ql552sls-gawk-5.3.1/bin:/nix/store/0wqn2k3v5xzrc9rwinijdyr2ywwl82x4-gnutar-1.35/bin:/nix/store/5y240z436gf3rwmkwbhn1a17pqw509w4-gzip-1.13/bin:/nix/store/1yyryxp7mh7zsciapi8f1n0mnxkigmf8-bzip2-1.0.8-bin/bin:/nix/store/hbzw8k8ygv6bfzvsvnd4gb8qmm8xjbvn-gnumake-4.4.1/bin:/nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37/bin:/nix/store/rfrjws98w6scfx7m63grb0m6sg925ahd-patch-2.7.6/bin:/nix/store/4i4mjaf7z6gddspar487grxk5k1j4dcd-xz-5.6.3-bin/bin:/nix/store/9wbpsj6ksd16x1qdqs29xli1dpz3fnl0-file-5.45/bin' +export PATH +PKG_CONFIG='pkg-config' +export PKG_CONFIG +PKG_CONFIG_PATH='/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/lib/pkgconfig:/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/lib/pkgconfig' +export PKG_CONFIG_PATH +PS4='+ ' +RANLIB='ranlib' +export RANLIB +READELF='readelf' +export READELF +SHELL='/nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37/bin/bash' +export SHELL +SIZE='size' +export SIZE +SOURCE_DATE_EPOCH='315532800' +export SOURCE_DATE_EPOCH +STRINGS='strings' +export STRINGS +STRIP='strip' +export STRIP +XDG_DATA_DIRS='/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2/share:/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/share:/nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5/share:/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0/share' +export XDG_DATA_DIRS +__structuredAttrs='' +export __structuredAttrs +_substituteStream_has_warned_replace_deprecation='false' +buildInputs='' +export buildInputs +buildPhase='{ echo "------------------------------------------------------------"; + echo " WARNING: the existence of this path is not guaranteed."; + echo " It is an internal implementation detail for pkgs.mkShell."; + echo "------------------------------------------------------------"; + echo; + # Record all build inputs as runtime dependencies + export; +} >> "$out" +' +export buildPhase +builder='/nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37/bin/bash' +export builder +cmakeFlags='' +export cmakeFlags +configureFlags='' +export configureFlags +configurePhase='cmakeConfigurePhase' +defaultBuildInputs='' +defaultNativeBuildInputs='/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0 /nix/store/ap724yhgv28mpsi1mmqcwypj4rrfhqmg-update-autotools-gnu-config-scripts-hook /nix/store/h9lc1dpi14z7is86ffhl3ld569138595-audit-tmpdir.sh /nix/store/m54bmrhj6fqz8nds5zcj97w9s9bckc9v-compress-man-pages.sh /nix/store/wgrbkkaldkrlrni33ccvm3b6vbxzb656-make-symlinks-relative.sh /nix/store/5yzw0vhkyszf2d179m0qfkgxmp5wjjx4-move-docs.sh /nix/store/fyaryjvghbkpfnsyw97hb3lyb37s1pd6-move-lib64.sh /nix/store/kd4xwxjpjxi71jkm6ka0np72if9rm3y0-move-sbin.sh /nix/store/pag6l61paj1dc9sv15l7bm5c17xn5kyk-move-systemd-user-units.sh /nix/store/jivxp510zxakaaic7qkrb7v1dd2rdbw9-multiple-outputs.sh /nix/store/12lvf0c7xric9cny7slvf9cmhypl1p67-patch-shebangs.sh /nix/store/cickvswrvann041nqxb0rxilc46svw1n-prune-libtool-files.sh /nix/store/xyff06pkhki3qy1ls77w10s0v79c9il0-reproducible-builds.sh /nix/store/aazf105snicrlvyzzbdj85sx4179rpfp-set-source-date-epoch-to-latest.sh /nix/store/gps9qrh99j7g02840wv5x78ykmz30byp-strip.sh /nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0' +depsBuildBuild='' +export depsBuildBuild +depsBuildBuildPropagated='' +export depsBuildBuildPropagated +depsBuildTarget='' +export depsBuildTarget +depsBuildTargetPropagated='' +export depsBuildTargetPropagated +depsHostHost='' +export depsHostHost +depsHostHostPropagated='' +export depsHostHostPropagated +depsTargetTarget='' +export depsTargetTarget +depsTargetTargetPropagated='' +export depsTargetTargetPropagated +doCheck='' +export doCheck +doInstallCheck='' +export doInstallCheck +dontAddDisableDepTrack='1' +export dontAddDisableDepTrack +declare -a envBuildBuildHooks=() +declare -a envBuildHostHooks=() +declare -a envBuildTargetHooks=() +declare -a envHostHostHooks=('pkgConfigWrapper_addPkgConfigPath' 'addCMakeParams' 'ccWrapper_addCVars' 'bintoolsWrapper_addLDVars' ) +declare -a envHostTargetHooks=('pkgConfigWrapper_addPkgConfigPath' 'addCMakeParams' 'ccWrapper_addCVars' 'bintoolsWrapper_addLDVars' ) +declare -a envTargetTargetHooks=() +declare -a fixupOutputHooks=('if [ -z "${dontPatchELF-}" ]; then patchELF "$prefix"; fi' 'if [[ -z "${noAuditTmpdir-}" && -e "$prefix" ]]; then auditTmpdir "$prefix"; fi' 'if [ -z "${dontGzipMan-}" ]; then compressManPages "$prefix"; fi' '_moveLib64' '_moveSbin' '_moveSystemdUserUnits' 'patchShebangsAuto' '_pruneLibtoolFiles' '_doStrip' ) +flag='-L/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0/lib' +guess='12' +iframework_seen='' +initialPath='/nix/store/6wgd8c9vq93mqxzc7jhkl86mv6qbc360-coreutils-9.5 /nix/store/r99d2m4swgmrv9jvm4l9di40hvanq1aq-findutils-4.10.0 /nix/store/3sln66ij8pg114apkd8p6nr04y37q5z2-diffutils-3.10 /nix/store/yq39xdwm4z0fhx7dsm8mlpgvcz3vbfg3-gnused-4.9 /nix/store/vniy1y5n8g28c55y7788npwc4h09fh7c-gnugrep-3.11 /nix/store/scgfwh3z1s3l2vhvyjsgfgx5ql552sls-gawk-5.3.1 /nix/store/0wqn2k3v5xzrc9rwinijdyr2ywwl82x4-gnutar-1.35 /nix/store/5y240z436gf3rwmkwbhn1a17pqw509w4-gzip-1.13 /nix/store/1yyryxp7mh7zsciapi8f1n0mnxkigmf8-bzip2-1.0.8-bin /nix/store/hbzw8k8ygv6bfzvsvnd4gb8qmm8xjbvn-gnumake-4.4.1 /nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37 /nix/store/rfrjws98w6scfx7m63grb0m6sg925ahd-patch-2.7.6 /nix/store/4i4mjaf7z6gddspar487grxk5k1j4dcd-xz-5.6.3-bin /nix/store/9wbpsj6ksd16x1qdqs29xli1dpz3fnl0-file-5.45' +isystem_seen='' +mesonFlags='' +export mesonFlags +name='nix-shell-env' +export name +nativeBuildInputs='/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2 /nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev /nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc /nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev /nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5' +export nativeBuildInputs +out='/home/nickorlow/programming/personal/anthracite/outputs/out' +export out +outputBin='out' +outputDev='out' +outputDevdoc='REMOVE' +outputDevman='out' +outputDoc='out' +outputInclude='out' +outputInfo='out' +outputLib='out' +outputMan='out' +outputs='out' +export outputs +patches='' +export patches +phases='buildPhase' +export phases +pkg='/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0' +declare -a pkgsBuildBuild=() +declare -a pkgsBuildHost=('/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2' '/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev' '/nix/store/rc5j45brxiyl55fgd3adbcc43pdfr29v-openssl-3.3.2-bin' '/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2' '/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc' '/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev' '/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0' '/nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5' '/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0' '/nix/store/ap724yhgv28mpsi1mmqcwypj4rrfhqmg-update-autotools-gnu-config-scripts-hook' '/nix/store/h9lc1dpi14z7is86ffhl3ld569138595-audit-tmpdir.sh' '/nix/store/m54bmrhj6fqz8nds5zcj97w9s9bckc9v-compress-man-pages.sh' '/nix/store/wgrbkkaldkrlrni33ccvm3b6vbxzb656-make-symlinks-relative.sh' '/nix/store/5yzw0vhkyszf2d179m0qfkgxmp5wjjx4-move-docs.sh' '/nix/store/fyaryjvghbkpfnsyw97hb3lyb37s1pd6-move-lib64.sh' '/nix/store/kd4xwxjpjxi71jkm6ka0np72if9rm3y0-move-sbin.sh' '/nix/store/pag6l61paj1dc9sv15l7bm5c17xn5kyk-move-systemd-user-units.sh' '/nix/store/jivxp510zxakaaic7qkrb7v1dd2rdbw9-multiple-outputs.sh' '/nix/store/12lvf0c7xric9cny7slvf9cmhypl1p67-patch-shebangs.sh' '/nix/store/cickvswrvann041nqxb0rxilc46svw1n-prune-libtool-files.sh' '/nix/store/xyff06pkhki3qy1ls77w10s0v79c9il0-reproducible-builds.sh' '/nix/store/aazf105snicrlvyzzbdj85sx4179rpfp-set-source-date-epoch-to-latest.sh' '/nix/store/gps9qrh99j7g02840wv5x78ykmz30byp-strip.sh' '/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0' '/nix/store/qlzvmgr8w9prdlyys7irqf86p7bndf5b-binutils-wrapper-2.43.1' ) +declare -a pkgsBuildTarget=() +declare -a pkgsHostHost=() +declare -a pkgsHostTarget=() +declare -a pkgsTargetTarget=() +declare -a postFixupHooks=('cmakePcfileCheckPhase' '_makeSymlinksRelativeInAllOutputs' '_multioutPropagateDev' ) +declare -a postHooks=('makeCmakeFindLibs' ) +declare -a postUnpackHooks=('_updateSourceDateEpochFromSourceRoot' ) +declare -a preConfigureHooks=('_multioutConfig' ) +preConfigurePhases=' updateAutotoolsGnuConfigScriptsPhase' +declare -a preFixupHooks=('_moveToShare' '_multioutDocs' '_multioutDevs' ) +preferLocalBuild='1' +export preferLocalBuild +prefix='/home/nickorlow/programming/personal/anthracite/outputs/out' +declare -a propagatedBuildDepFiles=('propagated-build-build-deps' 'propagated-native-build-inputs' 'propagated-build-target-deps' ) +propagatedBuildInputs='' +export propagatedBuildInputs +declare -a propagatedHostDepFiles=('propagated-host-host-deps' 'propagated-build-inputs' ) +propagatedNativeBuildInputs='' +export propagatedNativeBuildInputs +declare -a propagatedTargetDepFiles=('propagated-target-target-deps' ) +setOutputFlags='' +shell='/nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37/bin/bash' +export shell +shellHook='export OPENSSL_DIR="/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev" +export PKG_CONFIG_PATH="/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/lib/pkgconfig" +export OPENSSL_NO_VENDOR=1 +export OPENSSL_LIB_DIR="/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2/lib" +' +export shellHook +stdenv='/nix/store/lzrs17sc8bhi87nb1y1q1bas73j6q10y-stdenv-linux' +export stdenv +strictDeps='' +export strictDeps +system='x86_64-linux' +export system +declare -a unpackCmdHooks=('_defaultUnpack' ) +_activatePkgs () +{ + + local hostOffset targetOffset; + local pkg; + for hostOffset in "${allPlatOffsets[@]}"; + do + local pkgsVar="${pkgAccumVarVars[hostOffset + 1]}"; + for targetOffset in "${allPlatOffsets[@]}"; + do + (( hostOffset <= targetOffset )) || continue; + local pkgsRef="${pkgsVar}[$targetOffset - $hostOffset]"; + local pkgsSlice="${!pkgsRef}[@]"; + for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; + do + activatePackage "$pkg" "$hostOffset" "$targetOffset"; + done; + done; + done +} +_addRpathPrefix () +{ + + if [ "${NIX_NO_SELF_RPATH:-0}" != 1 ]; then + export NIX_LDFLAGS="-rpath $1/lib ${NIX_LDFLAGS-}"; + fi +} +_addToEnv () +{ + + local depHostOffset depTargetOffset; + local pkg; + for depHostOffset in "${allPlatOffsets[@]}"; + do + local hookVar="${pkgHookVarVars[depHostOffset + 1]}"; + local pkgsVar="${pkgAccumVarVars[depHostOffset + 1]}"; + for depTargetOffset in "${allPlatOffsets[@]}"; + do + (( depHostOffset <= depTargetOffset )) || continue; + local hookRef="${hookVar}[$depTargetOffset - $depHostOffset]"; + if [[ -z "${strictDeps-}" ]]; then + local visitedPkgs=""; + for pkg in "${pkgsBuildBuild[@]}" "${pkgsBuildHost[@]}" "${pkgsBuildTarget[@]}" "${pkgsHostHost[@]}" "${pkgsHostTarget[@]}" "${pkgsTargetTarget[@]}"; + do + if [[ "$visitedPkgs" = *"$pkg"* ]]; then + continue; + fi; + runHook "${!hookRef}" "$pkg"; + visitedPkgs+=" $pkg"; + done; + else + local pkgsRef="${pkgsVar}[$depTargetOffset - $depHostOffset]"; + local pkgsSlice="${!pkgsRef}[@]"; + for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; + do + runHook "${!hookRef}" "$pkg"; + done; + fi; + done; + done +} +_allFlags () +{ + + export system pname name version; + while IFS='' read -r varName; do + nixTalkativeLog "@${varName}@ -> ${!varName}"; + args+=("--subst-var" "$varName"); + done < <(awk 'BEGIN { for (v in ENVIRON) if (v ~ /^[a-z][a-zA-Z0-9_]*$/) print v }') +} +_assignFirst () +{ + + local varName="$1"; + local _var; + local REMOVE=REMOVE; + shift; + for _var in "$@"; + do + if [ -n "${!_var-}" ]; then + eval "${varName}"="${_var}"; + return; + fi; + done; + echo; + echo "error: _assignFirst: could not find a non-empty variable whose name to assign to ${varName}."; + echo " The following variables were all unset or empty:"; + echo " $*"; + if [ -z "${out:-}" ]; then + echo ' If you do not want an "out" output in your derivation, make sure to define'; + echo ' the other specific required outputs. This can be achieved by picking one'; + echo " of the above as an output."; + echo ' You do not have to remove "out" if you want to have a different default'; + echo ' output, because the first output is taken as a default.'; + echo; + fi; + return 1 +} +_callImplicitHook () +{ + + local def="$1"; + local hookName="$2"; + if declare -F "$hookName" > /dev/null; then + nixTalkativeLog "calling implicit '$hookName' function hook"; + "$hookName"; + else + if type -p "$hookName" > /dev/null; then + nixTalkativeLog "sourcing implicit '$hookName' script hook"; + source "$hookName"; + else + if [ -n "${!hookName:-}" ]; then + nixTalkativeLog "evaling implicit '$hookName' string hook"; + eval "${!hookName}"; + else + return "$def"; + fi; + fi; + fi +} +_defaultUnpack () +{ + + local fn="$1"; + local destination; + if [ -d "$fn" ]; then + destination="$(stripHash "$fn")"; + if [ -e "$destination" ]; then + echo "Cannot copy $fn to $destination: destination already exists!"; + echo "Did you specify two \"srcs\" with the same \"name\"?"; + return 1; + fi; + cp -pr --reflink=auto -- "$fn" "$destination"; + else + case "$fn" in + *.tar.xz | *.tar.lzma | *.txz) + ( XZ_OPT="--threads=$NIX_BUILD_CORES" xz -d < "$fn"; + true ) | tar xf - --mode=+w --warning=no-timestamp + ;; + *.tar | *.tar.* | *.tgz | *.tbz2 | *.tbz) + tar xf "$fn" --mode=+w --warning=no-timestamp + ;; + *) + return 1 + ;; + esac; + fi +} +_doStrip () +{ + + local -ra flags=(dontStripHost dontStripTarget); + local -ra debugDirs=(stripDebugList stripDebugListTarget); + local -ra allDirs=(stripAllList stripAllListTarget); + local -ra stripCmds=(STRIP STRIP_FOR_TARGET); + local -ra ranlibCmds=(RANLIB RANLIB_FOR_TARGET); + stripDebugList=${stripDebugList[*]:-lib lib32 lib64 libexec bin sbin}; + stripDebugListTarget=${stripDebugListTarget[*]:-}; + stripAllList=${stripAllList[*]:-}; + stripAllListTarget=${stripAllListTarget[*]:-}; + local i; + for i in ${!stripCmds[@]}; + do + local -n flag="${flags[$i]}"; + local -n debugDirList="${debugDirs[$i]}"; + local -n allDirList="${allDirs[$i]}"; + local -n stripCmd="${stripCmds[$i]}"; + local -n ranlibCmd="${ranlibCmds[$i]}"; + if [[ -n "${dontStrip-}" || -n "${flag-}" ]] || ! type -f "${stripCmd-}" 2> /dev/null 1>&2; then + continue; + fi; + stripDirs "$stripCmd" "$ranlibCmd" "$debugDirList" "${stripDebugFlags[*]:--S -p}"; + stripDirs "$stripCmd" "$ranlibCmd" "$allDirList" "${stripAllFlags[*]:--s -p}"; + done +} +_eval () +{ + + if declare -F "$1" > /dev/null 2>&1; then + "$@"; + else + eval "$1"; + fi +} +_logHook () +{ + + if [[ -z ${NIX_LOG_FD-} ]]; then + return; + fi; + local hookKind="$1"; + local hookExpr="$2"; + shift 2; + if declare -F "$hookExpr" > /dev/null 2>&1; then + nixTalkativeLog "calling '$hookKind' function hook '$hookExpr'" "$@"; + else + if type -p "$hookExpr" > /dev/null; then + nixTalkativeLog "sourcing '$hookKind' script hook '$hookExpr'"; + else + if [[ "$hookExpr" != "_callImplicitHook"* ]]; then + local exprToOutput; + if [[ ${NIX_DEBUG:-0} -ge 5 ]]; then + exprToOutput="$hookExpr"; + else + local hookExprLine; + while IFS= read -r hookExprLine; do + hookExprLine="${hookExprLine#"${hookExprLine%%[![:space:]]*}"}"; + if [[ -n "$hookExprLine" ]]; then + exprToOutput+="$hookExprLine\\n "; + fi; + done <<< "$hookExpr"; + exprToOutput="${exprToOutput%%\\n }"; + fi; + nixTalkativeLog "evaling '$hookKind' string hook '$exprToOutput'"; + fi; + fi; + fi +} +_makeSymlinksRelative () +{ + + local symlinkTarget; + if [ "${dontRewriteSymlinks-}" ] || [ ! -e "$prefix" ]; then + return; + fi; + while IFS= read -r -d '' f; do + symlinkTarget=$(readlink "$f"); + if [[ "$symlinkTarget"/ != "$prefix"/* ]]; then + continue; + fi; + if [ ! -e "$symlinkTarget" ]; then + echo "the symlink $f is broken, it points to $symlinkTarget (which is missing)"; + fi; + echo "rewriting symlink $f to be relative to $prefix"; + ln -snrf "$symlinkTarget" "$f"; + done < <(find $prefix -type l -print0) +} +_makeSymlinksRelativeInAllOutputs () +{ + + local output; + for output in $(getAllOutputNames); + do + prefix="${!output}" _makeSymlinksRelative; + done +} +_moveLib64 () +{ + + if [ "${dontMoveLib64-}" = 1 ]; then + return; + fi; + if [ ! -e "$prefix/lib64" -o -L "$prefix/lib64" ]; then + return; + fi; + echo "moving $prefix/lib64/* to $prefix/lib"; + mkdir -p $prefix/lib; + shopt -s dotglob; + for i in $prefix/lib64/*; + do + mv --no-clobber "$i" $prefix/lib; + done; + shopt -u dotglob; + rmdir $prefix/lib64; + ln -s lib $prefix/lib64 +} +_moveSbin () +{ + + if [ "${dontMoveSbin-}" = 1 ]; then + return; + fi; + if [ ! -e "$prefix/sbin" -o -L "$prefix/sbin" ]; then + return; + fi; + echo "moving $prefix/sbin/* to $prefix/bin"; + mkdir -p $prefix/bin; + shopt -s dotglob; + for i in $prefix/sbin/*; + do + mv "$i" $prefix/bin; + done; + shopt -u dotglob; + rmdir $prefix/sbin; + ln -s bin $prefix/sbin +} +_moveSystemdUserUnits () +{ + + if [ "${dontMoveSystemdUserUnits:-0}" = 1 ]; then + return; + fi; + if [ ! -e "${prefix:?}/lib/systemd/user" ]; then + return; + fi; + local source="$prefix/lib/systemd/user"; + local target="$prefix/share/systemd/user"; + echo "moving $source/* to $target"; + mkdir -p "$target"; + ( shopt -s dotglob; + for i in "$source"/*; + do + mv "$i" "$target"; + done ); + rmdir "$source"; + ln -s "$target" "$source" +} +_moveToShare () +{ + + if [ -n "$__structuredAttrs" ]; then + if [ -z "${forceShare-}" ]; then + forceShare=(man doc info); + fi; + else + forceShare=(${forceShare:-man doc info}); + fi; + if [[ -z "$out" ]]; then + return; + fi; + for d in "${forceShare[@]}"; + do + if [ -d "$out/$d" ]; then + if [ -d "$out/share/$d" ]; then + echo "both $d/ and share/$d/ exist!"; + else + echo "moving $out/$d to $out/share/$d"; + mkdir -p $out/share; + mv $out/$d $out/share/; + fi; + fi; + done +} +_multioutConfig () +{ + + if [ "$(getAllOutputNames)" = "out" ] || [ -z "${setOutputFlags-1}" ]; then + return; + fi; + if [ -z "${shareDocName:-}" ]; then + local confScript="${configureScript:-}"; + if [ -z "$confScript" ] && [ -x ./configure ]; then + confScript=./configure; + fi; + if [ -f "$confScript" ]; then + local shareDocName="$(sed -n "s/^PACKAGE_TARNAME='\(.*\)'$/\1/p" < "$confScript")"; + fi; + if [ -z "$shareDocName" ] || echo "$shareDocName" | grep -q '[^a-zA-Z0-9_-]'; then + shareDocName="$(echo "$name" | sed 's/-[^a-zA-Z].*//')"; + fi; + fi; + prependToVar configureFlags --bindir="${!outputBin}"/bin --sbindir="${!outputBin}"/sbin --includedir="${!outputInclude}"/include --oldincludedir="${!outputInclude}"/include --mandir="${!outputMan}"/share/man --infodir="${!outputInfo}"/share/info --docdir="${!outputDoc}"/share/doc/"${shareDocName}" --libdir="${!outputLib}"/lib --libexecdir="${!outputLib}"/libexec --localedir="${!outputLib}"/share/locale; + prependToVar installFlags pkgconfigdir="${!outputDev}"/lib/pkgconfig m4datadir="${!outputDev}"/share/aclocal aclocaldir="${!outputDev}"/share/aclocal +} +_multioutDevs () +{ + + if [ "$(getAllOutputNames)" = "out" ] || [ -z "${moveToDev-1}" ]; then + return; + fi; + moveToOutput include "${!outputInclude}"; + moveToOutput lib/pkgconfig "${!outputDev}"; + moveToOutput share/pkgconfig "${!outputDev}"; + moveToOutput lib/cmake "${!outputDev}"; + moveToOutput share/aclocal "${!outputDev}"; + for f in "${!outputDev}"/{lib,share}/pkgconfig/*.pc; + do + echo "Patching '$f' includedir to output ${!outputInclude}"; + sed -i "/^includedir=/s,=\${prefix},=${!outputInclude}," "$f"; + done +} +_multioutDocs () +{ + + local REMOVE=REMOVE; + moveToOutput share/info "${!outputInfo}"; + moveToOutput share/doc "${!outputDoc}"; + moveToOutput share/gtk-doc "${!outputDevdoc}"; + moveToOutput share/devhelp/books "${!outputDevdoc}"; + moveToOutput share/man "${!outputMan}"; + moveToOutput share/man/man3 "${!outputDevman}" +} +_multioutPropagateDev () +{ + + if [ "$(getAllOutputNames)" = "out" ]; then + return; + fi; + local outputFirst; + for outputFirst in $(getAllOutputNames); + do + break; + done; + local propagaterOutput="$outputDev"; + if [ -z "$propagaterOutput" ]; then + propagaterOutput="$outputFirst"; + fi; + if [ -z "${propagatedBuildOutputs+1}" ]; then + local po_dirty="$outputBin $outputInclude $outputLib"; + set +o pipefail; + propagatedBuildOutputs=`echo "$po_dirty" | tr -s ' ' '\n' | grep -v -F "$propagaterOutput" | sort -u | tr '\n' ' ' `; + set -o pipefail; + fi; + if [ -z "$propagatedBuildOutputs" ]; then + return; + fi; + mkdir -p "${!propagaterOutput}"/nix-support; + for output in $propagatedBuildOutputs; + do + echo -n " ${!output}" >> "${!propagaterOutput}"/nix-support/propagated-build-inputs; + done +} +_overrideFirst () +{ + + if [ -z "${!1-}" ]; then + _assignFirst "$@"; + fi +} +_pruneLibtoolFiles () +{ + + if [ "${dontPruneLibtoolFiles-}" ] || [ ! -e "$prefix" ]; then + return; + fi; + find "$prefix" -type f -name '*.la' -exec grep -q '^# Generated by .*libtool' {} \; -exec grep -q "^old_library=''" {} \; -exec sed -i {} -e "/^dependency_libs='[^']/ c dependency_libs='' #pruned" \; +} +_updateSourceDateEpochFromSourceRoot () +{ + + if [ -n "$sourceRoot" ]; then + updateSourceDateEpoch "$sourceRoot"; + fi +} +activatePackage () +{ + + local pkg="$1"; + local -r hostOffset="$2"; + local -r targetOffset="$3"; + (( hostOffset <= targetOffset )) || exit 1; + if [ -f "$pkg" ]; then + nixTalkativeLog "sourcing setup hook '$pkg'"; + source "$pkg"; + fi; + if [[ -z "${strictDeps-}" || "$hostOffset" -le -1 ]]; then + addToSearchPath _PATH "$pkg/bin"; + fi; + if (( hostOffset <= -1 )); then + addToSearchPath _XDG_DATA_DIRS "$pkg/share"; + fi; + if [[ "$hostOffset" -eq 0 && -d "$pkg/bin" ]]; then + addToSearchPath _HOST_PATH "$pkg/bin"; + fi; + if [[ -f "$pkg/nix-support/setup-hook" ]]; then + nixTalkativeLog "sourcing setup hook '$pkg/nix-support/setup-hook'"; + source "$pkg/nix-support/setup-hook"; + fi +} +addCMakeParams () +{ + + addToSearchPath NIXPKGS_CMAKE_PREFIX_PATH $1 +} +addEnvHooks () +{ + + local depHostOffset="$1"; + shift; + local pkgHookVarsSlice="${pkgHookVarVars[$depHostOffset + 1]}[@]"; + local pkgHookVar; + for pkgHookVar in "${!pkgHookVarsSlice}"; + do + eval "${pkgHookVar}s"'+=("$@")'; + done +} +addToSearchPath () +{ + + addToSearchPathWithCustomDelimiter ":" "$@" +} +addToSearchPathWithCustomDelimiter () +{ + + local delimiter="$1"; + local varName="$2"; + local dir="$3"; + if [[ -d "$dir" && "${!varName:+${delimiter}${!varName}${delimiter}}" != *"${delimiter}${dir}${delimiter}"* ]]; then + export "${varName}=${!varName:+${!varName}${delimiter}}${dir}"; + fi +} +appendToVar () +{ + + local -n nameref="$1"; + local useArray type; + if [ -n "$__structuredAttrs" ]; then + useArray=true; + else + useArray=false; + fi; + if type=$(declare -p "$1" 2> /dev/null); then + case "${type#* }" in + -A*) + echo "appendToVar(): ERROR: trying to use appendToVar on an associative array, use variable+=([\"X\"]=\"Y\") instead." 1>&2; + return 1 + ;; + -a*) + useArray=true + ;; + *) + useArray=false + ;; + esac; + fi; + shift; + if $useArray; then + nameref=(${nameref+"${nameref[@]}"} "$@"); + else + nameref="${nameref-} $*"; + fi +} +auditTmpdir () +{ + + local dir="$1"; + [ -e "$dir" ] || return 0; + echo "checking for references to $TMPDIR/ in $dir..."; + local i; + find "$dir" -type f -print0 | while IFS= read -r -d '' i; do + if [[ "$i" =~ .build-id ]]; then + continue; + fi; + if isELF "$i"; then + if { + printf :; + patchelf --print-rpath "$i" + } | grep -q -F ":$TMPDIR/"; then + echo "RPATH of binary $i contains a forbidden reference to $TMPDIR/"; + exit 1; + fi; + fi; + if isScript "$i"; then + if [ -e "$(dirname "$i")/.$(basename "$i")-wrapped" ]; then + if grep -q -F "$TMPDIR/" "$i"; then + echo "wrapper script $i contains a forbidden reference to $TMPDIR/"; + exit 1; + fi; + fi; + fi; + done +} +bintoolsWrapper_addLDVars () +{ + + local role_post; + getHostRoleEnvHook; + if [[ -d "$1/lib64" && ! -L "$1/lib64" ]]; then + export NIX_LDFLAGS${role_post}+=" -L$1/lib64"; + fi; + if [[ -d "$1/lib" ]]; then + local -a glob=($1/lib/lib*); + if [ "${#glob[*]}" -gt 0 ]; then + export NIX_LDFLAGS${role_post}+=" -L$1/lib"; + fi; + fi +} +buildPhase () +{ + + runHook preBuild; + if [[ -z "${makeFlags-}" && -z "${makefile:-}" && ! ( -e Makefile || -e makefile || -e GNUmakefile ) ]]; then + echo "no Makefile or custom buildPhase, doing nothing"; + else + foundMakefile=1; + local flagsArray=(${enableParallelBuilding:+-j${NIX_BUILD_CORES}} SHELL="$SHELL"); + concatTo flagsArray makeFlags makeFlagsArray buildFlags buildFlagsArray; + echoCmd 'build flags' "${flagsArray[@]}"; + make ${makefile:+-f $makefile} "${flagsArray[@]}"; + unset flagsArray; + fi; + runHook postBuild +} +ccWrapper_addCVars () +{ + + local role_post; + getHostRoleEnvHook; + if [ -d "$1/include" ]; then + export NIX_CFLAGS_COMPILE${role_post}+=" -isystem $1/include"; + fi; + if [ -d "$1/Library/Frameworks" ]; then + export NIX_CFLAGS_COMPILE${role_post}+=" -iframework $1/Library/Frameworks"; + fi +} +checkPhase () +{ + + runHook preCheck; + if [[ -z "${foundMakefile:-}" ]]; then + echo "no Makefile or custom checkPhase, doing nothing"; + runHook postCheck; + return; + fi; + if [[ -z "${checkTarget:-}" ]]; then + if make -n ${makefile:+-f $makefile} check > /dev/null 2>&1; then + checkTarget="check"; + else + if make -n ${makefile:+-f $makefile} test > /dev/null 2>&1; then + checkTarget="test"; + fi; + fi; + fi; + if [[ -z "${checkTarget:-}" ]]; then + echo "no check/test target in ${makefile:-Makefile}, doing nothing"; + else + local flagsArray=(${enableParallelChecking:+-j${NIX_BUILD_CORES}} SHELL="$SHELL"); + concatTo flagsArray makeFlags makeFlagsArray checkFlags=VERBOSE=y checkFlagsArray checkTarget; + echoCmd 'check flags' "${flagsArray[@]}"; + make ${makefile:+-f $makefile} "${flagsArray[@]}"; + unset flagsArray; + fi; + runHook postCheck +} +cmakeConfigurePhase () +{ + + runHook preConfigure; + : ${cmakeBuildDir:=build}; + export CTEST_OUTPUT_ON_FAILURE=1; + if [ -n "${enableParallelChecking-1}" ]; then + export CTEST_PARALLEL_LEVEL=$NIX_BUILD_CORES; + fi; + if [ -z "${dontFixCmake-}" ]; then + fixCmakeFiles .; + fi; + if [ -z "${dontUseCmakeBuildDir-}" ]; then + mkdir -p "$cmakeBuildDir"; + cd "$cmakeBuildDir"; + : ${cmakeDir:=..}; + else + : ${cmakeDir:=.}; + fi; + if [ -z "${dontAddPrefix-}" ]; then + prependToVar cmakeFlags "-DCMAKE_INSTALL_PREFIX=$prefix"; + fi; + prependToVar cmakeFlags "-DCMAKE_CXX_COMPILER=$CXX"; + prependToVar cmakeFlags "-DCMAKE_C_COMPILER=$CC"; + prependToVar cmakeFlags "-DCMAKE_AR=$(command -v $AR)"; + prependToVar cmakeFlags "-DCMAKE_RANLIB=$(command -v $RANLIB)"; + prependToVar cmakeFlags "-DCMAKE_STRIP=$(command -v $STRIP)"; + prependToVar cmakeFlags "-DCMAKE_FIND_FRAMEWORK=LAST"; + prependToVar cmakeFlags "-DCMAKE_POLICY_DEFAULT_CMP0025=NEW"; + prependToVar cmakeFlags "-DCMAKE_INSTALL_NAME_DIR=${!outputLib}/lib"; + if [[ -z "$shareDocName" ]]; then + local cmakeLists="${cmakeDir}/CMakeLists.txt"; + if [[ -f "$cmakeLists" ]]; then + local shareDocName="$(grep --only-matching --perl-regexp --ignore-case '\bproject\s*\(\s*"?\K([^[:space:]")]+)' < "$cmakeLists" | head -n1)"; + fi; + if [[ -z "$shareDocName" ]] || echo "$shareDocName" | grep -q '[^a-zA-Z0-9_+-]'; then + if [[ -n "${pname-}" ]]; then + shareDocName="$pname"; + else + shareDocName="$(echo "$name" | sed 's/-[^a-zA-Z].*//')"; + fi; + fi; + fi; + prependToVar cmakeFlags "-DCMAKE_INSTALL_BINDIR=${!outputBin}/bin"; + prependToVar cmakeFlags "-DCMAKE_INSTALL_SBINDIR=${!outputBin}/sbin"; + prependToVar cmakeFlags "-DCMAKE_INSTALL_INCLUDEDIR=${!outputInclude}/include"; + prependToVar cmakeFlags "-DCMAKE_INSTALL_OLDINCLUDEDIR=${!outputInclude}/include"; + prependToVar cmakeFlags "-DCMAKE_INSTALL_MANDIR=${!outputMan}/share/man"; + prependToVar cmakeFlags "-DCMAKE_INSTALL_INFODIR=${!outputInfo}/share/info"; + prependToVar cmakeFlags "-DCMAKE_INSTALL_DOCDIR=${!outputDoc}/share/doc/${shareDocName}"; + prependToVar cmakeFlags "-DCMAKE_INSTALL_LIBDIR=${!outputLib}/lib"; + prependToVar cmakeFlags "-DCMAKE_INSTALL_LIBEXECDIR=${!outputLib}/libexec"; + prependToVar cmakeFlags "-DCMAKE_INSTALL_LOCALEDIR=${!outputLib}/share/locale"; + if [ -z "${doCheck-}" ]; then + prependToVar cmakeFlags "-DBUILD_TESTING=OFF"; + fi; + prependToVar cmakeFlags "-DCMAKE_BUILD_TYPE=${cmakeBuildType:-Release}"; + prependToVar cmakeFlags "-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON"; + prependToVar cmakeFlags "-DCMAKE_FIND_USE_PACKAGE_REGISTRY=OFF"; + prependToVar cmakeFlags "-DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=OFF"; + if [ "${buildPhase-}" = ninjaBuildPhase ]; then + prependToVar cmakeFlags "-GNinja"; + fi; + local flagsArray=(); + concatTo flagsArray cmakeFlags cmakeFlagsArray; + echoCmd 'cmake flags' "${flagsArray[@]}"; + cmake "$cmakeDir" "${flagsArray[@]}"; + if ! [[ -v enableParallelBuilding ]]; then + enableParallelBuilding=1; + echo "cmake: enabled parallel building"; + fi; + if ! [[ -v enableParallelInstalling ]]; then + enableParallelInstalling=1; + echo "cmake: enabled parallel installing"; + fi; + runHook postConfigure +} +cmakePcfileCheckPhase () +{ + + while IFS= read -rd '' file; do + grepout=$(grep --line-number '}//nix/store' "$file" || true); + if [ -n "$grepout" ]; then + { + echo "Broken paths found in a .pc file! $file"; + echo "The following lines have issues (specifically '//' in paths)."; + echo "$grepout"; + echo "It is very likely that paths are being joined improperly."; + echo 'ex: "${prefix}/@CMAKE_INSTALL_LIBDIR@" should be "@CMAKE_INSTALL_FULL_LIBDIR@"'; + echo "Please see https://github.com/NixOS/nixpkgs/issues/144170 for more details."; + exit 1 + } 1>&2; + fi; + done < <(find "${!outputDev}" -iname "*.pc" -print0) +} +compressManPages () +{ + + local dir="$1"; + if [ -L "$dir"/share ] || [ -L "$dir"/share/man ] || [ ! -d "$dir/share/man" ]; then + return; + fi; + echo "gzipping man pages under $dir/share/man/"; + find "$dir"/share/man/ -type f -a '!' -regex '.*\.\(bz2\|gz\|xz\)$' -print0 | while IFS= read -r -d '' f; do + if gzip -c -n "$f" > "$f".gz; then + rm "$f"; + else + rm "$f".gz; + fi; + done; + find "$dir"/share/man/ -type l -a '!' -regex '.*\.\(bz2\|gz\|xz\)$' -print0 | sort -z | while IFS= read -r -d '' f; do + local target; + target="$(readlink -f "$f")"; + if [ -f "$target".gz ]; then + ln -sf "$target".gz "$f".gz && rm "$f"; + fi; + done +} +concatStringsSep () +{ + + local sep="$1"; + local name="$2"; + local type oldifs; + if type=$(declare -p "$name" 2> /dev/null); then + local -n nameref="$name"; + case "${type#* }" in + -A*) + echo "concatStringsSep(): ERROR: trying to use concatStringsSep on an associative array." 1>&2; + return 1 + ;; + -a*) + local IFS="$sep"; + echo -n "${nameref[*]}" + ;; + *) + echo -n "${nameref// /"${sep}"}" + ;; + esac; + fi +} +concatTo () +{ + + local -; + set -o noglob; + local -n targetref="$1"; + shift; + local arg default name type; + for arg in "$@"; + do + IFS="=" read -r name default <<< "$arg"; + local -n nameref="$name"; + if [[ -z "${nameref[*]}" && -n "$default" ]]; then + targetref+=("$default"); + else + if type=$(declare -p "$name" 2> /dev/null); then + case "${type#* }" in + -A*) + echo "concatTo(): ERROR: trying to use concatTo on an associative array." 1>&2; + return 1 + ;; + -a*) + targetref+=("${nameref[@]}") + ;; + *) + if [[ "$name" = *"Array" ]]; then + nixErrorLog "concatTo(): $name is not declared as array, treating as a singleton. This will become an error in future"; + targetref+=(${nameref+"${nameref[@]}"}); + else + targetref+=(${nameref-}); + fi + ;; + esac; + fi; + fi; + done +} +configurePhase () +{ + + runHook preConfigure; + : "${configureScript=}"; + if [[ -z "$configureScript" && -x ./configure ]]; then + configureScript=./configure; + fi; + if [ -z "${dontFixLibtool:-}" ]; then + export lt_cv_deplibs_check_method="${lt_cv_deplibs_check_method-pass_all}"; + local i; + find . -iname "ltmain.sh" -print0 | while IFS='' read -r -d '' i; do + echo "fixing libtool script $i"; + fixLibtool "$i"; + done; + CONFIGURE_MTIME_REFERENCE=$(mktemp configure.mtime.reference.XXXXXX); + find . -executable -type f -name configure -exec grep -l 'GNU Libtool is free software; you can redistribute it and/or modify' {} \; -exec touch -r {} "$CONFIGURE_MTIME_REFERENCE" \; -exec sed -i s_/usr/bin/file_file_g {} \; -exec touch -r "$CONFIGURE_MTIME_REFERENCE" {} \;; + rm -f "$CONFIGURE_MTIME_REFERENCE"; + fi; + if [[ -z "${dontAddPrefix:-}" && -n "$prefix" ]]; then + prependToVar configureFlags "${prefixKey:---prefix=}$prefix"; + fi; + if [[ -f "$configureScript" ]]; then + if [ -z "${dontAddDisableDepTrack:-}" ]; then + if grep -q dependency-tracking "$configureScript"; then + prependToVar configureFlags --disable-dependency-tracking; + fi; + fi; + if [ -z "${dontDisableStatic:-}" ]; then + if grep -q enable-static "$configureScript"; then + prependToVar configureFlags --disable-static; + fi; + fi; + if [ -z "${dontPatchShebangsInConfigure:-}" ]; then + patchShebangs --build "$configureScript"; + fi; + fi; + if [ -n "$configureScript" ]; then + local -a flagsArray; + concatTo flagsArray configureFlags configureFlagsArray; + echoCmd 'configure flags' "${flagsArray[@]}"; + $configureScript "${flagsArray[@]}"; + unset flagsArray; + else + echo "no configure script, doing nothing"; + fi; + runHook postConfigure +} +consumeEntire () +{ + + if IFS='' read -r -d '' "$1"; then + echo "consumeEntire(): ERROR: Input null bytes, won't process" 1>&2; + return 1; + fi +} +distPhase () +{ + + runHook preDist; + local flagsArray=(); + concatTo flagsArray distFlags distFlagsArray distTarget=dist; + echo 'dist flags: %q' "${flagsArray[@]}"; + make ${makefile:+-f $makefile} "${flagsArray[@]}"; + if [ "${dontCopyDist:-0}" != 1 ]; then + mkdir -p "$out/tarballs"; + cp -pvd ${tarballs[*]:-*.tar.gz} "$out/tarballs"; + fi; + runHook postDist +} +dumpVars () +{ + + if [ "${noDumpEnvVars:-0}" != 1 ]; then + { + install -m 0600 /dev/null "$NIX_BUILD_TOP/env-vars" && export 2> /dev/null >| "$NIX_BUILD_TOP/env-vars" + } || true; + fi +} +echoCmd () +{ + + printf "%s:" "$1"; + shift; + printf ' %q' "$@"; + echo +} +exitHandler () +{ + + exitCode="$?"; + set +e; + if [ -n "${showBuildStats:-}" ]; then + read -r -d '' -a buildTimes < <(times); + echo "build times:"; + echo "user time for the shell ${buildTimes[0]}"; + echo "system time for the shell ${buildTimes[1]}"; + echo "user time for all child processes ${buildTimes[2]}"; + echo "system time for all child processes ${buildTimes[3]}"; + fi; + if (( "$exitCode" != 0 )); then + runHook failureHook; + if [ -n "${succeedOnFailure:-}" ]; then + echo "build failed with exit code $exitCode (ignored)"; + mkdir -p "$out/nix-support"; + printf "%s" "$exitCode" > "$out/nix-support/failed"; + exit 0; + fi; + else + runHook exitHook; + fi; + return "$exitCode" +} +findInputs () +{ + + local -r pkg="$1"; + local -r hostOffset="$2"; + local -r targetOffset="$3"; + (( hostOffset <= targetOffset )) || exit 1; + local varVar="${pkgAccumVarVars[hostOffset + 1]}"; + local varRef="$varVar[$((targetOffset - hostOffset))]"; + local var="${!varRef}"; + unset -v varVar varRef; + local varSlice="$var[*]"; + case "${!varSlice-}" in + *" $pkg "*) + return 0 + ;; + esac; + unset -v varSlice; + eval "$var"'+=("$pkg")'; + if ! [ -e "$pkg" ]; then + echo "build input $pkg does not exist" 1>&2; + exit 1; + fi; + function mapOffset () + { + local -r inputOffset="$1"; + local -n outputOffset="$2"; + if (( inputOffset <= 0 )); then + outputOffset=$((inputOffset + hostOffset)); + else + outputOffset=$((inputOffset - 1 + targetOffset)); + fi + }; + local relHostOffset; + for relHostOffset in "${allPlatOffsets[@]}"; + do + local files="${propagatedDepFilesVars[relHostOffset + 1]}"; + local hostOffsetNext; + mapOffset "$relHostOffset" hostOffsetNext; + (( -1 <= hostOffsetNext && hostOffsetNext <= 1 )) || continue; + local relTargetOffset; + for relTargetOffset in "${allPlatOffsets[@]}"; + do + (( "$relHostOffset" <= "$relTargetOffset" )) || continue; + local fileRef="${files}[$relTargetOffset - $relHostOffset]"; + local file="${!fileRef}"; + unset -v fileRef; + local targetOffsetNext; + mapOffset "$relTargetOffset" targetOffsetNext; + (( -1 <= hostOffsetNext && hostOffsetNext <= 1 )) || continue; + [[ -f "$pkg/nix-support/$file" ]] || continue; + local pkgNext; + read -r -d '' pkgNext < "$pkg/nix-support/$file" || true; + for pkgNext in $pkgNext; + do + findInputs "$pkgNext" "$hostOffsetNext" "$targetOffsetNext"; + done; + done; + done +} +fixCmakeFiles () +{ + + echo "fixing cmake files..."; + find "$1" -type f \( -name "*.cmake" -o -name "*.cmake.in" -o -name CMakeLists.txt \) -print | while read fn; do + sed -e 's^/usr\([ /]\|$\)^/var/empty\1^g' -e 's^/opt\([ /]\|$\)^/var/empty\1^g' < "$fn" > "$fn.tmp"; + mv "$fn.tmp" "$fn"; + done +} +fixLibtool () +{ + + local search_path; + for flag in $NIX_LDFLAGS; + do + case $flag in + -L*) + search_path+=" ${flag#-L}" + ;; + esac; + done; + sed -i "$1" -e "s^eval \(sys_lib_search_path=\).*^\1'${search_path:-}'^" -e 's^eval sys_lib_.+search_path=.*^^' +} +fixupPhase () +{ + + local output; + for output in $(getAllOutputNames); + do + if [ -e "${!output}" ]; then + chmod -R u+w,u-s,g-s "${!output}"; + fi; + done; + runHook preFixup; + local output; + for output in $(getAllOutputNames); + do + prefix="${!output}" runHook fixupOutput; + done; + recordPropagatedDependencies; + if [ -n "${setupHook:-}" ]; then + mkdir -p "${!outputDev}/nix-support"; + substituteAll "$setupHook" "${!outputDev}/nix-support/setup-hook"; + fi; + if [ -n "${setupHooks:-}" ]; then + mkdir -p "${!outputDev}/nix-support"; + local hook; + for hook in ${setupHooks[@]}; + do + local content; + consumeEntire content < "$hook"; + substituteAllStream content "file '$hook'" >> "${!outputDev}/nix-support/setup-hook"; + unset -v content; + done; + unset -v hook; + fi; + if [ -n "${propagatedUserEnvPkgs:-}" ]; then + mkdir -p "${!outputBin}/nix-support"; + printWords $propagatedUserEnvPkgs > "${!outputBin}/nix-support/propagated-user-env-packages"; + fi; + runHook postFixup +} +genericBuild () +{ + + export GZIP_NO_TIMESTAMPS=1; + if [ -f "${buildCommandPath:-}" ]; then + source "$buildCommandPath"; + return; + fi; + if [ -n "${buildCommand:-}" ]; then + eval "$buildCommand"; + return; + fi; + if [ -z "${phases[*]:-}" ]; then + phases="${prePhases[*]:-} unpackPhase patchPhase ${preConfigurePhases[*]:-} configurePhase ${preBuildPhases[*]:-} buildPhase checkPhase ${preInstallPhases[*]:-} installPhase ${preFixupPhases[*]:-} fixupPhase installCheckPhase ${preDistPhases[*]:-} distPhase ${postPhases[*]:-}"; + fi; + for curPhase in ${phases[*]}; + do + runPhase "$curPhase"; + done +} +getAllOutputNames () +{ + + if [ -n "$__structuredAttrs" ]; then + echo "${!outputs[*]}"; + else + echo "$outputs"; + fi +} +getHostRole () +{ + + getRole "$hostOffset" +} +getHostRoleEnvHook () +{ + + getRole "$depHostOffset" +} +getRole () +{ + + case $1 in + -1) + role_post='_FOR_BUILD' + ;; + 0) + role_post='' + ;; + 1) + role_post='_FOR_TARGET' + ;; + *) + echo "binutils-wrapper-2.43.1: used as improper sort of dependency" 1>&2; + return 1 + ;; + esac +} +getTargetRole () +{ + + getRole "$targetOffset" +} +getTargetRoleEnvHook () +{ + + getRole "$depTargetOffset" +} +getTargetRoleWrapper () +{ + + case $targetOffset in + -1) + export NIX_BINTOOLS_WRAPPER_TARGET_BUILD_x86_64_unknown_linux_gnu=1 + ;; + 0) + export NIX_BINTOOLS_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu=1 + ;; + 1) + export NIX_BINTOOLS_WRAPPER_TARGET_TARGET_x86_64_unknown_linux_gnu=1 + ;; + *) + echo "binutils-wrapper-2.43.1: used as improper sort of dependency" 1>&2; + return 1 + ;; + esac +} +installCheckPhase () +{ + + runHook preInstallCheck; + if [[ -z "${foundMakefile:-}" ]]; then + echo "no Makefile or custom installCheckPhase, doing nothing"; + else + if [[ -z "${installCheckTarget:-}" ]] && ! make -n ${makefile:+-f $makefile} "${installCheckTarget:-installcheck}" > /dev/null 2>&1; then + echo "no installcheck target in ${makefile:-Makefile}, doing nothing"; + else + local flagsArray=(${enableParallelChecking:+-j${NIX_BUILD_CORES}} SHELL="$SHELL"); + concatTo flagsArray makeFlags makeFlagsArray installCheckFlags installCheckFlagsArray installCheckTarget=installcheck; + echoCmd 'installcheck flags' "${flagsArray[@]}"; + make ${makefile:+-f $makefile} "${flagsArray[@]}"; + unset flagsArray; + fi; + fi; + runHook postInstallCheck +} +installPhase () +{ + + runHook preInstall; + if [[ -z "${makeFlags-}" && -z "${makefile:-}" && ! ( -e Makefile || -e makefile || -e GNUmakefile ) ]]; then + echo "no Makefile or custom installPhase, doing nothing"; + runHook postInstall; + return; + else + foundMakefile=1; + fi; + if [ -n "$prefix" ]; then + mkdir -p "$prefix"; + fi; + local flagsArray=(${enableParallelInstalling:+-j${NIX_BUILD_CORES}} SHELL="$SHELL"); + concatTo flagsArray makeFlags makeFlagsArray installFlags installFlagsArray installTargets=install; + echoCmd 'install flags' "${flagsArray[@]}"; + make ${makefile:+-f $makefile} "${flagsArray[@]}"; + unset flagsArray; + runHook postInstall +} +isELF () +{ + + local fn="$1"; + local fd; + local magic; + exec {fd}< "$fn"; + read -r -n 4 -u "$fd" magic; + exec {fd}>&-; + if [ "$magic" = 'ELF' ]; then + return 0; + else + return 1; + fi +} +isMachO () +{ + + local fn="$1"; + local fd; + local magic; + exec {fd}< "$fn"; + read -r -n 4 -u "$fd" magic; + exec {fd}>&-; + if [[ "$magic" = $(echo -ne "\xfe\xed\xfa\xcf") || "$magic" = $(echo -ne "\xcf\xfa\xed\xfe") ]]; then + return 0; + else + if [[ "$magic" = $(echo -ne "\xfe\xed\xfa\xce") || "$magic" = $(echo -ne "\xce\xfa\xed\xfe") ]]; then + return 0; + else + if [[ "$magic" = $(echo -ne "\xca\xfe\xba\xbe") || "$magic" = $(echo -ne "\xbe\xba\xfe\xca") ]]; then + return 0; + else + return 1; + fi; + fi; + fi +} +isScript () +{ + + local fn="$1"; + local fd; + local magic; + exec {fd}< "$fn"; + read -r -n 2 -u "$fd" magic; + exec {fd}>&-; + if [[ "$magic" =~ \#! ]]; then + return 0; + else + return 1; + fi +} +makeCmakeFindLibs () +{ + + isystem_seen=; + iframework_seen=; + for flag in ${NIX_CFLAGS_COMPILE-} ${NIX_LDFLAGS-}; + do + if test -n "$isystem_seen" && test -d "$flag"; then + isystem_seen=; + addToSearchPath CMAKE_INCLUDE_PATH "${flag}"; + else + if test -n "$iframework_seen" && test -d "$flag"; then + iframework_seen=; + addToSearchPath CMAKE_FRAMEWORK_PATH "${flag}"; + else + isystem_seen=; + iframework_seen=; + case $flag in + -I*) + addToSearchPath CMAKE_INCLUDE_PATH "${flag:2}" + ;; + -L*) + addToSearchPath CMAKE_LIBRARY_PATH "${flag:2}" + ;; + -F*) + addToSearchPath CMAKE_FRAMEWORK_PATH "${flag:2}" + ;; + -isystem) + isystem_seen=1 + ;; + -iframework) + iframework_seen=1 + ;; + esac; + fi; + fi; + done +} +mapOffset () +{ + + local -r inputOffset="$1"; + local -n outputOffset="$2"; + if (( inputOffset <= 0 )); then + outputOffset=$((inputOffset + hostOffset)); + else + outputOffset=$((inputOffset - 1 + targetOffset)); + fi +} +moveToOutput () +{ + + local patt="$1"; + local dstOut="$2"; + local output; + for output in $(getAllOutputNames); + do + if [ "${!output}" = "$dstOut" ]; then + continue; + fi; + local srcPath; + for srcPath in "${!output}"/$patt; + do + if [ ! -e "$srcPath" ] && [ ! -L "$srcPath" ]; then + continue; + fi; + if [ "$dstOut" = REMOVE ]; then + echo "Removing $srcPath"; + rm -r "$srcPath"; + else + local dstPath="$dstOut${srcPath#${!output}}"; + echo "Moving $srcPath to $dstPath"; + if [ -d "$dstPath" ] && [ -d "$srcPath" ]; then + rmdir "$srcPath" --ignore-fail-on-non-empty; + if [ -d "$srcPath" ]; then + mv -t "$dstPath" "$srcPath"/*; + rmdir "$srcPath"; + fi; + else + mkdir -p "$(readlink -m "$dstPath/..")"; + mv "$srcPath" "$dstPath"; + fi; + fi; + local srcParent="$(readlink -m "$srcPath/..")"; + if [ -n "$(find "$srcParent" -maxdepth 0 -type d -empty 2> /dev/null)" ]; then + echo "Removing empty $srcParent/ and (possibly) its parents"; + rmdir -p --ignore-fail-on-non-empty "$srcParent" 2> /dev/null || true; + fi; + done; + done +} +nixChattyLog () +{ + + if [[ -z ${NIX_LOG_FD-} ]] || [[ ${NIX_DEBUG:-0} -lt 5 ]]; then + return; + fi; + printf "%s\n" "$*" >&"$NIX_LOG_FD" +} +nixDebugLog () +{ + + if [[ -z ${NIX_LOG_FD-} ]] || [[ ${NIX_DEBUG:-0} -lt 6 ]]; then + return; + fi; + printf "%s\n" "$*" >&"$NIX_LOG_FD" +} +nixErrorLog () +{ + + if [[ -z ${NIX_LOG_FD-} ]] || [[ ${NIX_DEBUG:-0} -lt 0 ]]; then + return; + fi; + printf "%s\n" "$*" >&"$NIX_LOG_FD" +} +nixInfoLog () +{ + + if [[ -z ${NIX_LOG_FD-} ]] || [[ ${NIX_DEBUG:-0} -lt 3 ]]; then + return; + fi; + printf "%s\n" "$*" >&"$NIX_LOG_FD" +} +nixNoticeLog () +{ + + if [[ -z ${NIX_LOG_FD-} ]] || [[ ${NIX_DEBUG:-0} -lt 2 ]]; then + return; + fi; + printf "%s\n" "$*" >&"$NIX_LOG_FD" +} +nixTalkativeLog () +{ + + if [[ -z ${NIX_LOG_FD-} ]] || [[ ${NIX_DEBUG:-0} -lt 4 ]]; then + return; + fi; + printf "%s\n" "$*" >&"$NIX_LOG_FD" +} +nixVomitLog () +{ + + if [[ -z ${NIX_LOG_FD-} ]] || [[ ${NIX_DEBUG:-0} -lt 7 ]]; then + return; + fi; + printf "%s\n" "$*" >&"$NIX_LOG_FD" +} +nixWarnLog () +{ + + if [[ -z ${NIX_LOG_FD-} ]] || [[ ${NIX_DEBUG:-0} -lt 1 ]]; then + return; + fi; + printf "%s\n" "$*" >&"$NIX_LOG_FD" +} +patchELF () +{ + + local dir="$1"; + [ -e "$dir" ] || return 0; + echo "shrinking RPATHs of ELF executables and libraries in $dir"; + local i; + while IFS= read -r -d '' i; do + if [[ "$i" =~ .build-id ]]; then + continue; + fi; + if ! isELF "$i"; then + continue; + fi; + echo "shrinking $i"; + patchelf --shrink-rpath "$i" || true; + done < <(find "$dir" -type f -print0) +} +patchPhase () +{ + + runHook prePatch; + local -a patchesArray; + concatTo patchesArray patches; + for i in "${patchesArray[@]}"; + do + echo "applying patch $i"; + local uncompress=cat; + case "$i" in + *.gz) + uncompress="gzip -d" + ;; + *.bz2) + uncompress="bzip2 -d" + ;; + *.xz) + uncompress="xz -d" + ;; + *.lzma) + uncompress="lzma -d" + ;; + esac; + local -a flagsArray; + concatTo flagsArray patchFlags=-p1; + $uncompress < "$i" 2>&1 | patch "${flagsArray[@]}"; + done; + runHook postPatch +} +patchShebangs () +{ + + local pathName; + local update; + while [[ $# -gt 0 ]]; do + case "$1" in + --host) + pathName=HOST_PATH; + shift + ;; + --build) + pathName=PATH; + shift + ;; + --update) + update=true; + shift + ;; + --) + shift; + break + ;; + -* | --*) + echo "Unknown option $1 supplied to patchShebangs" 1>&2; + return 1 + ;; + *) + break + ;; + esac; + done; + echo "patching script interpreter paths in $@"; + local f; + local oldPath; + local newPath; + local arg0; + local args; + local oldInterpreterLine; + local newInterpreterLine; + if [[ $# -eq 0 ]]; then + echo "No arguments supplied to patchShebangs" 1>&2; + return 0; + fi; + local f; + while IFS= read -r -d '' f; do + isScript "$f" || continue; + read -r oldInterpreterLine < "$f" || [ "$oldInterpreterLine" ]; + read -r oldPath arg0 args <<< "${oldInterpreterLine:2}"; + if [[ -z "${pathName:-}" ]]; then + if [[ -n $strictDeps && $f == "$NIX_STORE"* ]]; then + pathName=HOST_PATH; + else + pathName=PATH; + fi; + fi; + if [[ "$oldPath" == *"/bin/env" ]]; then + if [[ $arg0 == "-S" ]]; then + arg0=${args%% *}; + args=${args#* }; + newPath="$(PATH="${!pathName}" type -P "env" || true)"; + args="-S $(PATH="${!pathName}" type -P "$arg0" || true) $args"; + else + if [[ $arg0 == "-"* || $arg0 == *"="* ]]; then + echo "$f: unsupported interpreter directive \"$oldInterpreterLine\" (set dontPatchShebangs=1 and handle shebang patching yourself)" 1>&2; + exit 1; + else + newPath="$(PATH="${!pathName}" type -P "$arg0" || true)"; + fi; + fi; + else + if [[ -z $oldPath ]]; then + oldPath="/bin/sh"; + fi; + newPath="$(PATH="${!pathName}" type -P "$(basename "$oldPath")" || true)"; + args="$arg0 $args"; + fi; + newInterpreterLine="$newPath $args"; + newInterpreterLine=${newInterpreterLine%${newInterpreterLine##*[![:space:]]}}; + if [[ -n "$oldPath" && ( "$update" == true || "${oldPath:0:${#NIX_STORE}}" != "$NIX_STORE" ) ]]; then + if [[ -n "$newPath" && "$newPath" != "$oldPath" ]]; then + echo "$f: interpreter directive changed from \"$oldInterpreterLine\" to \"$newInterpreterLine\""; + escapedInterpreterLine=${newInterpreterLine//\\/\\\\}; + timestamp=$(stat --printf "%y" "$f"); + sed -i -e "1 s|.*|#\!$escapedInterpreterLine|" "$f"; + touch --date "$timestamp" "$f"; + fi; + fi; + done < <(find "$@" -type f -perm -0100 -print0) +} +patchShebangsAuto () +{ + + if [[ -z "${dontPatchShebangs-}" && -e "$prefix" ]]; then + if [[ "$output" != out && "$output" = "$outputDev" ]]; then + patchShebangs --build "$prefix"; + else + patchShebangs --host "$prefix"; + fi; + fi +} +pkgConfigWrapper_addPkgConfigPath () +{ + + local role_post; + getHostRoleEnvHook; + addToSearchPath "PKG_CONFIG_PATH${role_post}" "$1/lib/pkgconfig"; + addToSearchPath "PKG_CONFIG_PATH${role_post}" "$1/share/pkgconfig" +} +prependToVar () +{ + + local -n nameref="$1"; + local useArray type; + if [ -n "$__structuredAttrs" ]; then + useArray=true; + else + useArray=false; + fi; + if type=$(declare -p "$1" 2> /dev/null); then + case "${type#* }" in + -A*) + echo "prependToVar(): ERROR: trying to use prependToVar on an associative array." 1>&2; + return 1 + ;; + -a*) + useArray=true + ;; + *) + useArray=false + ;; + esac; + fi; + shift; + if $useArray; then + nameref=("$@" ${nameref+"${nameref[@]}"}); + else + nameref="$* ${nameref-}"; + fi +} +printLines () +{ + + (( "$#" > 0 )) || return 0; + printf '%s\n' "$@" +} +printWords () +{ + + (( "$#" > 0 )) || return 0; + printf '%s ' "$@" +} +recordPropagatedDependencies () +{ + + declare -ra flatVars=(depsBuildBuildPropagated propagatedNativeBuildInputs depsBuildTargetPropagated depsHostHostPropagated propagatedBuildInputs depsTargetTargetPropagated); + declare -ra flatFiles=("${propagatedBuildDepFiles[@]}" "${propagatedHostDepFiles[@]}" "${propagatedTargetDepFiles[@]}"); + local propagatedInputsIndex; + for propagatedInputsIndex in "${!flatVars[@]}"; + do + local propagatedInputsSlice="${flatVars[$propagatedInputsIndex]}[@]"; + local propagatedInputsFile="${flatFiles[$propagatedInputsIndex]}"; + [[ -n "${!propagatedInputsSlice}" ]] || continue; + mkdir -p "${!outputDev}/nix-support"; + printWords ${!propagatedInputsSlice} > "${!outputDev}/nix-support/$propagatedInputsFile"; + done +} +runHook () +{ + + local hookName="$1"; + shift; + local hooksSlice="${hookName%Hook}Hooks[@]"; + local hook; + for hook in "_callImplicitHook 0 $hookName" ${!hooksSlice+"${!hooksSlice}"}; + do + _logHook "$hookName" "$hook" "$@"; + _eval "$hook" "$@"; + done; + return 0 +} +runOneHook () +{ + + local hookName="$1"; + shift; + local hooksSlice="${hookName%Hook}Hooks[@]"; + local hook ret=1; + for hook in "_callImplicitHook 1 $hookName" ${!hooksSlice+"${!hooksSlice}"}; + do + _logHook "$hookName" "$hook" "$@"; + if _eval "$hook" "$@"; then + ret=0; + break; + fi; + done; + return "$ret" +} +runPhase () +{ + + local curPhase="$*"; + if [[ "$curPhase" = unpackPhase && -n "${dontUnpack:-}" ]]; then + return; + fi; + if [[ "$curPhase" = patchPhase && -n "${dontPatch:-}" ]]; then + return; + fi; + if [[ "$curPhase" = configurePhase && -n "${dontConfigure:-}" ]]; then + return; + fi; + if [[ "$curPhase" = buildPhase && -n "${dontBuild:-}" ]]; then + return; + fi; + if [[ "$curPhase" = checkPhase && -z "${doCheck:-}" ]]; then + return; + fi; + if [[ "$curPhase" = installPhase && -n "${dontInstall:-}" ]]; then + return; + fi; + if [[ "$curPhase" = fixupPhase && -n "${dontFixup:-}" ]]; then + return; + fi; + if [[ "$curPhase" = installCheckPhase && -z "${doInstallCheck:-}" ]]; then + return; + fi; + if [[ "$curPhase" = distPhase && -z "${doDist:-}" ]]; then + return; + fi; + showPhaseHeader "$curPhase"; + dumpVars; + local startTime endTime; + startTime=$(date +"%s"); + eval "${!curPhase:-$curPhase}"; + endTime=$(date +"%s"); + showPhaseFooter "$curPhase" "$startTime" "$endTime"; + if [ "$curPhase" = unpackPhase ]; then + [ -n "${sourceRoot:-}" ] && chmod +x -- "${sourceRoot}"; + cd -- "${sourceRoot:-.}"; + fi +} +showPhaseFooter () +{ + + local phase="$1"; + local startTime="$2"; + local endTime="$3"; + local delta=$(( endTime - startTime )); + (( delta < 30 )) && return; + local H=$((delta/3600)); + local M=$((delta%3600/60)); + local S=$((delta%60)); + echo -n "$phase completed in "; + (( H > 0 )) && echo -n "$H hours "; + (( M > 0 )) && echo -n "$M minutes "; + echo "$S seconds" +} +showPhaseHeader () +{ + + local phase="$1"; + echo "Running phase: $phase"; + if [[ -z ${NIX_LOG_FD-} ]]; then + return; + fi; + printf "@nix { \"action\": \"setPhase\", \"phase\": \"%s\" }\n" "$phase" >&"$NIX_LOG_FD" +} +stripDirs () +{ + + local cmd="$1"; + local ranlibCmd="$2"; + local paths="$3"; + local stripFlags="$4"; + local excludeFlags=(); + local pathsNew=; + [ -z "$cmd" ] && echo "stripDirs: Strip command is empty" 1>&2 && exit 1; + [ -z "$ranlibCmd" ] && echo "stripDirs: Ranlib command is empty" 1>&2 && exit 1; + local pattern; + if [ -n "${stripExclude:-}" ]; then + for pattern in "${stripExclude[@]}"; + do + excludeFlags+=(-a '!' '(' -name "$pattern" -o -wholename "$prefix/$pattern" ')'); + done; + fi; + local p; + for p in ${paths}; + do + if [ -e "$prefix/$p" ]; then + pathsNew="${pathsNew} $prefix/$p"; + fi; + done; + paths=${pathsNew}; + if [ -n "${paths}" ]; then + echo "stripping (with command $cmd and flags $stripFlags) in $paths"; + local striperr; + striperr="$(mktemp --tmpdir="$TMPDIR" 'striperr.XXXXXX')"; + find $paths -type f "${excludeFlags[@]}" -a '!' -path "$prefix/lib/debug/*" -printf '%D-%i,%p\0' | sort -t, -k1,1 -u -z | cut -d, -f2- -z | xargs -r -0 -n1 -P "$NIX_BUILD_CORES" -- $cmd $stripFlags 2> "$striperr" || exit_code=$?; + [[ "$exit_code" = 123 || -z "$exit_code" ]] || ( cat "$striperr" 1>&2 && exit 1 ); + rm "$striperr"; + find $paths -name '*.a' -type f -exec $ranlibCmd '{}' \; 2> /dev/null; + fi +} +stripHash () +{ + + local strippedName casematchOpt=0; + strippedName="$(basename -- "$1")"; + shopt -q nocasematch && casematchOpt=1; + shopt -u nocasematch; + if [[ "$strippedName" =~ ^[a-z0-9]{32}- ]]; then + echo "${strippedName:33}"; + else + echo "$strippedName"; + fi; + if (( casematchOpt )); then + shopt -s nocasematch; + fi +} +substitute () +{ + + local input="$1"; + local output="$2"; + shift 2; + if [ ! -f "$input" ]; then + echo "substitute(): ERROR: file '$input' does not exist" 1>&2; + return 1; + fi; + local content; + consumeEntire content < "$input"; + if [ -e "$output" ]; then + chmod +w "$output"; + fi; + substituteStream content "file '$input'" "$@" > "$output" +} +substituteAll () +{ + + local input="$1"; + local output="$2"; + local -a args=(); + _allFlags; + substitute "$input" "$output" "${args[@]}" +} +substituteAllInPlace () +{ + + local fileName="$1"; + shift; + substituteAll "$fileName" "$fileName" "$@" +} +substituteAllStream () +{ + + local -a args=(); + _allFlags; + substituteStream "$1" "$2" "${args[@]}" +} +substituteInPlace () +{ + + local -a fileNames=(); + for arg in "$@"; + do + if [[ "$arg" = "--"* ]]; then + break; + fi; + fileNames+=("$arg"); + shift; + done; + if ! [[ "${#fileNames[@]}" -gt 0 ]]; then + echo "substituteInPlace called without any files to operate on (files must come before options!)" 1>&2; + return 1; + fi; + for file in "${fileNames[@]}"; + do + substitute "$file" "$file" "$@"; + done +} +substituteStream () +{ + + local var=$1; + local description=$2; + shift 2; + while (( "$#" )); do + local replace_mode="$1"; + case "$1" in + --replace) + if ! "$_substituteStream_has_warned_replace_deprecation"; then + echo "substituteStream() in derivation $name: WARNING: '--replace' is deprecated, use --replace-{fail,warn,quiet}. ($description)" 1>&2; + _substituteStream_has_warned_replace_deprecation=true; + fi; + replace_mode='--replace-warn' + ;& + --replace-quiet | --replace-warn | --replace-fail) + pattern="$2"; + replacement="$3"; + shift 3; + local savedvar; + savedvar="${!var}"; + eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}'; + if [ "$pattern" != "$replacement" ]; then + if [ "${!var}" == "$savedvar" ]; then + if [ "$replace_mode" == --replace-warn ]; then + printf "substituteStream() in derivation $name: WARNING: pattern %q doesn't match anything in %s\n" "$pattern" "$description" 1>&2; + else + if [ "$replace_mode" == --replace-fail ]; then + printf "substituteStream() in derivation $name: ERROR: pattern %q doesn't match anything in %s\n" "$pattern" "$description" 1>&2; + return 1; + fi; + fi; + fi; + fi + ;; + --subst-var) + local varName="$2"; + shift 2; + if ! [[ "$varName" =~ ^[a-zA-Z_][a-zA-Z0-9_]*$ ]]; then + echo "substituteStream() in derivation $name: ERROR: substitution variables must be valid Bash names, \"$varName\" isn't." 1>&2; + return 1; + fi; + if [ -z ${!varName+x} ]; then + echo "substituteStream() in derivation $name: ERROR: variable \$$varName is unset" 1>&2; + return 1; + fi; + pattern="@$varName@"; + replacement="${!varName}"; + eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}' + ;; + --subst-var-by) + pattern="@$2@"; + replacement="$3"; + eval "$var"'=${'"$var"'//"$pattern"/"$replacement"}'; + shift 3 + ;; + *) + echo "substituteStream() in derivation $name: ERROR: Invalid command line argument: $1" 1>&2; + return 1 + ;; + esac; + done; + printf "%s" "${!var}" +} +unpackFile () +{ + + curSrc="$1"; + echo "unpacking source archive $curSrc"; + if ! runOneHook unpackCmd "$curSrc"; then + echo "do not know how to unpack source archive $curSrc"; + exit 1; + fi +} +unpackPhase () +{ + + runHook preUnpack; + if [ -z "${srcs:-}" ]; then + if [ -z "${src:-}" ]; then + echo 'variable $src or $srcs should point to the source'; + exit 1; + fi; + srcs="$src"; + fi; + local -a srcsArray; + concatTo srcsArray srcs; + local dirsBefore=""; + for i in *; + do + if [ -d "$i" ]; then + dirsBefore="$dirsBefore $i "; + fi; + done; + for i in "${srcsArray[@]}"; + do + unpackFile "$i"; + done; + : "${sourceRoot=}"; + if [ -n "${setSourceRoot:-}" ]; then + runOneHook setSourceRoot; + else + if [ -z "$sourceRoot" ]; then + for i in *; + do + if [ -d "$i" ]; then + case $dirsBefore in + *\ $i\ *) + + ;; + *) + if [ -n "$sourceRoot" ]; then + echo "unpacker produced multiple directories"; + exit 1; + fi; + sourceRoot="$i" + ;; + esac; + fi; + done; + fi; + fi; + if [ -z "$sourceRoot" ]; then + echo "unpacker appears to have produced no directories"; + exit 1; + fi; + echo "source root is $sourceRoot"; + if [ "${dontMakeSourcesWritable:-0}" != 1 ]; then + chmod -R u+w -- "$sourceRoot"; + fi; + runHook postUnpack +} +updateAutotoolsGnuConfigScriptsPhase () +{ + + if [ -n "${dontUpdateAutotoolsGnuConfigScripts-}" ]; then + return; + fi; + for script in config.sub config.guess; + do + for f in $(find . -type f -name "$script"); + do + echo "Updating Autotools / GNU config script to a newer upstream version: $f"; + cp -f "/nix/store/0yl1wf4jim6830k2m3c3v5kyp2l9z8ay-gnu-config-2024-01-01/$script" "$f"; + done; + done +} +updateSourceDateEpoch () +{ + + local path="$1"; + [[ $path == -* ]] && path="./$path"; + local -a res=($(find "$path" -type f -not -newer "$NIX_BUILD_TOP/.." -printf '%T@ %p\0' | sort -n --zero-terminated | tail -n1 --zero-terminated | head -c -1)); + local time="${res[0]//\.[0-9]*/}"; + local newestFile="${res[1]}"; + if [ "${time:-0}" -gt "$SOURCE_DATE_EPOCH" ]; then + echo "setting SOURCE_DATE_EPOCH to timestamp $time of file $newestFile"; + export SOURCE_DATE_EPOCH="$time"; + local now="$(date +%s)"; + if [ "$time" -gt $((now - 60)) ]; then + echo "warning: file $newestFile may be generated; SOURCE_DATE_EPOCH may be non-deterministic"; + fi; + fi +} +PATH="$PATH${nix_saved_PATH:+:$nix_saved_PATH}" +XDG_DATA_DIRS="$XDG_DATA_DIRS${nix_saved_XDG_DATA_DIRS:+:$nix_saved_XDG_DATA_DIRS}" +export NIX_BUILD_TOP="$(mktemp -d -t nix-shell.XXXXXX)" +export TMP="$NIX_BUILD_TOP" +export TMPDIR="$NIX_BUILD_TOP" +export TEMP="$NIX_BUILD_TOP" +export TEMPDIR="$NIX_BUILD_TOP" +eval "${shellHook:-}" diff --git a/lib/config/config.hpp b/lib/config/config.hpp new file mode 100644 index 0000000..3fcfbd3 --- /dev/null +++ b/lib/config/config.hpp @@ -0,0 +1,54 @@ +#pragma once + +#include +#include +#include + +namespace anthracite::config { + class http_config { + uint16_t _port; + public: + http_config(uint16_t port) : _port(port) {}; + virtual ~http_config() {}; + + uint16_t port() { return _port; } + }; + + class https_config : public http_config { + std::string _cert_path; + std::string _key_path; + public: + https_config(uint16_t port, std::string cert_path, std::string key_path) : + http_config(port), _cert_path(cert_path), _key_path(key_path) {}; + }; + + class config { + uint16_t _worker_threads; + std::optional _http_config; + std::optional _https_config; + + public: + config(uint16_t worker_threads) : _worker_threads(worker_threads) { + } + + void add_http_config(http_config config) { + _http_config = config; + } + + void add_https_config(https_config config) { + _https_config = config; + } + + uint16_t worker_threads() { + return _worker_threads; + } + + std::optional& http_cfg() { + return _http_config; + } + + std::optional& https_cfg() { + return _https_config; + } + }; +}; diff --git a/lib/thread_mgr/event_loop.cpp b/lib/thread_mgr/event_loop.cpp new file mode 100644 index 0000000..1a9c55f --- /dev/null +++ b/lib/thread_mgr/event_loop.cpp @@ -0,0 +1,176 @@ +#include "./event_loop.hpp" +#include "../log/log.hpp" +#include "../socket/openssl_socket.hpp" +#include +#include +#include + +using std::chrono::high_resolution_clock; +using std::chrono::duration_cast; +using std::chrono::duration; +using std::chrono::milliseconds; + +namespace anthracite::thread_mgr { + event_loop::event::event(socket::anthracite_socket* socket) : + _socket(socket) {} + + socket::anthracite_socket* event_loop::event::socket() { + return _socket; + } + + event_loop::event_loop(backends::backend& backend, config::config& config) : thread_mgr(backend, config), _error_backend("./www") {} + + bool event_loop::event_handler(event& event) { + std::string raw_request = event.socket()->recv_message(http::HEADER_BYTES); + + // We're doing the start here even though it would ideally be done + // before the first line since if we leave the connection open for + // HTTP 1.1, we can spend a bit of time waiting + auto start = high_resolution_clock::now(); + + if (raw_request == "") { + event.socket()->close_conn(); + delete event.socket(); + return false; + } + + http::request req(raw_request, event.socket()->get_client_ip()); + std::unique_ptr resp = req.is_supported_version() ? _backend.handle_request(req) : _error_backend.handle_error(http::status_codes::HTTP_VERSION_NOT_SUPPORTED); + std::string header = resp->header_to_string(); + event.socket()->send_message(header); + event.socket()->send_message(resp->content()); + + auto end = high_resolution_clock::now(); + auto ms_int = duration_cast(end-start); + //log_request_and_response(req, resp , ms_int.count()); + + resp.reset(); + if (req.close_connection()) { + event.socket()->close_conn(); + delete event.socket(); + return false; + } + + return true; + } + + void event_loop::worker_thread_loop(int threadno) { + unsigned char buf[sizeof(class event)]; + + std::osyncstream(log::info) << "Starting worker thread " << threadno << std::endl; + while(_run) { + // Get event from queue + std::unique_lock lock(_event_mtx); + + + event* ev = nullptr; + + if (_events.size() > 0) { + ev = new (buf) event(_events.back()); + _events.pop(); + lock.unlock(); + } else { + _event_cv.wait(lock, [this]{ return this->_events.size() > 0 || !_run; }); + + if (!_run) { + break; + } + + ev = new (buf) event(_events.back()); + _events.pop(); + lock.unlock(); + } + + + // process + bool requeue = event_handler(*ev); + + // if necessary, requeue + if (requeue) { + { + std::lock_guard lg(_event_mtx); + _events.push(*ev); + } + _event_cv.notify_one(); + } + } + + std::osyncstream(log::info) << "Stopping worker thread " << threadno << std::endl; + } + + void event_loop::listener_thread_loop(config::http_config& http_config) { + socket::anthracite_socket* socket; + + config::http_config* http_ptr = &http_config; + config::https_config* https_ptr = dynamic_cast(http_ptr); + + bool is_tls = https_ptr != nullptr; + + if (is_tls){ + socket = new socket::openssl_socket(https_ptr->port()); + } else { + socket = new socket::anthracite_socket(http_ptr->port()); + } + + std::osyncstream(log::info) << "Listening for " << (is_tls ? "HTTPS" : "HTTP" ) << " connections on port " << http_ptr->port() << std::endl; + + while (_run) { + if(socket->wait_for_conn()) { + socket::anthracite_socket* client_sock; + + if (is_tls){ + socket::openssl_socket* ssl_sock = dynamic_cast(socket); + client_sock = new socket::openssl_socket(*ssl_sock); + } else { + client_sock = new socket::anthracite_socket(*socket); + } + + std::lock_guard lg(_event_mtx); + _events.push(event(client_sock)); + _event_cv.notify_one(); + } + } + + std::osyncstream(log::info) << "Stopping listening for " << (is_tls ? "HTTPS" : "HTTP") << " connections on port " << http_ptr->port() << std::endl; + + delete socket; + } + + void event_loop::start() { + log::info << "Starting event_loop Thread Manager" << std::endl; + + _run = true; + + std::vector listener_threads; + std::vector worker_threads; + + for(int i = 0; i < _config.worker_threads(); i++) { + auto thread = std::thread(&event_loop::worker_thread_loop, this, i); + worker_threads.push_back(std::move(thread)); + } + + if (_config.http_cfg().has_value()) { + auto thread = std::thread(&event_loop::listener_thread_loop, this, std::ref(_config.http_cfg().value())); + listener_threads.push_back(std::move(thread)); + } + + if (_config.https_cfg().has_value()) { + auto thread = std::thread(&event_loop::listener_thread_loop, this, std::ref(_config.https_cfg().value())); + listener_threads.push_back(std::move(thread)); + } + + for(std::thread& t : worker_threads) { + t.join(); + } + + for(std::thread& t : listener_threads) { + t.join(); + } + } + + void event_loop::stop() { + _run = false; + std::lock_guard lg(_event_mtx); + _event_cv.notify_all(); + } +} diff --git a/lib/thread_mgr/event_loop.hpp b/lib/thread_mgr/event_loop.hpp new file mode 100644 index 0000000..917af6b --- /dev/null +++ b/lib/thread_mgr/event_loop.hpp @@ -0,0 +1,31 @@ +#include "./thread_mgr.hpp" +#include "../socket/socket.hpp" +#include "../backends/file_backend.hpp" +#include +#include +#include + +namespace anthracite::thread_mgr { + class event_loop : public virtual thread_mgr { + class event { + socket::anthracite_socket* _socket; + public: + event(socket::anthracite_socket* socket); + socket::anthracite_socket* socket(); + }; + + std::mutex _event_mtx; + std::condition_variable _event_cv; + std::queue _events; + backends::file_backend _error_backend; + + void worker_thread_loop(int threadno); + void listener_thread_loop(config::http_config& http_config); + bool event_handler(event& ev); + + public: + event_loop(backends::backend& backend, config::config& config); + void start() override; + void stop() override; + }; +}; diff --git a/lib/thread_mgr/thread_mgr.hpp b/lib/thread_mgr/thread_mgr.hpp new file mode 100644 index 0000000..faae7ec --- /dev/null +++ b/lib/thread_mgr/thread_mgr.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include "../backends/backend.hpp" +#include "../config/config.hpp" + +namespace anthracite::thread_mgr { + class thread_mgr { + protected: + bool _run; + backends::backend& _backend; + config::config& _config; + public: + thread_mgr(backends::backend& backend, config::config& config): _backend(backend), _config(config) {} + virtual ~thread_mgr() = default; + virtual void start() = 0; + virtual void stop() = 0; + }; +}; From 409024e04adb6459d1baf0ab54ecf28569b07a4f Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Fri, 21 Feb 2025 18:24:28 -0500 Subject: [PATCH 08/15] polished up event loop changes --- CHANGELOG.md | 3 +- CMakeLists.txt | 3 - README.md | 5 +- lib/anthracite.cpp | 44 ----- lib/anthracite.hpp | 6 - lib/config/config.hpp | 11 +- lib/http/request.cpp | 145 +++++++-------- lib/log/log.cpp | 5 + lib/log/log.hpp | 8 +- lib/socket/openssl_socket.cpp | 22 ++- lib/socket/socket.cpp | 6 +- lib/socket/socket.hpp | 2 + lib/thread_mgr/event_loop.cpp | 329 +++++++++++++++++++--------------- lib/thread_mgr/event_loop.hpp | 7 +- src/api_main.cpp | 112 ------------ src/file_main.cpp | 34 +++- tests/speed_tests.cpp | 31 ++-- tests/unit_tests.cpp | 9 +- 18 files changed, 354 insertions(+), 428 deletions(-) delete mode 100644 lib/anthracite.cpp delete mode 100644 lib/anthracite.hpp delete mode 100644 src/api_main.cpp diff --git a/CHANGELOG.md b/CHANGELOG.md index b26e941..c3e9edb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # 0.3.0 - SSL support via OpenSSL -- Switched from thread per connection to event driven threading model +- Added "Thread Manager" class to allow for multiple (or custom) threading models (process per thread, event loop) +- Defaul threading model is now event loop - Rewrote request parser for readability and speed - Added improved logging with different log levels - Separated anthracite into libanthracite and anthracite-bin to allow for other projects to implement anthracite (example in ./src/api_main.cpp) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2449bbd..a1fbdf8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,9 +39,6 @@ target_link_libraries(anthracite-bin anthracite) add_dependencies(anthracite-bin build-supplemental) add_dependencies(anthracite-bin anthracite) -add_executable(anthracite-api-bin src/api_main.cpp) -target_link_libraries(anthracite-api-bin anthracite) - include(FetchContent) FetchContent_Declare( googletest diff --git a/README.md b/README.md index 0352b11..629ba55 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # Anthracite -A simple web server written in C++. Supports HTTP 1.0 & 1.1. + +Anthracite is an extensible, low-dependency, fast web server. ## Developing @@ -16,7 +17,7 @@ Create a `build/` directory, run `cmake ..`, and then `make` to build. - [x] HTTP/1.1 - [x] Enhance logging - [x] Create library that can be used to implement custom backends (i.e. webapi, fileserver, etc) -- [ ] Faster parsing +- [x] Faster parsing - [ ] HTTP/2 - [ ] Improve benchmarking infrastructure - [ ] Fix glaring security issues diff --git a/lib/anthracite.cpp b/lib/anthracite.cpp deleted file mode 100644 index 450deaf..0000000 --- a/lib/anthracite.cpp +++ /dev/null @@ -1,44 +0,0 @@ -#include "./anthracite.hpp" -#include "./log/log.hpp" -#include -#include -#include -#include -#include "./config/config.hpp" -#include "./thread_mgr/event_loop.hpp" -#include - -using namespace anthracite; - -void log_request_and_response(http::request& req, std::unique_ptr& resp, uint32_t micros); - -constexpr int default_port = 80; -constexpr int max_worker_threads = 128; - -thread_mgr::event_loop* elp = nullptr; - - -extern "C" void signalHandler(int signum) { - log::warn << "Caught signal #" << signum << ", exiting Anthracite" << std::endl; - elp->stop(); -} - - -int anthracite_main(backends::backend& be, config::config& config) -{ - signal(SIGTERM, signalHandler); - signal(SIGSEGV, signalHandler); - signal(SIGINT, signalHandler); - signal(SIGABRT, signalHandler); - - log::logger.initialize(log::LOG_LEVEL_INFO); - thread_mgr::event_loop el(be, config); - elp = ⪙ - el.start(); - return 0; -} - -void log_request_and_response(http::request& req, std::unique_ptr& resp, uint32_t micros) -{ - log::info << "[" << resp->status_code() << " " + http::status_map.find(resp->status_code())->second + "] " + req.client_ip() + " " + http::reverse_method_map.find(req.get_method())->second + " " + req.path() << " in " << micros << " usecs" << std::endl; -} diff --git a/lib/anthracite.hpp b/lib/anthracite.hpp deleted file mode 100644 index f342344..0000000 --- a/lib/anthracite.hpp +++ /dev/null @@ -1,6 +0,0 @@ -#include "backends/backend.hpp" -#include "config/config.hpp" - -using namespace anthracite; - -int anthracite_main(backends::backend& be, config::config& cfg); diff --git a/lib/config/config.hpp b/lib/config/config.hpp index 3fcfbd3..f353a21 100644 --- a/lib/config/config.hpp +++ b/lib/config/config.hpp @@ -23,12 +23,13 @@ namespace anthracite::config { }; class config { - uint16_t _worker_threads; + int _worker_threads; + int _max_clients; std::optional _http_config; std::optional _https_config; public: - config(uint16_t worker_threads) : _worker_threads(worker_threads) { + config(int worker_threads, int max_clients) : _worker_threads(worker_threads), _max_clients(max_clients) { } void add_http_config(http_config config) { @@ -39,9 +40,13 @@ namespace anthracite::config { _https_config = config; } - uint16_t worker_threads() { + int worker_threads() { return _worker_threads; } + + int max_clients() { + return _max_clients; + } std::optional& http_cfg() { return _http_config; diff --git a/lib/http/request.cpp b/lib/http/request.cpp index 75d90df..ca4b648 100644 --- a/lib/http/request.cpp +++ b/lib/http/request.cpp @@ -1,87 +1,91 @@ #include "request.hpp" #include "../log/log.hpp" #include "constants.hpp" -#include #include +#include #include namespace anthracite::http { -void request::parse_header(std::string& raw_line) { +void request::parse_header(std::string& raw_line) +{ auto delim_pos = raw_line.find_first_of(':'); - auto value_pos = raw_line.find_first_not_of(' ', delim_pos+1); + auto value_pos = raw_line.find_first_not_of(' ', delim_pos + 1); - std::string header_name = raw_line.substr(0,delim_pos); + std::string header_name = raw_line.substr(0, delim_pos); std::string header_val = raw_line.substr(value_pos); _headers[header_name] = header_val; } -void request::parse_query_param(std::string& raw_param) { +void request::parse_query_param(std::string& raw_param) +{ auto delim_pos = raw_param.find_first_of('='); - auto value_pos = delim_pos+1; + auto value_pos = delim_pos + 1; - std::string query_name = raw_param.substr(0,delim_pos); + std::string query_name = raw_param.substr(0, delim_pos); std::string query_val = raw_param.substr(value_pos); _query_params[query_name] = query_val; } -void request::parse_path(char* raw_path) { +void request::parse_path(char* raw_path) +{ char* saveptr = nullptr; char* tok = strtok_r(raw_path, "?", &saveptr); - if (tok){ + if (tok) { _path = tok; } tok = strtok_r(nullptr, "?", &saveptr); - while(tok) { + while (tok) { std::string rtok(tok); parse_query_param(rtok); tok = strtok_r(nullptr, "?", &saveptr); } } -void request::parse_request_line(char* raw_line) { - request_line_parser_state state = METHOD; +void request::parse_request_line(char* raw_line) +{ + request_line_parser_state state = METHOD; - char* saveptr = nullptr; - char* tok = strtok_r(raw_line, " \r", &saveptr); + char* saveptr = nullptr; + char* tok = strtok_r(raw_line, " \r", &saveptr); - while(tok){ - switch(state) { - case METHOD: { - auto search = method_map.find(tok); - if (search != method_map.end()) { - _method = search->second; - } else { - _method = method::UNKNOWN; - } - - state = PATH; - break; - }; - - case PATH: { - std::string str_tok(tok); - parse_path(tok); - state = VERSION; - break; - }; - - case VERSION: { - auto search = version_map.find(tok); - if (search != version_map.end()) { - _http_version = search->second; - } else { - _http_version = version::HTTP_1_0; - } - return; - }; + while (tok) { + switch (state) { + case METHOD: { + auto search = method_map.find(tok); + if (search != method_map.end()) { + _method = search->second; + } else { + _method = method::UNKNOWN; } - tok = strtok_r(nullptr, " \r", &saveptr); + + state = PATH; + break; + }; + + case PATH: { + std::string str_tok(tok); + parse_path(tok); + state = VERSION; + break; + }; + + case VERSION: { + auto search = version_map.find(tok); + if (search != version_map.end()) { + _http_version = search->second; + } else { + _http_version = version::HTTP_1_0; + } + return; + }; } + tok = strtok_r(nullptr, " \r", &saveptr); + } } request::request(std::string& raw_data, const std::string& client_ip) @@ -94,26 +98,27 @@ request::request(std::string& raw_data, const std::string& client_ip) char* saveptr = nullptr; char* tok = strtok_r(raw_data.data(), "\r\n", &saveptr); - while(tok && state != BODY_CONTENT){ - switch(state) { - case REQUEST_LINE: { - parse_request_line(tok); - state = HEADERS; - tok = strtok_r(nullptr, "\n", &saveptr); - break; - }; - case HEADERS: { - if (tok[0] == '\r') { - state = BODY_CONTENT; - } else { - std::string rtok(tok); - rtok.pop_back(); - parse_header(rtok); + while (tok && state != BODY_CONTENT) { + switch (state) { + case REQUEST_LINE: { + parse_request_line(tok); + state = HEADERS; + tok = strtok_r(nullptr, "\n", &saveptr); + break; + }; + case HEADERS: { + if (tok[0] == '\r') { + state = BODY_CONTENT; + } else { + std::string rtok(tok); + rtok.pop_back(); + parse_header(rtok); tok = strtok_r(nullptr, "\n", &saveptr); - } - break; - }; - case BODY_CONTENT: break; + } + break; + }; + case BODY_CONTENT: + break; } } @@ -121,9 +126,9 @@ request::request(std::string& raw_data, const std::string& client_ip) if (tok) { _body_content = std::string(tok); } - //if (getline(line_stream, line, '\0')) { - // _body_content = line; - //} + // if (getline(line_stream, line, '\0')) { + // _body_content = line; + // } } std::string request::path() { return _path; } @@ -147,11 +152,11 @@ bool request::close_connection() const auto& header = _headers.find("Connection"); const bool found = header != _headers.end(); - if (found && header->second == "keep-alive") { - return false; + if (found && header->second == "close") { + return true; } - return true; + return false; } std::string request::to_string() diff --git a/lib/log/log.cpp b/lib/log/log.cpp index 4c59e8b..63399d0 100644 --- a/lib/log/log.cpp +++ b/lib/log/log.cpp @@ -10,6 +10,11 @@ void Logger::initialize(enum LOG_LEVEL level) _level = level; } +void Logger::log_request_and_response(http::request& req, std::unique_ptr& resp, uint32_t micros) +{ + log::info << "[" << resp->status_code() << " " + http::status_map.find(resp->status_code())->second + "] " + req.client_ip() + " " + http::reverse_method_map.find(req.get_method())->second + " " + req.path() << " in " << micros << " usecs" << std::endl; +} + LogBuf::LogBuf(std::ostream& output_stream, const std::string& tag, enum LOG_LEVEL level) : _output_stream(output_stream) , _tag(tag) diff --git a/lib/log/log.hpp b/lib/log/log.hpp index ba1bb05..6a99749 100644 --- a/lib/log/log.hpp +++ b/lib/log/log.hpp @@ -3,6 +3,10 @@ #include #include #include +#include +#include +#include "../http/request.hpp" +#include "../http/response.hpp" namespace anthracite::log { enum LOG_LEVEL { @@ -21,9 +25,9 @@ namespace anthracite::log { Logger(); void initialize(enum LOG_LEVEL level); + void log_request_and_response(http::request& req, std::unique_ptr& resp, uint32_t micros); }; - class LogBuf : public std::stringbuf { std::string _tag; @@ -50,4 +54,6 @@ namespace anthracite::log { static class LogBuf debugBuf{std::cout, "DEBG", LOG_LEVEL_DEBUG}; static std::ostream debug(&debugBuf); + + }; diff --git a/lib/socket/openssl_socket.cpp b/lib/socket/openssl_socket.cpp index eddf655..9bb4639 100644 --- a/lib/socket/openssl_socket.cpp +++ b/lib/socket/openssl_socket.cpp @@ -1,18 +1,18 @@ -#include -#include -#include #include "./openssl_socket.hpp" +#include "../log/log.hpp" #include #include +#include +#include #include #include +#include +#include #include #include #include #include #include -#include -#include "../log/log.hpp" namespace anthracite::socket { @@ -21,7 +21,7 @@ SSL_CTX* openssl_socket::_context = nullptr; openssl_socket::openssl_socket(int port, int max_queue) : anthracite_socket(port, max_queue) { - const SSL_METHOD *method = TLS_server_method(); + const SSL_METHOD* method = TLS_server_method(); if (_context == nullptr) { _context = SSL_CTX_new(method); @@ -37,7 +37,7 @@ openssl_socket::openssl_socket(int port, int max_queue) throw std::exception(); } - if (SSL_CTX_use_PrivateKey_file(_context, "key.pem", SSL_FILETYPE_PEM) <= 0 ) { + if (SSL_CTX_use_PrivateKey_file(_context, "key.pem", SSL_FILETYPE_PEM) <= 0) { log::err << "Unable to open key.pem" << std::endl; throw std::exception(); } @@ -48,11 +48,11 @@ openssl_socket::~openssl_socket() = default; bool openssl_socket::wait_for_conn() { client_ip = ""; - struct timeval tv = {.tv_sec = 1, .tv_usec = 0}; + struct timeval tv = { .tv_sec = 1, .tv_usec = 0 }; fd_set read_fd; FD_ZERO(&read_fd); FD_SET(server_socket, &read_fd); - if (select(server_socket+1, &read_fd, NULL, NULL, &wait_timeout)) { + if (select(server_socket + 1, &read_fd, NULL, NULL, &wait_timeout)) { client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); std::array ip_str { 0 }; inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN); @@ -70,8 +70,6 @@ bool openssl_socket::wait_for_conn() } else { return false; } - - } void openssl_socket::close_conn() @@ -98,7 +96,7 @@ std::string openssl_socket::recv_message(int buffer_size) setsockopt(client_socket, SOL_SOCKET, SO_RCVTIMEO, &timeout_tv, sizeof timeout_tv); std::vector response(buffer_size + 1); - ssize_t result = SSL_read(_ssl, response.data(), buffer_size+1); + ssize_t result = SSL_read(_ssl, response.data(), buffer_size + 1); if (result < 1) { return ""; diff --git a/lib/socket/socket.cpp b/lib/socket/socket.cpp index 5a54ac1..f8ec4ae 100644 --- a/lib/socket/socket.cpp +++ b/lib/socket/socket.cpp @@ -11,7 +11,7 @@ namespace anthracite::socket { -const struct timeval anthracite_socket::timeout_tv = { .tv_sec = 5, .tv_usec = 0}; +const struct timeval anthracite_socket::timeout_tv = { .tv_sec = 5, .tv_usec = 0 }; anthracite_socket::anthracite_socket(int port, int max_queue) : server_socket(::socket(AF_INET, SOCK_STREAM, 0)) @@ -32,11 +32,11 @@ anthracite_socket::anthracite_socket(int port, int max_queue) bool anthracite_socket::wait_for_conn() { client_ip = ""; - struct timeval tv = {.tv_sec = 1, .tv_usec = 0}; + struct timeval tv = { .tv_sec = 1, .tv_usec = 0 }; fd_set read_fd; FD_ZERO(&read_fd); FD_SET(server_socket, &read_fd); - if (select(server_socket+1, &read_fd, NULL, NULL, &wait_timeout)) { + if (select(server_socket + 1, &read_fd, NULL, NULL, &wait_timeout)) { client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); std::array ip_str { 0 }; inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN); diff --git a/lib/socket/socket.hpp b/lib/socket/socket.hpp index 5df0fde..5ac3562 100644 --- a/lib/socket/socket.hpp +++ b/lib/socket/socket.hpp @@ -32,6 +32,8 @@ public: virtual void close_conn(); virtual void send_message(std::string& msg); virtual std::string recv_message(int buffer_size); + + int csock() { return client_socket; } }; }; diff --git a/lib/thread_mgr/event_loop.cpp b/lib/thread_mgr/event_loop.cpp index 1a9c55f..1cf4d2a 100644 --- a/lib/thread_mgr/event_loop.cpp +++ b/lib/thread_mgr/event_loop.cpp @@ -1,176 +1,217 @@ #include "./event_loop.hpp" #include "../log/log.hpp" #include "../socket/openssl_socket.hpp" -#include +#include "assert.h" +#include "sys/epoll.h" #include +#include #include -using std::chrono::high_resolution_clock; -using std::chrono::duration_cast; using std::chrono::duration; +using std::chrono::duration_cast; +using std::chrono::high_resolution_clock; using std::chrono::milliseconds; namespace anthracite::thread_mgr { - event_loop::event::event(socket::anthracite_socket* socket) : - _socket(socket) {} +event_loop::event::event(socket::anthracite_socket* socket, std::chrono::time_point timestamp) + : _socket(socket) + , _ts(timestamp) +{ +} - socket::anthracite_socket* event_loop::event::socket() { - return _socket; +socket::anthracite_socket* event_loop::event::socket() +{ + return _socket; +} + +std::chrono::time_point& event_loop::event::timestamp() +{ + return _ts; +} + +event_loop::event_loop(backends::backend& backend, config::config& config) + : thread_mgr(backend, config) + , _error_backend("./www") +{ +} + +bool event_loop::event_handler(event& event) +{ + std::string raw_request = event.socket()->recv_message(http::HEADER_BYTES); + + if (raw_request == "") { + return false; } - event_loop::event_loop(backends::backend& backend, config::config& config) : thread_mgr(backend, config), _error_backend("./www") {} + http::request req(raw_request, event.socket()->get_client_ip()); + std::unique_ptr resp = req.is_supported_version() ? _backend.handle_request(req) : _error_backend.handle_error(http::status_codes::HTTP_VERSION_NOT_SUPPORTED); + std::string header = resp->header_to_string(); + event.socket()->send_message(header); + event.socket()->send_message(resp->content()); - bool event_loop::event_handler(event& event) { - std::string raw_request = event.socket()->recv_message(http::HEADER_BYTES); - - // We're doing the start here even though it would ideally be done - // before the first line since if we leave the connection open for - // HTTP 1.1, we can spend a bit of time waiting - auto start = high_resolution_clock::now(); - - if (raw_request == "") { - event.socket()->close_conn(); - delete event.socket(); - return false; - } - - http::request req(raw_request, event.socket()->get_client_ip()); - std::unique_ptr resp = req.is_supported_version() ? _backend.handle_request(req) : _error_backend.handle_error(http::status_codes::HTTP_VERSION_NOT_SUPPORTED); - std::string header = resp->header_to_string(); - event.socket()->send_message(header); - event.socket()->send_message(resp->content()); - - auto end = high_resolution_clock::now(); - auto ms_int = duration_cast(end-start); - //log_request_and_response(req, resp , ms_int.count()); - - resp.reset(); - if (req.close_connection()) { - event.socket()->close_conn(); - delete event.socket(); - return false; - } + auto end = high_resolution_clock::now(); + auto ms_int = duration_cast(end - event.timestamp()); + log::logger.log_request_and_response(req, resp, ms_int.count()); - return true; + resp.reset(); + if (req.close_connection()) { + return false; } - void event_loop::worker_thread_loop(int threadno) { - unsigned char buf[sizeof(class event)]; + return true; +} - std::osyncstream(log::info) << "Starting worker thread " << threadno << std::endl; - while(_run) { - // Get event from queue - std::unique_lock lock(_event_mtx); +void event_loop::worker_thread_loop(int threadno) +{ + unsigned char buf[sizeof(class event)]; + std::osyncstream(log::info) << "Starting worker thread " << threadno << std::endl; + while (_run) { + // Get event from queue + std::unique_lock lock(_event_mtx); - event* ev = nullptr; + event* ev = nullptr; - if (_events.size() > 0) { - ev = new (buf) event(_events.back()); - _events.pop(); - lock.unlock(); - } else { - _event_cv.wait(lock, [this]{ return this->_events.size() > 0 || !_run; }); - - if (!_run) { - break; - } - - ev = new (buf) event(_events.back()); - _events.pop(); - lock.unlock(); + if (_events.size() > 0) { + if (!lock.owns_lock()) { + lock.lock(); } - - - // process - bool requeue = event_handler(*ev); - - // if necessary, requeue - if (requeue) { - { - std::lock_guard lg(_event_mtx); - _events.push(*ev); - } - _event_cv.notify_one(); - } - } - - std::osyncstream(log::info) << "Stopping worker thread " << threadno << std::endl; - } - - void event_loop::listener_thread_loop(config::http_config& http_config) { - socket::anthracite_socket* socket; - - config::http_config* http_ptr = &http_config; - config::https_config* https_ptr = dynamic_cast(http_ptr); - - bool is_tls = https_ptr != nullptr; - - if (is_tls){ - socket = new socket::openssl_socket(https_ptr->port()); + assert(lock.owns_lock()); + ev = new (buf) event(_events.front()); + _events.pop(); + lock.unlock(); } else { - socket = new socket::anthracite_socket(http_ptr->port()); - } + _event_cv.wait(lock, [this] { return this->_events.size() > 0 || !_run; }); - std::osyncstream(log::info) << "Listening for " << (is_tls ? "HTTPS" : "HTTP" ) << " connections on port " << http_ptr->port() << std::endl; - - while (_run) { - if(socket->wait_for_conn()) { - socket::anthracite_socket* client_sock; - - if (is_tls){ - socket::openssl_socket* ssl_sock = dynamic_cast(socket); - client_sock = new socket::openssl_socket(*ssl_sock); - } else { - client_sock = new socket::anthracite_socket(*socket); - } - - std::lock_guard lg(_event_mtx); - _events.push(event(client_sock)); - _event_cv.notify_one(); + if (!_run) { + break; } + + assert(lock.owns_lock()); + ev = new (buf) event(_events.front()); + _events.pop(); + lock.unlock(); } - std::osyncstream(log::info) << "Stopping listening for " << (is_tls ? "HTTPS" : "HTTP") << " connections on port " << http_ptr->port() << std::endl; - - delete socket; - } - - void event_loop::start() { - log::info << "Starting event_loop Thread Manager" << std::endl; - - _run = true; - - std::vector listener_threads; - std::vector worker_threads; - - for(int i = 0; i < _config.worker_threads(); i++) { - auto thread = std::thread(&event_loop::worker_thread_loop, this, i); - worker_threads.push_back(std::move(thread)); - } - - if (_config.http_cfg().has_value()) { - auto thread = std::thread(&event_loop::listener_thread_loop, this, std::ref(_config.http_cfg().value())); - listener_threads.push_back(std::move(thread)); - } - - if (_config.https_cfg().has_value()) { - auto thread = std::thread(&event_loop::listener_thread_loop, this, std::ref(_config.https_cfg().value())); - listener_threads.push_back(std::move(thread)); - } - - for(std::thread& t : worker_threads) { - t.join(); - } - - for(std::thread& t : listener_threads) { - t.join(); + if (event_handler(*ev)) { + struct epoll_event event; + event.events = EPOLLIN; + event.data.ptr = ev->socket(); + epoll_ctl(_epoll_fd, EPOLL_CTL_ADD, ev->socket()->csock(), &event); + } else { + ev->socket()->close_conn(); + delete ev->socket(); } } - void event_loop::stop() { - _run = false; - std::lock_guard lg(_event_mtx); - _event_cv.notify_all(); + std::osyncstream(log::info) << "Stopping worker thread " << threadno << std::endl; +} + +void event_loop::eventer_thread_loop() +{ + struct epoll_event* events = new struct epoll_event[_config.max_clients()]; + std::osyncstream(log::info) << "epoll() thread started" << std::endl; + while (_run) { + int ready_fds = epoll_wait(_epoll_fd, events, _config.max_clients(), 1000); + + if (ready_fds > 0) { + std::lock_guard lg(_event_mtx); + for (int i = 0; i < ready_fds; i++) { + socket::anthracite_socket* sockptr = reinterpret_cast(events[i].data.ptr); + struct epoll_event ev; + epoll_ctl(_epoll_fd, EPOLL_CTL_DEL, sockptr->csock(), &events[i]); + _events.push(event(sockptr, std::chrono::high_resolution_clock::now())); + } + _event_cv.notify_one(); + } + } + delete[] events; + std::osyncstream(log::info) << "epoll() thread exited" << std::endl; +} + +void event_loop::listener_thread_loop(config::http_config& http_config) +{ + socket::anthracite_socket* socket; + + config::http_config* http_ptr = &http_config; + config::https_config* https_ptr = dynamic_cast(http_ptr); + + bool is_tls = https_ptr != nullptr; + + if (is_tls) { + socket = new socket::openssl_socket(https_ptr->port()); + } else { + socket = new socket::anthracite_socket(http_ptr->port()); + } + + std::osyncstream(log::info) << "Listening for " << (is_tls ? "HTTPS" : "HTTP") << " connections on port " << http_ptr->port() << std::endl; + + while (_run) { + if (socket->wait_for_conn()) { + socket::anthracite_socket* client_sock; + + if (is_tls) { + socket::openssl_socket* ssl_sock = dynamic_cast(socket); + client_sock = new socket::openssl_socket(*ssl_sock); + } else { + client_sock = new socket::anthracite_socket(*socket); + } + + struct epoll_event event; + event.events = EPOLLIN; + event.data.ptr = client_sock; + epoll_ctl(_epoll_fd, EPOLL_CTL_ADD, client_sock->csock(), &event); + } + } + + std::osyncstream(log::info) << "Stopping listening for " << (is_tls ? "HTTPS" : "HTTP") << " connections on port " << http_ptr->port() << std::endl; + + delete socket; +} + +void event_loop::start() +{ + log::info << "Starting event_loop Thread Manager" << std::endl; + + _epoll_fd = epoll_create(1); + _run = true; + + std::vector listener_threads; + std::vector worker_threads; + + for (int i = 0; i < _config.worker_threads(); i++) { + auto thread = std::thread(&event_loop::worker_thread_loop, this, i); + worker_threads.push_back(std::move(thread)); + } + + if (_config.http_cfg().has_value()) { + auto thread = std::thread(&event_loop::listener_thread_loop, this, std::ref(_config.http_cfg().value())); + listener_threads.push_back(std::move(thread)); + } + + if (_config.https_cfg().has_value()) { + auto thread = std::thread(&event_loop::listener_thread_loop, this, std::ref(_config.https_cfg().value())); + listener_threads.push_back(std::move(thread)); + } + + { + auto thread = std::thread(&event_loop::eventer_thread_loop, this); + listener_threads.push_back(std::move(thread)); + } + + for (std::thread& t : worker_threads) { + t.join(); + } + + for (std::thread& t : listener_threads) { + t.join(); } } + +void event_loop::stop() +{ + _run = false; + std::lock_guard lg(_event_mtx); + _event_cv.notify_all(); +} +} diff --git a/lib/thread_mgr/event_loop.hpp b/lib/thread_mgr/event_loop.hpp index 917af6b..d5ed923 100644 --- a/lib/thread_mgr/event_loop.hpp +++ b/lib/thread_mgr/event_loop.hpp @@ -1,6 +1,7 @@ #include "./thread_mgr.hpp" #include "../socket/socket.hpp" #include "../backends/file_backend.hpp" +#include #include #include #include @@ -9,11 +10,14 @@ namespace anthracite::thread_mgr { class event_loop : public virtual thread_mgr { class event { socket::anthracite_socket* _socket; + std::chrono::time_point _ts; public: - event(socket::anthracite_socket* socket); + event(socket::anthracite_socket* socket, std::chrono::time_point timestamp); socket::anthracite_socket* socket(); + std::chrono::time_point& timestamp(); }; + int _epoll_fd; std::mutex _event_mtx; std::condition_variable _event_cv; std::queue _events; @@ -21,6 +25,7 @@ namespace anthracite::thread_mgr { void worker_thread_loop(int threadno); void listener_thread_loop(config::http_config& http_config); + void eventer_thread_loop(); bool event_handler(event& ev); public: diff --git a/src/api_main.cpp b/src/api_main.cpp deleted file mode 100644 index 54d7b3b..0000000 --- a/src/api_main.cpp +++ /dev/null @@ -1,112 +0,0 @@ -#include "../lib/anthracite.hpp" -#include "../lib/backends/backend.hpp" -#include "../lib/http/constants.hpp" -#include -#include -#include -#include -#include -#include -#include - -using namespace anthracite; - -using CallbackType = std::unique_ptr (*)(http::request&); -class api_backend : public backends::backend { - - class RouteNode { - public: - std::optional callback; - - RouteNode() - : callback(std::nullopt) - { - } - std::unordered_map routes; - }; - - RouteNode root; - - std::unique_ptr default_route(http::request& req) - { - std::unique_ptr resp = std::make_unique(); - - resp->add_body("Not Found"); - resp->add_header(http::header("Content-Type", "application/json")); - resp->add_status(http::status_codes::NOT_FOUND); - - return resp; - } - - std::unique_ptr find_handler(http::request& req) - { - std::string filename = req.path().substr(1); - std::vector result; - std::stringstream ss(filename); - std::string item; - - RouteNode* cur = &root; - while (getline(ss, item, '/')) { - if (cur->routes.find(item) == cur->routes.end()) { - if (cur->routes.find("*") == cur->routes.end()) { - break; - } else { - cur = &cur->routes["*"]; - } - } else { - cur = &cur->routes[item]; - } - } - - if (cur->callback.has_value()) { - return cur->callback.value()(req); - } else { - return default_route(req); - } - } - - std::unique_ptr handle_request(http::request& req) override - { - return find_handler(req); - } - -public: - api_backend() - { - root.routes = std::unordered_map(); - } - - void register_endpoint(std::string pathspec, CallbackType callback) - { - std::vector result; - std::stringstream ss(pathspec); - std::string item; - - RouteNode* cur = &root; - while (getline(ss, item, '/')) { - cur->routes[item] = RouteNode {}; - cur = &cur->routes[item]; - } - - cur->callback = callback; - } -}; - -std::unique_ptr handle_request(http::request& req) -{ - std::unique_ptr resp = std::make_unique(); - - resp->add_body(R"({"user": "endpoint"}")"); - resp->add_header(http::header("Content-Type", "application/json")); - resp->add_status(http::status_codes::OK); - - return resp; -} - -int main(int argc, char** argv) -{ - auto args = std::span(argv, size_t(argc)); - api_backend ab; - ab.register_endpoint("users/*", handle_request); - //anthracite_main(argc, argv, ab); -} diff --git a/src/file_main.cpp b/src/file_main.cpp index 7a723a9..027bafd 100644 --- a/src/file_main.cpp +++ b/src/file_main.cpp @@ -1,15 +1,35 @@ -#include "../lib/anthracite.hpp" #include "../lib/backends/file_backend.hpp" #include "../lib/config/config.hpp" +#include "../lib/log/log.hpp" +#include "../lib/thread_mgr/event_loop.hpp" +#include "signal.h" +#include "string.h" +#include -using namespace anthracite; +std::shared_ptr server = nullptr; + +extern "C" void signalHandler(int signum) +{ + anthracite::log::warn << "Caught signal SIG" << sigabbrev_np(signum) << ", exiting Anthracite" << std::endl; + if (server != nullptr) { + server->stop(); + } +} int main(int argc, char** argv) { - backends::file_backend fb("./www"); - config::config cfg(5); - cfg.add_http_config(config::http_config(8080)); - cfg.add_https_config(config::https_config(8081, "", "")); + anthracite::log::logger.initialize(anthracite::log::LOG_LEVEL_INFO); + anthracite::log::info << "Starting Anthracite, a higher performance web server" << std::endl; + signal(SIGINT, signalHandler); - anthracite_main(fb, cfg); + anthracite::backends::file_backend fb("./www"); + anthracite::config::config cfg(5, 10000); + cfg.add_http_config(anthracite::config::http_config(8080)); + // cfg.add_https_config(config::https_config(8081, "", "")); + + server = std::make_shared(fb, cfg); + + server->start(); + + anthracite::log::info << "Stopping Anthracite, a higher performance web server" << std::endl; } diff --git a/tests/speed_tests.cpp b/tests/speed_tests.cpp index 9f6835f..e9da069 100644 --- a/tests/speed_tests.cpp +++ b/tests/speed_tests.cpp @@ -1,21 +1,21 @@ -#include -#include -#include #include "../lib/http/request.hpp" +#include +#include +#include #ifdef SPEEDTEST_COMPARE_BOOST #include #endif - -using std::chrono::high_resolution_clock; -using std::chrono::duration_cast; using std::chrono::duration; +using std::chrono::duration_cast; +using std::chrono::high_resolution_clock; using std::chrono::milliseconds; constexpr uint32_t num_requests = 10000000; -TEST(speed_tests, request_parse) { +TEST(speed_tests, request_parse) +{ std::ifstream t("./test_files/test_request.http"); std::stringstream buffer; buffer << t.rdbuf(); @@ -23,24 +23,25 @@ TEST(speed_tests, request_parse) { auto start = high_resolution_clock::now(); - for(int i = 0; i < num_requests; ++i) { - volatile anthracite::http::request req (raw_req, "0.0.0.0"); + for (int i = 0; i < num_requests; ++i) { + volatile anthracite::http::request req(raw_req, "0.0.0.0"); } auto end = high_resolution_clock::now(); - auto ms_int = duration_cast(end-start); + auto ms_int = duration_cast(end - start); double m_rps = ((1000.0 / ms_int.count()) * num_requests) / 1000000; - std::cout << "Parsed " << (num_requests/1000000) << " Million requests in " << ms_int << " ms"; + std::cout << "Parsed " << (num_requests / 1000000) << " Million requests in " << ms_int << " ms"; std::cout << " at " << m_rps << " Million RPS " << std::endl; ASSERT_LT(ms_int.count(), 2000); } #ifdef SPEEDTEST_COMPARE_BOOST -TEST(speed_tests, boost) { +TEST(speed_tests, boost) +{ std::ifstream t("./test_files/test_request.http"); std::stringstream buffer; buffer << t.rdbuf(); @@ -48,7 +49,7 @@ TEST(speed_tests, boost) { auto start = high_resolution_clock::now(); - for(int i = 0; i < num_requests; ++i) { + for (int i = 0; i < num_requests; ++i) { boost::system::error_code ec; boost::beast::http::request_parser p; p.put(boost::asio::buffer(raw_req), ec); @@ -56,11 +57,11 @@ TEST(speed_tests, boost) { } auto end = high_resolution_clock::now(); - auto ms_int = duration_cast(end-start); + auto ms_int = duration_cast(end - start); double m_rps = ((1000.0 / ms_int.count()) * num_requests) / 1000000; - std::cout << "Parsed " << (num_requests/1000000) << " Million requests in " << ms_int << " ms"; + std::cout << "Parsed " << (num_requests / 1000000) << " Million requests in " << ms_int << " ms"; std::cout << " at " << m_rps << " Million RPS " << std::endl; } #endif diff --git a/tests/unit_tests.cpp b/tests/unit_tests.cpp index c7e3117..e152679 100644 --- a/tests/unit_tests.cpp +++ b/tests/unit_tests.cpp @@ -1,9 +1,10 @@ -#include -#include #include "../lib/http/request.hpp" #include +#include +#include -TEST(unit_tests, single_request_parse) { +TEST(unit_tests, single_request_parse) +{ std::ifstream t("./test_files/test_request.http"); std::stringstream buffer; buffer << t.rdbuf(); @@ -11,7 +12,7 @@ TEST(unit_tests, single_request_parse) { std::string raw_req = buffer.str(); std::string expected = buffer.str(); - anthracite::http::request req (raw_req, "0.0.0.0"); + anthracite::http::request req(raw_req, "0.0.0.0"); ASSERT_EQ(expected, req.to_string()); } From 6c5feb867515900ac83bd46c4bad9c3e4692530e Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Sat, 22 Feb 2025 00:52:30 -0500 Subject: [PATCH 09/15] epoll per thd --- lib/socket/socket.cpp | 16 ++--- lib/thread_mgr/event_loop.cpp | 113 +++++++++++++++------------------- lib/thread_mgr/event_loop.hpp | 4 +- src/file_main.cpp | 2 +- 4 files changed, 61 insertions(+), 74 deletions(-) diff --git a/lib/socket/socket.cpp b/lib/socket/socket.cpp index f8ec4ae..7c748d8 100644 --- a/lib/socket/socket.cpp +++ b/lib/socket/socket.cpp @@ -32,19 +32,19 @@ anthracite_socket::anthracite_socket(int port, int max_queue) bool anthracite_socket::wait_for_conn() { client_ip = ""; - struct timeval tv = { .tv_sec = 1, .tv_usec = 0 }; - fd_set read_fd; - FD_ZERO(&read_fd); - FD_SET(server_socket, &read_fd); - if (select(server_socket + 1, &read_fd, NULL, NULL, &wait_timeout)) { + //struct timeval tv = { .tv_sec = 1, .tv_usec = 0 }; + //fd_set read_fd; + //FD_ZERO(&read_fd); + //FD_SET(server_socket, &read_fd); + //if (select(server_socket + 1, &read_fd, NULL, NULL, &wait_timeout)) { client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); std::array ip_str { 0 }; inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN); client_ip = std::string(ip_str.data()); return true; - } else { - return false; - } + //} else { + // return false; + //} } const std::string& anthracite_socket::get_client_ip() diff --git a/lib/thread_mgr/event_loop.cpp b/lib/thread_mgr/event_loop.cpp index 1cf4d2a..f271138 100644 --- a/lib/thread_mgr/event_loop.cpp +++ b/lib/thread_mgr/event_loop.cpp @@ -5,6 +5,7 @@ #include "sys/epoll.h" #include #include +#include #include using std::chrono::duration; @@ -35,23 +36,23 @@ event_loop::event_loop(backends::backend& backend, config::config& config) { } -bool event_loop::event_handler(event& event) +bool event_loop::event_handler(socket::anthracite_socket* sock) { - std::string raw_request = event.socket()->recv_message(http::HEADER_BYTES); + std::string raw_request = sock->recv_message(http::HEADER_BYTES); if (raw_request == "") { return false; } - http::request req(raw_request, event.socket()->get_client_ip()); + http::request req(raw_request, sock->get_client_ip()); std::unique_ptr resp = req.is_supported_version() ? _backend.handle_request(req) : _error_backend.handle_error(http::status_codes::HTTP_VERSION_NOT_SUPPORTED); std::string header = resp->header_to_string(); - event.socket()->send_message(header); - event.socket()->send_message(resp->content()); + sock->send_message(header); + sock->send_message(resp->content()); auto end = high_resolution_clock::now(); - auto ms_int = duration_cast(end - event.timestamp()); - log::logger.log_request_and_response(req, resp, ms_int.count()); + //auto ms_int = duration_cast(end - event.timestamp()); + //log::logger.log_request_and_response(req, resp, 9);//ms_int.count()); resp.reset(); if (req.close_connection()) { @@ -61,46 +62,30 @@ bool event_loop::event_handler(event& event) return true; } +#define QATATIME (50) + void event_loop::worker_thread_loop(int threadno) { - unsigned char buf[sizeof(class event)]; + struct epoll_event* events = new struct epoll_event[_config.max_clients()]; + int epoll_fd = _epoll_fds[threadno]; + + std::osyncstream(log::info) << "Starting worker thread " << threadno << " on pid " << syscall(SYS_gettid) << std::endl; - std::osyncstream(log::info) << "Starting worker thread " << threadno << std::endl; while (_run) { // Get event from queue - std::unique_lock lock(_event_mtx); + int ready_fds = epoll_wait(epoll_fd, events, _config.max_clients(), 1000); - event* ev = nullptr; + if (ready_fds > 0) { + std::lock_guard lg(_event_mtx); + for (int i = 0; i < ready_fds; i++) { + socket::anthracite_socket* sockptr = reinterpret_cast(events[i].data.ptr); - if (_events.size() > 0) { - if (!lock.owns_lock()) { - lock.lock(); + if (!event_handler(sockptr)) { + epoll_ctl(epoll_fd, EPOLL_CTL_DEL, sockptr->csock(), &events[i]); + sockptr->close_conn(); + delete sockptr; + } } - assert(lock.owns_lock()); - ev = new (buf) event(_events.front()); - _events.pop(); - lock.unlock(); - } else { - _event_cv.wait(lock, [this] { return this->_events.size() > 0 || !_run; }); - - if (!_run) { - break; - } - - assert(lock.owns_lock()); - ev = new (buf) event(_events.front()); - _events.pop(); - lock.unlock(); - } - - if (event_handler(*ev)) { - struct epoll_event event; - event.events = EPOLLIN; - event.data.ptr = ev->socket(); - epoll_ctl(_epoll_fd, EPOLL_CTL_ADD, ev->socket()->csock(), &event); - } else { - ev->socket()->close_conn(); - delete ev->socket(); } } @@ -109,24 +94,24 @@ void event_loop::worker_thread_loop(int threadno) void event_loop::eventer_thread_loop() { - struct epoll_event* events = new struct epoll_event[_config.max_clients()]; - std::osyncstream(log::info) << "epoll() thread started" << std::endl; - while (_run) { - int ready_fds = epoll_wait(_epoll_fd, events, _config.max_clients(), 1000); + //struct epoll_event* events = new struct epoll_event[_config.max_clients()]; + //std::osyncstream(log::info) << "epoll() thread started on pid " << getpid() << std::endl; + //while (_run) { + // int ready_fds = epoll_wait(_epoll_fd, events, _config.max_clients(), 1000); - if (ready_fds > 0) { - std::lock_guard lg(_event_mtx); - for (int i = 0; i < ready_fds; i++) { - socket::anthracite_socket* sockptr = reinterpret_cast(events[i].data.ptr); - struct epoll_event ev; - epoll_ctl(_epoll_fd, EPOLL_CTL_DEL, sockptr->csock(), &events[i]); - _events.push(event(sockptr, std::chrono::high_resolution_clock::now())); - } - _event_cv.notify_one(); - } - } - delete[] events; - std::osyncstream(log::info) << "epoll() thread exited" << std::endl; + // if (ready_fds > 0) { + // std::lock_guard lg(_event_mtx); + // for (int i = 0; i < ready_fds; i++) { + // socket::anthracite_socket* sockptr = reinterpret_cast(events[i].data.ptr); + // struct epoll_event ev; + // epoll_ctl(_epoll_fd, EPOLL_CTL_DEL, sockptr->csock(), &events[i]); + // _events.push(event(sockptr, std::chrono::high_resolution_clock::now())); + // } + // _event_cv.notify_all(); + // } + //} + //delete[] events; + //std::osyncstream(log::info) << "epoll() thread exited" << std::endl; } void event_loop::listener_thread_loop(config::http_config& http_config) @@ -144,8 +129,9 @@ void event_loop::listener_thread_loop(config::http_config& http_config) socket = new socket::anthracite_socket(http_ptr->port()); } - std::osyncstream(log::info) << "Listening for " << (is_tls ? "HTTPS" : "HTTP") << " connections on port " << http_ptr->port() << std::endl; + std::osyncstream(log::info) << "Listening for " << (is_tls ? "HTTPS" : "HTTP") << " connections on port " << http_ptr->port() << " on pid " << getpid() << std::endl; + int assign_thread = 0; while (_run) { if (socket->wait_for_conn()) { socket::anthracite_socket* client_sock; @@ -160,7 +146,8 @@ void event_loop::listener_thread_loop(config::http_config& http_config) struct epoll_event event; event.events = EPOLLIN; event.data.ptr = client_sock; - epoll_ctl(_epoll_fd, EPOLL_CTL_ADD, client_sock->csock(), &event); + epoll_ctl(_epoll_fds[assign_thread], EPOLL_CTL_ADD, client_sock->csock(), &event); + assign_thread = (assign_thread + 1) % _epoll_fds.size(); } } @@ -173,13 +160,13 @@ void event_loop::start() { log::info << "Starting event_loop Thread Manager" << std::endl; - _epoll_fd = epoll_create(1); _run = true; std::vector listener_threads; std::vector worker_threads; for (int i = 0; i < _config.worker_threads(); i++) { + _epoll_fds.push_back(epoll_create(1)); auto thread = std::thread(&event_loop::worker_thread_loop, this, i); worker_threads.push_back(std::move(thread)); } @@ -194,10 +181,10 @@ void event_loop::start() listener_threads.push_back(std::move(thread)); } - { - auto thread = std::thread(&event_loop::eventer_thread_loop, this); - listener_threads.push_back(std::move(thread)); - } + //{ + // auto thread = std::thread(&event_loop::eventer_thread_loop, this); + // listener_threads.push_back(std::move(thread)); + //} for (std::thread& t : worker_threads) { t.join(); diff --git a/lib/thread_mgr/event_loop.hpp b/lib/thread_mgr/event_loop.hpp index d5ed923..7f09c97 100644 --- a/lib/thread_mgr/event_loop.hpp +++ b/lib/thread_mgr/event_loop.hpp @@ -17,7 +17,7 @@ namespace anthracite::thread_mgr { std::chrono::time_point& timestamp(); }; - int _epoll_fd; + std::vector _epoll_fds; std::mutex _event_mtx; std::condition_variable _event_cv; std::queue _events; @@ -26,7 +26,7 @@ namespace anthracite::thread_mgr { void worker_thread_loop(int threadno); void listener_thread_loop(config::http_config& http_config); void eventer_thread_loop(); - bool event_handler(event& ev); + bool event_handler(socket::anthracite_socket*); public: event_loop(backends::backend& backend, config::config& config); diff --git a/src/file_main.cpp b/src/file_main.cpp index 027bafd..0ce2289 100644 --- a/src/file_main.cpp +++ b/src/file_main.cpp @@ -23,7 +23,7 @@ int main(int argc, char** argv) signal(SIGINT, signalHandler); anthracite::backends::file_backend fb("./www"); - anthracite::config::config cfg(5, 10000); + anthracite::config::config cfg(3, 1000); cfg.add_http_config(anthracite::config::http_config(8080)); // cfg.add_https_config(config::https_config(8081, "", "")); From c07f3ebf8109ec40d5ec5019ca4b54dd888aa864 Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Sun, 23 Feb 2025 17:06:32 -0500 Subject: [PATCH 10/15] faster nonblocking io --- Dockerfile | 3 +- docker-compose.yml | 2 +- lib/log/log.cpp | 5 ++- lib/socket/socket.cpp | 44 +++++++++++++++------- lib/socket/socket.hpp | 4 +- lib/thread_mgr/event_loop.cpp | 71 +++++++++++------------------------ lib/thread_mgr/event_loop.hpp | 5 +-- src/file_main.cpp | 4 +- 8 files changed, 66 insertions(+), 72 deletions(-) diff --git a/Dockerfile b/Dockerfile index c1f60f9..b7247e4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,9 @@ FROM alpine as build-env -RUN apk add --no-cache build-base python3 cmake +RUN apk add --no-cache build-base python3 cmake openssl-dev COPY ./src ./src COPY ./lib ./lib +COPY ./tests ./tests COPY ./default_www ./default_www COPY ./build_supp ./build_supp COPY ./CMakeLists.txt . diff --git a/docker-compose.yml b/docker-compose.yml index e89cc51..ff74b34 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ services: anthracite-web: build: . ports: - - "8080:80" + - "8080:8080" volumes: - type: bind source: ./default_www/docker_compose/ diff --git a/lib/log/log.cpp b/lib/log/log.cpp index 63399d0..8095903 100644 --- a/lib/log/log.cpp +++ b/lib/log/log.cpp @@ -1,4 +1,5 @@ #include "./log.hpp" +#include namespace anthracite::log { enum LOG_LEVEL Logger::_level = LOG_LEVEL_NONE; @@ -25,7 +26,9 @@ LogBuf::LogBuf(std::ostream& output_stream, const std::string& tag, enum LOG_LEV int LogBuf::sync() { if (this->_level <= logger._level) { - std::cout << "[" << this->_tag << "] " << this->str(); + char thread_name[100]; + pthread_getname_np(pthread_self(), thread_name, 100); + std::osyncstream(std::cout) << "[" << this->_tag << "] [" << syscall(SYS_gettid) << ":" << thread_name << "] "<< this->str(); std::cout.flush(); } this->str(""); diff --git a/lib/socket/socket.cpp b/lib/socket/socket.cpp index 7c748d8..0c2046d 100644 --- a/lib/socket/socket.cpp +++ b/lib/socket/socket.cpp @@ -4,18 +4,26 @@ #include #include #include +#include #include #include #include #include +#include +#include +#include "assert.h" +#include +#include + namespace anthracite::socket { const struct timeval anthracite_socket::timeout_tv = { .tv_sec = 5, .tv_usec = 0 }; -anthracite_socket::anthracite_socket(int port, int max_queue) +anthracite_socket::anthracite_socket(int port, int max_queue, bool nonblocking) : server_socket(::socket(AF_INET, SOCK_STREAM, 0)) , client_ip("") + , _nonblocking(nonblocking) { struct sockaddr_in address {}; address.sin_family = AF_INET; @@ -26,25 +34,28 @@ anthracite_socket::anthracite_socket(int port, int max_queue) setsockopt(server_socket, SOL_SOCKET, SO_REUSEADDR, &reuse_opt, sizeof(reuse_opt)); bind(server_socket, reinterpret_cast(&address), sizeof(address)); + if (_nonblocking) { + fcntl(server_socket, F_SETFL, O_NONBLOCK); + } + listen(server_socket, max_queue); } bool anthracite_socket::wait_for_conn() { client_ip = ""; - //struct timeval tv = { .tv_sec = 1, .tv_usec = 0 }; - //fd_set read_fd; - //FD_ZERO(&read_fd); - //FD_SET(server_socket, &read_fd); - //if (select(server_socket + 1, &read_fd, NULL, NULL, &wait_timeout)) { - client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); + client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); + if (client_socket > 0) { + if (_nonblocking) { + fcntl(client_socket, F_SETFL, O_NONBLOCK); + } std::array ip_str { 0 }; inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN); client_ip = std::string(ip_str.data()); return true; - //} else { - // return false; - //} + } else { + return false; + } } const std::string& anthracite_socket::get_client_ip() @@ -52,8 +63,7 @@ const std::string& anthracite_socket::get_client_ip() return client_ip; } -void anthracite_socket::close_conn() -{ +void anthracite_socket::close_conn() { close(client_socket); client_socket = -1; } @@ -66,13 +76,21 @@ void anthracite_socket::send_message(std::string& msg) send(client_socket, &msg[0], msg.length(), 0); } +bool anthracite_socket::has_client() { + return client_socket > 0; +} + std::string anthracite_socket::recv_message(int buffer_size) { if (client_socket == -1) { return ""; } - setsockopt(client_socket, SOL_SOCKET, SO_RCVTIMEO, &timeout_tv, sizeof timeout_tv); + //setsockopt(client_socket, SOL_SOCKET, SO_RCVTIMEO, &timeout_tv, sizeof timeout_tv); + + int nodelay_opt = 1; + assert(setsockopt(client_socket, SOL_TCP, TCP_NODELAY, &nodelay_opt, sizeof(nodelay_opt)) == 0); + std::vector response(buffer_size + 1); ssize_t result = recv(client_socket, response.data(), buffer_size + 1, 0); diff --git a/lib/socket/socket.hpp b/lib/socket/socket.hpp index 5ac3562..ea8622d 100644 --- a/lib/socket/socket.hpp +++ b/lib/socket/socket.hpp @@ -14,6 +14,7 @@ namespace anthracite::socket { class anthracite_socket { protected: + bool _nonblocking; struct timeval wait_timeout = { .tv_sec = 1, .tv_usec = 0}; int server_socket; int client_socket {}; @@ -24,10 +25,11 @@ protected: static const int MAX_QUEUE_LENGTH = 100; public: - anthracite_socket(int port, int max_queue = MAX_QUEUE_LENGTH); + anthracite_socket(int port, int max_queue = MAX_QUEUE_LENGTH, bool nonblocking = false); virtual const std::string& get_client_ip() final; + virtual bool has_client(); virtual bool wait_for_conn(); virtual void close_conn(); virtual void send_message(std::string& msg); diff --git a/lib/thread_mgr/event_loop.cpp b/lib/thread_mgr/event_loop.cpp index f271138..92c5bfb 100644 --- a/lib/thread_mgr/event_loop.cpp +++ b/lib/thread_mgr/event_loop.cpp @@ -6,7 +6,9 @@ #include #include #include +#include #include +#include "signal.h" using std::chrono::duration; using std::chrono::duration_cast; @@ -50,11 +52,6 @@ bool event_loop::event_handler(socket::anthracite_socket* sock) sock->send_message(header); sock->send_message(resp->content()); - auto end = high_resolution_clock::now(); - //auto ms_int = duration_cast(end - event.timestamp()); - //log::logger.log_request_and_response(req, resp, 9);//ms_int.count()); - - resp.reset(); if (req.close_connection()) { return false; } @@ -62,26 +59,30 @@ bool event_loop::event_handler(socket::anthracite_socket* sock) return true; } -#define QATATIME (50) - void event_loop::worker_thread_loop(int threadno) { - struct epoll_event* events = new struct epoll_event[_config.max_clients()]; - int epoll_fd = _epoll_fds[threadno]; + std::stringstream ss; + ss << "worker " << threadno; + pthread_setname_np(pthread_self(), ss.str().c_str()); - std::osyncstream(log::info) << "Starting worker thread " << threadno << " on pid " << syscall(SYS_gettid) << std::endl; + struct epoll_event* events = new struct epoll_event[_config.max_clients()]; + int timeout_ms = 1000; + + if (_nonblocking) { + timeout_ms = 0; + } + + log::info << "Starting worker thread " << threadno << std::endl; while (_run) { - // Get event from queue - int ready_fds = epoll_wait(epoll_fd, events, _config.max_clients(), 1000); + int ready_fds = epoll_wait(_epoll_fd, events, _config.max_clients(), timeout_ms); if (ready_fds > 0) { - std::lock_guard lg(_event_mtx); for (int i = 0; i < ready_fds; i++) { socket::anthracite_socket* sockptr = reinterpret_cast(events[i].data.ptr); if (!event_handler(sockptr)) { - epoll_ctl(epoll_fd, EPOLL_CTL_DEL, sockptr->csock(), &events[i]); + epoll_ctl(_epoll_fd, EPOLL_CTL_DEL, sockptr->csock(), &events[i]); sockptr->close_conn(); delete sockptr; } @@ -89,29 +90,7 @@ void event_loop::worker_thread_loop(int threadno) } } - std::osyncstream(log::info) << "Stopping worker thread " << threadno << std::endl; -} - -void event_loop::eventer_thread_loop() -{ - //struct epoll_event* events = new struct epoll_event[_config.max_clients()]; - //std::osyncstream(log::info) << "epoll() thread started on pid " << getpid() << std::endl; - //while (_run) { - // int ready_fds = epoll_wait(_epoll_fd, events, _config.max_clients(), 1000); - - // if (ready_fds > 0) { - // std::lock_guard lg(_event_mtx); - // for (int i = 0; i < ready_fds; i++) { - // socket::anthracite_socket* sockptr = reinterpret_cast(events[i].data.ptr); - // struct epoll_event ev; - // epoll_ctl(_epoll_fd, EPOLL_CTL_DEL, sockptr->csock(), &events[i]); - // _events.push(event(sockptr, std::chrono::high_resolution_clock::now())); - // } - // _event_cv.notify_all(); - // } - //} - //delete[] events; - //std::osyncstream(log::info) << "epoll() thread exited" << std::endl; + log::info << "Stopping worker thread " << threadno << std::endl; } void event_loop::listener_thread_loop(config::http_config& http_config) @@ -129,9 +108,8 @@ void event_loop::listener_thread_loop(config::http_config& http_config) socket = new socket::anthracite_socket(http_ptr->port()); } - std::osyncstream(log::info) << "Listening for " << (is_tls ? "HTTPS" : "HTTP") << " connections on port " << http_ptr->port() << " on pid " << getpid() << std::endl; + std::osyncstream(log::info) << "Listening for " << (is_tls ? "HTTPS" : "HTTP") << " connections on port " << http_ptr->port() << std::endl; - int assign_thread = 0; while (_run) { if (socket->wait_for_conn()) { socket::anthracite_socket* client_sock; @@ -144,10 +122,9 @@ void event_loop::listener_thread_loop(config::http_config& http_config) } struct epoll_event event; - event.events = EPOLLIN; + event.events = EPOLLIN | EPOLLEXCLUSIVE;// | EPOLLET; event.data.ptr = client_sock; - epoll_ctl(_epoll_fds[assign_thread], EPOLL_CTL_ADD, client_sock->csock(), &event); - assign_thread = (assign_thread + 1) % _epoll_fds.size(); + epoll_ctl(_epoll_fd, EPOLL_CTL_ADD, client_sock->csock(), &event); } } @@ -158,15 +135,16 @@ void event_loop::listener_thread_loop(config::http_config& http_config) void event_loop::start() { + signal(SIGPIPE, SIG_IGN); log::info << "Starting event_loop Thread Manager" << std::endl; _run = true; + _epoll_fd = epoll_create(1); std::vector listener_threads; std::vector worker_threads; for (int i = 0; i < _config.worker_threads(); i++) { - _epoll_fds.push_back(epoll_create(1)); auto thread = std::thread(&event_loop::worker_thread_loop, this, i); worker_threads.push_back(std::move(thread)); } @@ -181,11 +159,6 @@ void event_loop::start() listener_threads.push_back(std::move(thread)); } - //{ - // auto thread = std::thread(&event_loop::eventer_thread_loop, this); - // listener_threads.push_back(std::move(thread)); - //} - for (std::thread& t : worker_threads) { t.join(); } @@ -198,7 +171,5 @@ void event_loop::start() void event_loop::stop() { _run = false; - std::lock_guard lg(_event_mtx); - _event_cv.notify_all(); } } diff --git a/lib/thread_mgr/event_loop.hpp b/lib/thread_mgr/event_loop.hpp index 7f09c97..3fa8ea5 100644 --- a/lib/thread_mgr/event_loop.hpp +++ b/lib/thread_mgr/event_loop.hpp @@ -17,11 +17,10 @@ namespace anthracite::thread_mgr { std::chrono::time_point& timestamp(); }; - std::vector _epoll_fds; + int _epoll_fd; std::mutex _event_mtx; - std::condition_variable _event_cv; - std::queue _events; backends::file_backend _error_backend; + bool _nonblocking; void worker_thread_loop(int threadno); void listener_thread_loop(config::http_config& http_config); diff --git a/src/file_main.cpp b/src/file_main.cpp index 0ce2289..ad3dcdd 100644 --- a/src/file_main.cpp +++ b/src/file_main.cpp @@ -10,7 +10,7 @@ std::shared_ptr server = nullptr; extern "C" void signalHandler(int signum) { - anthracite::log::warn << "Caught signal SIG" << sigabbrev_np(signum) << ", exiting Anthracite" << std::endl; + //anthracite::log::warn << "Caught signal SIG" << sigabbrev_np(signum) << ", exiting Anthracite" << std::endl; if (server != nullptr) { server->stop(); } @@ -23,7 +23,7 @@ int main(int argc, char** argv) signal(SIGINT, signalHandler); anthracite::backends::file_backend fb("./www"); - anthracite::config::config cfg(3, 1000); + anthracite::config::config cfg(1, 10); cfg.add_http_config(anthracite::config::http_config(8080)); // cfg.add_https_config(config::https_config(8081, "", "")); From 9b5719f9bebdb894c22b9e8da8297f046c77de3b Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Mon, 24 Feb 2025 19:29:43 -0500 Subject: [PATCH 11/15] Release v0.3.0 --- .github/workflows/docker-publish.yml | 56 +++++ CHANGELOG.md | 4 +- CMakeLists.txt | 3 +- Dockerfile | 1 + build_supp/default_config.cfg | 6 + docker-compose.yml | 3 + lib/config/config.hpp | 59 ------ lib/log/log.cpp | 5 +- lib/log/log.hpp | 1 + lib/socket/openssl_socket.cpp | 103 ++++++---- lib/socket/openssl_socket.hpp | 23 ++- lib/socket/socket.cpp | 147 ++++++++----- lib/socket/socket.hpp | 50 +++-- lib/thread_mgr/event_loop.cpp | 135 +++++------- lib/thread_mgr/event_loop.hpp | 25 +-- lib/thread_mgr/thread_mgr.hpp | 4 +- src/file_main.cpp | 295 +++++++++++++++++++++++++-- 17 files changed, 612 insertions(+), 308 deletions(-) create mode 100644 .github/workflows/docker-publish.yml create mode 100644 build_supp/default_config.cfg delete mode 100644 lib/config/config.hpp diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 0000000..565eda4 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,56 @@ +name: Build and Publish Docker Image + +on: + release: + types: [created] +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf + + - name: Get Release Tag + run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV + + - name: Write Version + run: | + echo Building with version number $RELEASE_VERSION + echo $RELEASE_VERSION > src/build/version.txt + + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/CHANGELOG.md b/CHANGELOG.md index c3e9edb..d34c86f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,9 @@ # 0.3.0 - SSL support via OpenSSL - Added "Thread Manager" class to allow for multiple (or custom) threading models (process per thread, event loop) -- Defaul threading model is now event loop +- Default (and only included) threading model is now event loop - Rewrote request parser for readability and speed +- Rewrote socket system for readability and speed - Added improved logging with different log levels - Separated anthracite into libanthracite and anthracite-bin to allow for other projects to implement anthracite (example in ./src/api_main.cpp) - Cleaned up code and seperated most code into headers & source @@ -10,7 +11,6 @@ - Moved CI/CD over to Forgejo - General system stability improvements were made to enhance the user's experience - ## HTTP Request Parser Rewrite The following benchmark (source in ./tests/speed_tests.cpp) shows the speed diff --git a/CMakeLists.txt b/CMakeLists.txt index a1fbdf8..9af19e2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -18,8 +18,9 @@ add_custom_target(build-version add_custom_target(build-supplemental COMMAND cd ../build_supp && python3 ./error_gen.py COMMAND mkdir -p www && cp -r ../default_www/regular/* ./www/ + COMMAND cp ../build_supp/default_config.cfg ./anthracite.cfg DEPENDS build_supp/version.txt ../default_www/regular/* build_supp/error_gen.py build-version - COMMENT "Generated supplemental build files (default www dir + error pages)" + COMMENT "Generated supplemental build files (default www dir + default config + error pages)" ) add_custom_target(run diff --git a/Dockerfile b/Dockerfile index b7247e4..3c5c6c2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,5 +18,6 @@ FROM alpine RUN apk add --no-cache libgcc libstdc++ COPY --from=build-env /build/anthracite-bin /anthracite-bin COPY --from=build-env /build/error_pages /error_pages +COPY --from=build-env /build_supp/default_config.cfg /anthracite.cfg COPY /default_www/docker /www CMD ["/anthracite-bin"] diff --git a/build_supp/default_config.cfg b/build_supp/default_config.cfg new file mode 100644 index 0000000..380e4fd --- /dev/null +++ b/build_supp/default_config.cfg @@ -0,0 +1,6 @@ +log_level INFO + +http 8080 1000 blocking + +event_loop 6 10000 +www_dir ./www diff --git a/docker-compose.yml b/docker-compose.yml index ff74b34..3d71054 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,3 +7,6 @@ services: - type: bind source: ./default_www/docker_compose/ target: /www + - type: bind + source: ./build_supp/default_config.cfg + target: /anthracite.cfg diff --git a/lib/config/config.hpp b/lib/config/config.hpp deleted file mode 100644 index f353a21..0000000 --- a/lib/config/config.hpp +++ /dev/null @@ -1,59 +0,0 @@ -#pragma once - -#include -#include -#include - -namespace anthracite::config { - class http_config { - uint16_t _port; - public: - http_config(uint16_t port) : _port(port) {}; - virtual ~http_config() {}; - - uint16_t port() { return _port; } - }; - - class https_config : public http_config { - std::string _cert_path; - std::string _key_path; - public: - https_config(uint16_t port, std::string cert_path, std::string key_path) : - http_config(port), _cert_path(cert_path), _key_path(key_path) {}; - }; - - class config { - int _worker_threads; - int _max_clients; - std::optional _http_config; - std::optional _https_config; - - public: - config(int worker_threads, int max_clients) : _worker_threads(worker_threads), _max_clients(max_clients) { - } - - void add_http_config(http_config config) { - _http_config = config; - } - - void add_https_config(https_config config) { - _https_config = config; - } - - int worker_threads() { - return _worker_threads; - } - - int max_clients() { - return _max_clients; - } - - std::optional& http_cfg() { - return _http_config; - } - - std::optional& https_cfg() { - return _https_config; - } - }; -}; diff --git a/lib/log/log.cpp b/lib/log/log.cpp index 8095903..2fba0fd 100644 --- a/lib/log/log.cpp +++ b/lib/log/log.cpp @@ -1,5 +1,4 @@ #include "./log.hpp" -#include namespace anthracite::log { enum LOG_LEVEL Logger::_level = LOG_LEVEL_NONE; @@ -28,8 +27,8 @@ int LogBuf::sync() if (this->_level <= logger._level) { char thread_name[100]; pthread_getname_np(pthread_self(), thread_name, 100); - std::osyncstream(std::cout) << "[" << this->_tag << "] [" << syscall(SYS_gettid) << ":" << thread_name << "] "<< this->str(); - std::cout.flush(); + _output_stream << "[" << this->_tag << "] [" << syscall(SYS_gettid) << ":" << thread_name << "] " << this->str(); + _output_stream.flush(); } this->str(""); return 0; diff --git a/lib/log/log.hpp b/lib/log/log.hpp index 6a99749..d9fd1e4 100644 --- a/lib/log/log.hpp +++ b/lib/log/log.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include "../http/request.hpp" #include "../http/response.hpp" diff --git a/lib/socket/openssl_socket.cpp b/lib/socket/openssl_socket.cpp index 9bb4639..775f170 100644 --- a/lib/socket/openssl_socket.cpp +++ b/lib/socket/openssl_socket.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -16,85 +17,101 @@ namespace anthracite::socket { -SSL_CTX* openssl_socket::_context = nullptr; - -openssl_socket::openssl_socket(int port, int max_queue) - : anthracite_socket(port, max_queue) +openssl_listener::openssl_listener(std::string& key_path, std::string& cert_path, int port, int max_queue, bool nonblocking) + : listener(port, max_queue, nonblocking) { const SSL_METHOD* method = TLS_server_method(); - if (_context == nullptr) { - _context = SSL_CTX_new(method); - } + _context = SSL_CTX_new(method); if (!_context) { log::err << "Unable to initialize SSL" << std::endl; throw std::exception(); } - if (SSL_CTX_use_certificate_file(_context, "cert.pem", SSL_FILETYPE_PEM) <= 0) { - log::err << "Unable to open cert.pem" << std::endl; + if (SSL_CTX_use_certificate_file(_context, cert_path.c_str(), SSL_FILETYPE_PEM) <= 0) { + log::err << "Unable to open Cert file at: " << cert_path << std::endl; throw std::exception(); } - if (SSL_CTX_use_PrivateKey_file(_context, "key.pem", SSL_FILETYPE_PEM) <= 0) { - log::err << "Unable to open key.pem" << std::endl; + if (SSL_CTX_use_PrivateKey_file(_context, key_path.c_str(), SSL_FILETYPE_PEM) <= 0) { + log::err << "Unable to open Key file at: " << key_path << std::endl; throw std::exception(); } } -openssl_socket::~openssl_socket() = default; - -bool openssl_socket::wait_for_conn() +bool openssl_listener::wait_for_conn(server** client_sock_p) { - client_ip = ""; - struct timeval tv = { .tv_sec = 1, .tv_usec = 0 }; - fd_set read_fd; - FD_ZERO(&read_fd); - FD_SET(server_socket, &read_fd); - if (select(server_socket + 1, &read_fd, NULL, NULL, &wait_timeout)) { - client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); - std::array ip_str { 0 }; - inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN); - client_ip = std::string(ip_str.data()); - _ssl = SSL_new(_context); - SSL_set_fd(_ssl, client_socket); - if (SSL_accept(_ssl) <= 0) { - log::warn << "Unable to open SSL connection with client" << std::endl; - client_ip = ""; - close(client_socket); - client_socket = -1; + struct sockaddr_in client_addr {}; + socklen_t client_addr_len; + + int csock = accept(_sock_fd, reinterpret_cast(&client_addr), &client_addr_len); + + if (csock > 0) { + std::array ip_str { 0 }; + + if (inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN) == NULL) { + if (inet_ntop(AF_INET6, &client_addr.sin_addr, ip_str.data(), INET6_ADDRSTRLEN) == NULL) { + log::warn << "Unable to decode client's IP address" << std::endl; + } + } + + SSL* ssl = SSL_new(_context); + + if (ssl == NULL) { + for (int i = 0; i < 5 && close(csock) != 0; ++i) + ; return false; } + + if (SSL_set_fd(ssl, csock) == 0) { + SSL_free(ssl); + for (int i = 0; i < 5 && close(csock) != 0; ++i) + ; + return false; + } + + if (SSL_accept(ssl) <= 0) { + log::warn << "Unable to open SSL connection with client" << std::endl; + SSL_free(ssl); + for (int i = 0; i < 5 && close(csock) != 0; ++i) + ; + return false; + } + + std::string client_ip = std::string(ip_str.data()); + *client_sock_p = new openssl_server(csock, client_ip, _nonblocking, ssl); return true; } else { return false; } } -void openssl_socket::close_conn() +openssl_listener::~openssl_listener() {} + +openssl_server::openssl_server(int sock_fd, std::string client_ip, bool nonblocking, SSL* ssl) + : server(sock_fd, client_ip, nonblocking) + , _ssl(ssl) +{ +} + +openssl_server::~openssl_server() { SSL_shutdown(_ssl); SSL_free(_ssl); - close(client_socket); - client_socket = -1; } -void openssl_socket::send_message(std::string& msg) +void openssl_server::send_message(const std::string& msg) { - if (client_socket == -1) { - return; - } SSL_write(_ssl, &msg[0], msg.length()); } -std::string openssl_socket::recv_message(int buffer_size) +std::string openssl_server::recv_message(int buffer_size) { - if (client_socket == -1) { - return ""; - } + // Ignored because it's nonfatal, just slower + int nodelay_opt = 1; + (void)setsockopt(_sock_fd, SOL_TCP, TCP_NODELAY, &nodelay_opt, sizeof(nodelay_opt)); - setsockopt(client_socket, SOL_SOCKET, SO_RCVTIMEO, &timeout_tv, sizeof timeout_tv); std::vector response(buffer_size + 1); ssize_t result = SSL_read(_ssl, response.data(), buffer_size + 1); diff --git a/lib/socket/openssl_socket.hpp b/lib/socket/openssl_socket.hpp index 4d4df6f..24d135a 100644 --- a/lib/socket/openssl_socket.hpp +++ b/lib/socket/openssl_socket.hpp @@ -5,18 +5,25 @@ #include namespace anthracite::socket { -class openssl_socket : public anthracite_socket { +class openssl_server : public server{ private: - static SSL_CTX* _context; SSL* _ssl; - public: - openssl_socket(int port, int max_queue = MAX_QUEUE_LENGTH); - ~openssl_socket(); + openssl_server(int sock_fd, std::string client_ip, bool nonblocking, SSL* ssl); + ~openssl_server(); - bool wait_for_conn() override; - void close_conn() override; - void send_message(std::string& msg) override; + void send_message(const std::string& msg) override; std::string recv_message(int buffer_size) override; }; + +class openssl_listener : public listener { + private: + SSL_CTX* _context; + + public: + openssl_listener(std::string& key_path, std::string& cert_path, int port, int max_queue_length, bool nonblocking); + ~openssl_listener(); + + bool wait_for_conn(server** client_sock_) override; +}; }; diff --git a/lib/socket/socket.cpp b/lib/socket/socket.cpp index 0c2046d..acf9bf4 100644 --- a/lib/socket/socket.cpp +++ b/lib/socket/socket.cpp @@ -1,98 +1,120 @@ #include "./socket.hpp" +#include "../log/log.hpp" +#include "assert.h" #include #include +#include +#include +#include #include #include +#include #include #include #include #include #include #include -#include -#include -#include "assert.h" -#include -#include - namespace anthracite::socket { -const struct timeval anthracite_socket::timeout_tv = { .tv_sec = 5, .tv_usec = 0 }; - -anthracite_socket::anthracite_socket(int port, int max_queue, bool nonblocking) - : server_socket(::socket(AF_INET, SOCK_STREAM, 0)) - , client_ip("") - , _nonblocking(nonblocking) +socket::socket(bool nonblocking) + : _nonblocking(nonblocking) { +} + +listener::listener(int port, int max_queue, bool nonblocking) + : socket(nonblocking) + , _port(port) +{ + _sock_fd = ::socket(AF_INET, SOCK_STREAM, 0); + + if (_sock_fd == -1) { + log::err << "Listener was unable to open a socket" << std::endl; + throw std::exception(); + } + struct sockaddr_in address {}; address.sin_family = AF_INET; - address.sin_port = htons(port); + address.sin_port = htons(_port); address.sin_addr.s_addr = INADDR_ANY; int reuse_opt = 1; - setsockopt(server_socket, SOL_SOCKET, SO_REUSEADDR, &reuse_opt, sizeof(reuse_opt)); - bind(server_socket, reinterpret_cast(&address), sizeof(address)); - - if (_nonblocking) { - fcntl(server_socket, F_SETFL, O_NONBLOCK); + if (setsockopt(_sock_fd, SOL_SOCKET, SO_REUSEADDR, &reuse_opt, sizeof(reuse_opt)) < 0) { + log::err << "Listener was unable to set SO_REUSEADDR" << std::endl; + throw std::exception(); } - listen(server_socket, max_queue); + if (bind(_sock_fd, reinterpret_cast(&address), sizeof(address)) != 0) { + log::err << "Listener was unable to bind to address" << std::endl; + throw std::exception(); + } + + if (fcntl(_sock_fd, F_SETFL, O_NONBLOCK) == -1) { + log::err << "Listener was unable to fcntl(O_NONBLOCK)" << std::endl; + throw std::exception(); + } + + if (listen(_sock_fd, max_queue) == -1) { + log::err << "Listener was unable to begin listening" << std::endl; + throw std::exception(); + } } -bool anthracite_socket::wait_for_conn() +bool listener::wait_for_conn(server** client_sock_p) { - client_ip = ""; - client_socket = accept(server_socket, reinterpret_cast(&client_addr), &client_addr_len); - if (client_socket > 0) { - if (_nonblocking) { - fcntl(client_socket, F_SETFL, O_NONBLOCK); + struct sockaddr_in client_addr {}; + socklen_t client_addr_len; + + int csock = accept(_sock_fd, reinterpret_cast(&client_addr), &client_addr_len); + + if (csock > 0) { + std::array ip_str { 0 }; + + if (inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN) == NULL) { + if (inet_ntop(AF_INET6, &client_addr.sin_addr, ip_str.data(), INET6_ADDRSTRLEN) == NULL) { + log::warn << "Unable to decode client's IP address" << std::endl; + } } - std::array ip_str { 0 }; - inet_ntop(AF_INET, &client_addr.sin_addr, ip_str.data(), INET_ADDRSTRLEN); - client_ip = std::string(ip_str.data()); + + std::string client_ip = std::string(ip_str.data()); + *client_sock_p = new server(csock, client_ip, _nonblocking); + return true; } else { return false; } } -const std::string& anthracite_socket::get_client_ip() +server::server(int sock_fd, std::string client_ip, bool nonblocking) + : _sock_fd(sock_fd) + , _client_ip(std::move(client_ip)) + , socket(nonblocking) { - return client_ip; -} - -void anthracite_socket::close_conn() { - close(client_socket); - client_socket = -1; -} - -void anthracite_socket::send_message(std::string& msg) -{ - if (client_socket == -1) { - return; + if (_nonblocking) { + if (fcntl(_sock_fd, F_SETFL, O_NONBLOCK) == -1) { + log::err << "Server was unable to fcntl(O_NONBLOCK)" << std::endl; + throw std::exception(); + } } - send(client_socket, &msg[0], msg.length(), 0); } -bool anthracite_socket::has_client() { - return client_socket > 0; -} - -std::string anthracite_socket::recv_message(int buffer_size) +void server::send_message(const std::string& msg) { - if (client_socket == -1) { - return ""; - } - - //setsockopt(client_socket, SOL_SOCKET, SO_RCVTIMEO, &timeout_tv, sizeof timeout_tv); + // Ignored because if we fail to send, it probably means + // a HUP will occur and it'll be closed. TODO: Just close + // it here and add a return value + (void)send(_sock_fd, &msg[0], msg.length(), 0); +} +std::string server::recv_message(int buffer_size) +{ + // Ignored because it's nonfatal, just slower int nodelay_opt = 1; - assert(setsockopt(client_socket, SOL_TCP, TCP_NODELAY, &nodelay_opt, sizeof(nodelay_opt)) == 0); + (void)setsockopt(_sock_fd, SOL_TCP, TCP_NODELAY, &nodelay_opt, sizeof(nodelay_opt)); std::vector response(buffer_size + 1); - ssize_t result = recv(client_socket, response.data(), buffer_size + 1, 0); + ssize_t result = recv(_sock_fd, response.data(), buffer_size + 1, 0); if (result < 1) { return ""; @@ -102,4 +124,21 @@ std::string anthracite_socket::recv_message(int buffer_size) return { response.data() }; } +server::~server() +{ + for (int i = 0; i < 5 && close(_sock_fd) != 0; ++i) + ; +} + +listener::~listener() +{ + for (int i = 0; i < 5 && close(_sock_fd) != 0; ++i) + ; +} + +const std::string& server::client_ip() +{ + return _client_ip; +} + }; diff --git a/lib/socket/socket.hpp b/lib/socket/socket.hpp index ea8622d..18590d3 100644 --- a/lib/socket/socket.hpp +++ b/lib/socket/socket.hpp @@ -10,32 +10,42 @@ namespace anthracite::socket { +class socket { + protected: + bool _nonblocking; + socket(bool nonblocking); + public: + socket(){} + virtual ~socket(){} +}; -class anthracite_socket { +class server : public socket { + protected: + int _sock_fd; + std::string _client_ip; + public: + server(int sock_fd, std::string client_ip, bool nonblocking); + ~server(); -protected: - bool _nonblocking; - struct timeval wait_timeout = { .tv_sec = 1, .tv_usec = 0}; - int server_socket; - int client_socket {}; - std::string client_ip; - struct sockaddr_in client_addr {}; - socklen_t client_addr_len {}; - static const struct timeval timeout_tv; - static const int MAX_QUEUE_LENGTH = 100; + virtual void send_message(const std::string& msg); + virtual std::string recv_message(int buffer_size); + const std::string& client_ip(); -public: - anthracite_socket(int port, int max_queue = MAX_QUEUE_LENGTH, bool nonblocking = false); + int fd() { return _sock_fd; } +}; - virtual const std::string& get_client_ip() final; +class listener : public socket { + protected: + uint16_t _port; + int _sock_fd; + public: + listener(int port, int max_queue_length, bool nonblocking); + ~listener(); - virtual bool has_client(); - virtual bool wait_for_conn(); - virtual void close_conn(); - virtual void send_message(std::string& msg); - virtual std::string recv_message(int buffer_size); + virtual bool wait_for_conn(server** client_sock_p); - int csock() { return client_socket; } + int fd() { return _sock_fd; } + int port() { return _port; } }; }; diff --git a/lib/thread_mgr/event_loop.cpp b/lib/thread_mgr/event_loop.cpp index 92c5bfb..a78956e 100644 --- a/lib/thread_mgr/event_loop.cpp +++ b/lib/thread_mgr/event_loop.cpp @@ -1,14 +1,14 @@ #include "./event_loop.hpp" #include "../log/log.hpp" -#include "../socket/openssl_socket.hpp" #include "assert.h" +#include "signal.h" #include "sys/epoll.h" #include #include #include #include #include -#include "signal.h" +#include using std::chrono::duration; using std::chrono::duration_cast; @@ -16,29 +16,18 @@ using std::chrono::high_resolution_clock; using std::chrono::milliseconds; namespace anthracite::thread_mgr { -event_loop::event::event(socket::anthracite_socket* socket, std::chrono::time_point timestamp) - : _socket(socket) - , _ts(timestamp) -{ -} -socket::anthracite_socket* event_loop::event::socket() -{ - return _socket; -} - -std::chrono::time_point& event_loop::event::timestamp() -{ - return _ts; -} - -event_loop::event_loop(backends::backend& backend, config::config& config) - : thread_mgr(backend, config) +event_loop::event_loop(std::vector& listen_sockets, backends::backend& backend, int max_threads, int max_clients) + : thread_mgr(backend) , _error_backend("./www") + , _max_threads(max_threads) + , _listen_sockets(listen_sockets) + , _max_clients(max_clients) + , _nonblocking(false) { } -bool event_loop::event_handler(socket::anthracite_socket* sock) +bool event_loop::event_handler(socket::server* sock) { std::string raw_request = sock->recv_message(http::HEADER_BYTES); @@ -46,7 +35,7 @@ bool event_loop::event_handler(socket::anthracite_socket* sock) return false; } - http::request req(raw_request, sock->get_client_ip()); + http::request req(raw_request, sock->client_ip()); std::unique_ptr resp = req.is_supported_version() ? _backend.handle_request(req) : _error_backend.handle_error(http::status_codes::HTTP_VERSION_NOT_SUPPORTED); std::string header = resp->header_to_string(); sock->send_message(header); @@ -65,107 +54,81 @@ void event_loop::worker_thread_loop(int threadno) ss << "worker " << threadno; pthread_setname_np(pthread_self(), ss.str().c_str()); - struct epoll_event* events = new struct epoll_event[_config.max_clients()]; + struct epoll_event* events = new struct epoll_event[_max_clients]; int timeout_ms = 1000; if (_nonblocking) { timeout_ms = 0; } - log::info << "Starting worker thread " << threadno << std::endl; + std::osyncstream(log::info) << "Starting worker thread " << threadno << std::endl; + + int epoll_fd = epoll_create(1); + + for (socket::listener* sl : _listen_sockets) { + struct epoll_event event; + event.events = EPOLLIN | EPOLLEXCLUSIVE; + event.data.ptr = sl; + epoll_ctl(epoll_fd, EPOLL_CTL_ADD, sl->fd(), &event); + if (threadno == 0) { + std::osyncstream(log::info) << "Listening started on port " << sl->port() << std::endl; + } + } while (_run) { - int ready_fds = epoll_wait(_epoll_fd, events, _config.max_clients(), timeout_ms); + int ready_fds = epoll_wait(epoll_fd, events, _max_clients, timeout_ms); if (ready_fds > 0) { for (int i = 0; i < ready_fds; i++) { - socket::anthracite_socket* sockptr = reinterpret_cast(events[i].data.ptr); + socket::socket* sockptr = reinterpret_cast(events[i].data.ptr); + socket::server* server_ptr = dynamic_cast(sockptr); - if (!event_handler(sockptr)) { - epoll_ctl(_epoll_fd, EPOLL_CTL_DEL, sockptr->csock(), &events[i]); - sockptr->close_conn(); - delete sockptr; + if (server_ptr != nullptr) { + if (!event_handler(server_ptr)) { + delete server_ptr; + } + } else { + socket::listener* listen_ptr = dynamic_cast(sockptr); + if (listen_ptr != nullptr) { + socket::server* server_sock; + while (listen_ptr->wait_for_conn(&server_sock)) { + struct epoll_event event; + event.events = EPOLLIN | EPOLLEXCLUSIVE; + event.data.ptr = server_sock; + epoll_ctl(epoll_fd, EPOLL_CTL_ADD, server_sock->fd(), &event); + } + } else { + std::osyncstream(log::err) << "Had socket type that wasn't listener or server" << std::endl; + } } } } } - log::info << "Stopping worker thread " << threadno << std::endl; -} + delete[] events; -void event_loop::listener_thread_loop(config::http_config& http_config) -{ - socket::anthracite_socket* socket; - - config::http_config* http_ptr = &http_config; - config::https_config* https_ptr = dynamic_cast(http_ptr); - - bool is_tls = https_ptr != nullptr; - - if (is_tls) { - socket = new socket::openssl_socket(https_ptr->port()); - } else { - socket = new socket::anthracite_socket(http_ptr->port()); - } - - std::osyncstream(log::info) << "Listening for " << (is_tls ? "HTTPS" : "HTTP") << " connections on port " << http_ptr->port() << std::endl; - - while (_run) { - if (socket->wait_for_conn()) { - socket::anthracite_socket* client_sock; - - if (is_tls) { - socket::openssl_socket* ssl_sock = dynamic_cast(socket); - client_sock = new socket::openssl_socket(*ssl_sock); - } else { - client_sock = new socket::anthracite_socket(*socket); - } - - struct epoll_event event; - event.events = EPOLLIN | EPOLLEXCLUSIVE;// | EPOLLET; - event.data.ptr = client_sock; - epoll_ctl(_epoll_fd, EPOLL_CTL_ADD, client_sock->csock(), &event); - } - } - - std::osyncstream(log::info) << "Stopping listening for " << (is_tls ? "HTTPS" : "HTTP") << " connections on port " << http_ptr->port() << std::endl; - - delete socket; + std::osyncstream(log::info) << "Stopping worker thread " << threadno << std::endl; } void event_loop::start() { + std::lock_guard lg(_run_lock); + signal(SIGPIPE, SIG_IGN); log::info << "Starting event_loop Thread Manager" << std::endl; _run = true; - _epoll_fd = epoll_create(1); - std::vector listener_threads; std::vector worker_threads; - for (int i = 0; i < _config.worker_threads(); i++) { + for (int i = 0; i < _max_threads; i++) { auto thread = std::thread(&event_loop::worker_thread_loop, this, i); worker_threads.push_back(std::move(thread)); } - if (_config.http_cfg().has_value()) { - auto thread = std::thread(&event_loop::listener_thread_loop, this, std::ref(_config.http_cfg().value())); - listener_threads.push_back(std::move(thread)); - } - - if (_config.https_cfg().has_value()) { - auto thread = std::thread(&event_loop::listener_thread_loop, this, std::ref(_config.https_cfg().value())); - listener_threads.push_back(std::move(thread)); - } - for (std::thread& t : worker_threads) { t.join(); } - - for (std::thread& t : listener_threads) { - t.join(); - } } void event_loop::stop() diff --git a/lib/thread_mgr/event_loop.hpp b/lib/thread_mgr/event_loop.hpp index 3fa8ea5..41c3a18 100644 --- a/lib/thread_mgr/event_loop.hpp +++ b/lib/thread_mgr/event_loop.hpp @@ -1,34 +1,25 @@ #include "./thread_mgr.hpp" #include "../socket/socket.hpp" #include "../backends/file_backend.hpp" -#include -#include #include -#include +#include +#include "../socket/socket.hpp" namespace anthracite::thread_mgr { class event_loop : public virtual thread_mgr { - class event { - socket::anthracite_socket* _socket; - std::chrono::time_point _ts; - public: - event(socket::anthracite_socket* socket, std::chrono::time_point timestamp); - socket::anthracite_socket* socket(); - std::chrono::time_point& timestamp(); - }; - - int _epoll_fd; std::mutex _event_mtx; backends::file_backend _error_backend; + std::vector& _listen_sockets; bool _nonblocking; + std::mutex _run_lock; + int _max_threads; + int _max_clients; void worker_thread_loop(int threadno); - void listener_thread_loop(config::http_config& http_config); - void eventer_thread_loop(); - bool event_handler(socket::anthracite_socket*); + bool event_handler(socket::server*); public: - event_loop(backends::backend& backend, config::config& config); + event_loop(std::vector&, backends::backend& backend, int max_workers, int max_clients); void start() override; void stop() override; }; diff --git a/lib/thread_mgr/thread_mgr.hpp b/lib/thread_mgr/thread_mgr.hpp index faae7ec..cbb7614 100644 --- a/lib/thread_mgr/thread_mgr.hpp +++ b/lib/thread_mgr/thread_mgr.hpp @@ -1,16 +1,14 @@ #pragma once #include "../backends/backend.hpp" -#include "../config/config.hpp" namespace anthracite::thread_mgr { class thread_mgr { protected: bool _run; backends::backend& _backend; - config::config& _config; public: - thread_mgr(backends::backend& backend, config::config& config): _backend(backend), _config(config) {} + thread_mgr(backends::backend& backend): _backend(backend) {} virtual ~thread_mgr() = default; virtual void start() = 0; virtual void stop() = 0; diff --git a/src/file_main.cpp b/src/file_main.cpp index ad3dcdd..3e7285b 100644 --- a/src/file_main.cpp +++ b/src/file_main.cpp @@ -1,35 +1,306 @@ #include "../lib/backends/file_backend.hpp" -#include "../lib/config/config.hpp" #include "../lib/log/log.hpp" +#include "../lib/socket/openssl_socket.hpp" +#include "../lib/socket/socket.hpp" #include "../lib/thread_mgr/event_loop.hpp" -#include "signal.h" +#include "../lib/version.hpp" +#include "getopt.h" #include "string.h" +#include "sys/signal.h" +#include #include +#include +#include +#include +#include +#include + +struct event_loop_config { + int max_workers; + int max_clients; +}; + +struct server_config { + std::unordered_map listeners; + std::optional event_loop; + std::string serve_dir = "./www"; + enum anthracite::log::LOG_LEVEL log_level = anthracite::log::LOG_LEVEL_INFO; +}; std::shared_ptr server = nullptr; extern "C" void signalHandler(int signum) { - //anthracite::log::warn << "Caught signal SIG" << sigabbrev_np(signum) << ", exiting Anthracite" << std::endl; + anthracite::log::warn << "Caught signal SIGIN, exiting Anthracite" << std::endl; if (server != nullptr) { server->stop(); } } -int main(int argc, char** argv) +bool read_config(server_config& config, const std::string& config_path) +{ + std::ifstream config_stream(config_path); + if (!config_stream.is_open()) { + anthracite::log::err << "Unable to open configuration file at '" << config_path << "', ensure that the file exists and permissions are set correctly" << std::endl; + return false; + } + + std::string line; + for (int lineno = 1; std::getline(config_stream, line); lineno++) { + bool parse_failed = false; + + std::stringstream ss_line(line); + std::string directive; + ss_line >> directive; + + if (ss_line.fail()) { + continue; + } + + if (directive == "http" || directive == "https") { + // http PORT QUEUE_LEN NONBLOCK + // https PORT QUEUE_LEN NONBLOCK CRT_PATH KEY_PATH + int port; + int queue_len; + std::string block; + std::string crt_path; + std::string key_path; + ss_line >> port >> queue_len >> block; + + if (directive == "https") { + ss_line >> crt_path >> key_path; + } + + bool nonblocking = false; + if (block == "blocking") { + nonblocking = false; + } else if (block == "nonblocking") { + nonblocking = true; + } else { + anthracite::log::err << "BLOCK is not a string of value blocking or nonblocking"; + parse_failed = true; + } + + parse_failed |= ss_line.fail(); + parse_failed |= !ss_line.eof(); + + if (parse_failed) { + anthracite::log::err << "Invalid http/https config on line " << lineno << " of configuration file" << std::endl; + anthracite::log::err << std::endl; + anthracite::log::err << "Format is: " << std::endl; + anthracite::log::err << "http PORT QUEUE_LENGTH BLOCK" << std::endl; + anthracite::log::err << "https PORT QUEUE_LENGTH BLOCK CRT_PATH KEY_PATH" << std::endl; + anthracite::log::err << std::endl; + anthracite::log::err << "PORT, QUEUE_LENGTH are integers" << std::endl; + anthracite::log::err << "BLOCK is a string of value blocking or nonblocking" << std::endl; + anthracite::log::err << "CRT_PATH and KEY_PATH are strings for the path or the certificate and key files respectively" << std::endl; + anthracite::log::err << std::endl + << "Line was: " << std::endl + << line << std::endl; + anthracite::log::err << "Check for trailing whitespace!" << std::endl; + return false; + } + + if (config.listeners.contains(port)) { + anthracite::log::err << "Invalid http/https config on line " << lineno << " of configuration file" << std::endl; + anthracite::log::err << "Port " << port << " is already being used" << std::endl; + return false; + } + + if (directive == "https") { + config.listeners[port] = new anthracite::socket::openssl_listener(key_path, crt_path, port, queue_len, nonblocking); + } else { + config.listeners[port] = new anthracite::socket::listener(port, queue_len, nonblocking); + } + } else if (directive == "log_level") { + // log_level LEVEL + std::string log_level; + ss_line >> log_level; + + parse_failed |= ss_line.fail(); + parse_failed |= !ss_line.eof(); + + if (log_level == "DEBUG") { + config.log_level = anthracite::log::LOG_LEVEL::LOG_LEVEL_DEBUG; + } else if (log_level == "VERBOSE") { + config.log_level = anthracite::log::LOG_LEVEL::LOG_LEVEL_VERBOSE; + } else if (log_level == "INFO") { + config.log_level = anthracite::log::LOG_LEVEL::LOG_LEVEL_INFO; + } else if (log_level == "WARN") { + config.log_level = anthracite::log::LOG_LEVEL::LOG_LEVEL_WARN; + } else if (log_level == "ERROR") { + config.log_level = anthracite::log::LOG_LEVEL::LOG_LEVEL_ERROR; + } else { + parse_failed = true; + } + + if (parse_failed) { + anthracite::log::err << "Invalid log_level config on line " << lineno << " of configuration file" << std::endl; + anthracite::log::err << std::endl; + anthracite::log::err << "Format is: " << std::endl; + anthracite::log::err << "log_level LEVEL" << std::endl; + anthracite::log::err << std::endl; + anthracite::log::err << "LEVEL is string of value DEBUG, VERBOSE, INFO, WARN, ERROR" << std::endl; + anthracite::log::err << std::endl + << "Line was: " << std::endl + << line << std::endl; + anthracite::log::err << "Check for trailing whitespace!" << std::endl; + return false; + } + } else if (directive == "event_loop") { + // event_loop MAX_WORKERS MAX_CLIENS + int max_workers; + int max_clients; + + ss_line >> max_workers >> max_clients; + + parse_failed |= ss_line.fail(); + parse_failed |= !ss_line.eof(); + + if (parse_failed) { + anthracite::log::err << "Invalid event_loop config on line " << lineno << " of configuration file" << std::endl; + anthracite::log::err << std::endl; + anthracite::log::err << "Format is: " << std::endl; + anthracite::log::err << "event_loop MAX_WORKERS MAX_CLIENTS" << std::endl; + anthracite::log::err << std::endl; + anthracite::log::err << "MAX_WORKERS is the maximum number of worker threads" << std::endl; + anthracite::log::err << "MAX_CLIENTS is the maximum number of concurrent clients" << std::endl; + anthracite::log::err << std::endl + << "Line was: " << std::endl + << line << std::endl; + anthracite::log::err << "Check for trailing whitespace!" << std::endl; + return false; + } + + if (max_workers <= 0) { + anthracite::log::err << "Invalid event_loop config on line " << lineno << " of configuration file" << std::endl; + anthracite::log::err << "MAX_WORKERS must be a positive, nonzero number" << std::endl; + return false; + } + + if (max_clients <= 0) { + anthracite::log::err << "Invalid event_loop config on line " << lineno << " of configuration file" << std::endl; + anthracite::log::err << "MAX_CLIENTS must be a positive, nonzero number" << std::endl; + return false; + } + + if (config.event_loop.has_value()) { + anthracite::log::err << "Invalid event_loop config on line " << lineno << " of configuration file" << std::endl; + anthracite::log::err << "A thread manager was already specified. You may only specify one at a time as of now." << std::endl; + return false; + } + + // Eww + config.event_loop = { .max_workers = max_workers, .max_clients = max_clients }; + } else if (directive == "www_dir") { + std::string www_dir; + ss_line >> www_dir; + + parse_failed |= ss_line.fail(); + parse_failed |= !ss_line.eof(); + + if (parse_failed) { + anthracite::log::err << "Invalid www_dir config on line " << lineno << " of configuration file" << std::endl; + anthracite::log::err << std::endl; + anthracite::log::err << "Format is: " << std::endl; + anthracite::log::err << "www_dir PATH" << std::endl; + anthracite::log::err << std::endl; + anthracite::log::err << "PATH is a path to a directory containing files to serve" << std::endl; + anthracite::log::err << std::endl + << "Line was: " << std::endl + << line << std::endl; + anthracite::log::err << "Check for trailing whitespace!" << std::endl; + return false; + } + } else { + anthracite::log::err << "Invalid configuration. Unknown directive " << directive << " on line " << lineno << std::endl; + return false; + } + } + + if (!config.event_loop.has_value()) { + anthracite::log::err << "Invalid configuration. Missing a thread manager. Try adding an event_loop directive." << std::endl; + return false; + } + + if (config.listeners.size() == 0) { + anthracite::log::err << "Invalid configuration. Missing listeners. Try adding a http or https directive." << std::endl; + return false; + } + + return true; +} + +int main(int argc, char* argv[]) { - anthracite::log::logger.initialize(anthracite::log::LOG_LEVEL_INFO); - anthracite::log::info << "Starting Anthracite, a higher performance web server" << std::endl; signal(SIGINT, signalHandler); + anthracite::log::logger.initialize(anthracite::log::LOG_LEVEL_INFO); + if (pthread_setname_np(pthread_self(), "main") != 0) { + anthracite::log::err << "Failed to set thread name via pthread_setname_np" << std::endl; + } - anthracite::backends::file_backend fb("./www"); - anthracite::config::config cfg(1, 10); - cfg.add_http_config(anthracite::config::http_config(8080)); - // cfg.add_https_config(config::https_config(8081, "", "")); + int opt_index = 0; + option options[] = { + { "help", no_argument, 0, 'h' }, + { "config", required_argument, 0, 'c' } + }; - server = std::make_shared(fb, cfg); + char c; + std::string config_path = "./anthracite.cfg"; + bool config_set = false; + while ((c = getopt_long(argc, argv, "hc:", options, &opt_index)) != -1) { + switch (c) { + case 'h': { + std::cerr << "Anthracite " << ANTHRACITE_VERSION_STRING << " Help" << std::endl; + std::cerr << std::endl; + std::cerr << "-h, --help Prints this help menu " << std::endl; + std::cerr << std::endl; + std::cerr << "-c, --config string (optional) Specifies the path of the configuration" << std::endl; + std::cerr << " file. Default is './anthracite.cfg'" << std::endl; + std::cerr << std::endl; + return 0; + break; + }; + case 'c': { + if (config_set) { + anthracite::log::err << "You cannot specify multiple configuration files" << std::endl; + return 1; + } + config_set = true; + config_path = std::string(optarg); + break; + }; + } + } + + anthracite::log::info << "Loading configuration file at path '" << config_path << "'" << std::endl; + + server_config cfg; + if (!read_config(cfg, config_path)) { + anthracite::log::err << "Failed reading configuration file at path '" << config_path << "'" << std::endl; + return 1; + } + + anthracite::log::logger.initialize(cfg.log_level); + + anthracite::log::info << "Serving files in directory " << cfg.serve_dir << std::endl; + anthracite::backends::file_backend fb(cfg.serve_dir); + + std::vector listeners; + + for (auto lp : cfg.listeners) { + listeners.push_back(lp.second); + } + + server = std::make_shared(listeners, fb, cfg.event_loop->max_workers, cfg.event_loop->max_clients); + + anthracite::log::info << "Starting Anthracite, a very high performance web server" << std::endl; server->start(); - anthracite::log::info << "Stopping Anthracite, a higher performance web server" << std::endl; + for (auto listener : listeners) { + delete listener; + } + + anthracite::log::info << "Stopping Anthracite, a very high performance web server" << std::endl; } From 5c196c05fc8ca5db3472f57f28a4f810398d5b0b Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Mon, 24 Feb 2025 19:37:23 -0500 Subject: [PATCH 12/15] small changes before 0.3.0 --- .direnv/nix-profile-24.11-mma8n3yfap91nw44 | 2 +- .direnv/nix-profile-24.11-mma8n3yfap91nw44.rc | 65 +++++++++++++++---- CHANGELOG.md | 1 + README.md | 33 +++++----- shell.nix | 2 +- 5 files changed, 70 insertions(+), 33 deletions(-) diff --git a/.direnv/nix-profile-24.11-mma8n3yfap91nw44 b/.direnv/nix-profile-24.11-mma8n3yfap91nw44 index 34f60a5..48b10b8 120000 --- a/.direnv/nix-profile-24.11-mma8n3yfap91nw44 +++ b/.direnv/nix-profile-24.11-mma8n3yfap91nw44 @@ -1 +1 @@ -/nix/store/w578af1s1zz4s6s3q2mhr5m2x7jq0cpi-nix-shell-env \ No newline at end of file +/nix/store/gr8ifjf51b4w3v62vvinq4s8w97pn3ag-nix-shell-env \ No newline at end of file diff --git a/.direnv/nix-profile-24.11-mma8n3yfap91nw44.rc b/.direnv/nix-profile-24.11-mma8n3yfap91nw44.rc index 2ac7bb9..0ffdf16 100644 --- a/.direnv/nix-profile-24.11-mma8n3yfap91nw44.rc +++ b/.direnv/nix-profile-24.11-mma8n3yfap91nw44.rc @@ -10,14 +10,16 @@ export AS BASH='/nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37/bin/bash' CC='gcc' export CC -CMAKE_INCLUDE_PATH='/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/include:/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/include' +CMAKE_INCLUDE_PATH='/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/include:/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/include:/nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8/include:/nix/store/zps3l0mc26r7bvjqd8x0y1lbc6gmbdvn-gnumake-4.4.1/include' export CMAKE_INCLUDE_PATH -CMAKE_LIBRARY_PATH='/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2/lib:/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc/lib:/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0/lib' +CMAKE_LIBRARY_PATH='/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2/lib:/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc/lib:/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0/lib:/nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8/lib' export CMAKE_LIBRARY_PATH CONFIG_SHELL='/nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37/bin/bash' export CONFIG_SHELL CXX='g++' export CXX +DETERMINISTIC_BUILD='1' +export DETERMINISTIC_BUILD HOSTTYPE='x86_64' HOST_PATH='/nix/store/6wgd8c9vq93mqxzc7jhkl86mv6qbc360-coreutils-9.5/bin:/nix/store/r99d2m4swgmrv9jvm4l9di40hvanq1aq-findutils-4.10.0/bin:/nix/store/3sln66ij8pg114apkd8p6nr04y37q5z2-diffutils-3.10/bin:/nix/store/yq39xdwm4z0fhx7dsm8mlpgvcz3vbfg3-gnused-4.9/bin:/nix/store/vniy1y5n8g28c55y7788npwc4h09fh7c-gnugrep-3.11/bin:/nix/store/scgfwh3z1s3l2vhvyjsgfgx5ql552sls-gawk-5.3.1/bin:/nix/store/0wqn2k3v5xzrc9rwinijdyr2ywwl82x4-gnutar-1.35/bin:/nix/store/5y240z436gf3rwmkwbhn1a17pqw509w4-gzip-1.13/bin:/nix/store/1yyryxp7mh7zsciapi8f1n0mnxkigmf8-bzip2-1.0.8-bin/bin:/nix/store/hbzw8k8ygv6bfzvsvnd4gb8qmm8xjbvn-gnumake-4.4.1/bin:/nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37/bin:/nix/store/rfrjws98w6scfx7m63grb0m6sg925ahd-patch-2.7.6/bin:/nix/store/4i4mjaf7z6gddspar487grxk5k1j4dcd-xz-5.6.3-bin/bin:/nix/store/9wbpsj6ksd16x1qdqs29xli1dpz3fnl0-file-5.45/bin' export HOST_PATH @@ -29,7 +31,7 @@ LD='ld' export LD LINENO='76' MACHTYPE='x86_64-pc-linux-gnu' -NIXPKGS_CMAKE_PREFIX_PATH='/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2:/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev:/nix/store/rc5j45brxiyl55fgd3adbcc43pdfr29v-openssl-3.3.2-bin:/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2:/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc:/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev:/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0:/nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5:/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0:/nix/store/ap724yhgv28mpsi1mmqcwypj4rrfhqmg-update-autotools-gnu-config-scripts-hook:/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0:/nix/store/qlzvmgr8w9prdlyys7irqf86p7bndf5b-binutils-wrapper-2.43.1' +NIXPKGS_CMAKE_PREFIX_PATH='/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2:/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev:/nix/store/rc5j45brxiyl55fgd3adbcc43pdfr29v-openssl-3.3.2-bin:/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2:/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc:/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev:/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0:/nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5:/nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8:/nix/store/zps3l0mc26r7bvjqd8x0y1lbc6gmbdvn-gnumake-4.4.1:/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0:/nix/store/ap724yhgv28mpsi1mmqcwypj4rrfhqmg-update-autotools-gnu-config-scripts-hook:/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0:/nix/store/qlzvmgr8w9prdlyys7irqf86p7bndf5b-binutils-wrapper-2.43.1' export NIXPKGS_CMAKE_PREFIX_PATH NIX_BINTOOLS='/nix/store/qlzvmgr8w9prdlyys7irqf86p7bndf5b-binutils-wrapper-2.43.1' export NIX_BINTOOLS @@ -41,13 +43,13 @@ NIX_CC='/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0' export NIX_CC NIX_CC_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu='1' export NIX_CC_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu -NIX_CFLAGS_COMPILE=' -frandom-seed=w578af1s1z -isystem /nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/include -isystem /nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/include -isystem /nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/include -isystem /nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/include' +NIX_CFLAGS_COMPILE=' -frandom-seed=gr8ifjf51b -isystem /nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/include -isystem /nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/include -isystem /nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8/include -isystem /nix/store/zps3l0mc26r7bvjqd8x0y1lbc6gmbdvn-gnumake-4.4.1/include -isystem /nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/include -isystem /nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/include -isystem /nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8/include -isystem /nix/store/zps3l0mc26r7bvjqd8x0y1lbc6gmbdvn-gnumake-4.4.1/include' export NIX_CFLAGS_COMPILE NIX_ENFORCE_NO_NATIVE='1' export NIX_ENFORCE_NO_NATIVE NIX_HARDENING_ENABLE='bindnow format fortify fortify3 pic relro stackprotector strictoverflow zerocallusedregs' export NIX_HARDENING_ENABLE -NIX_LDFLAGS='-rpath /home/nickorlow/programming/personal/anthracite/outputs/out/lib -L/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2/lib -L/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc/lib -L/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0/lib -L/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2/lib -L/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc/lib -L/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0/lib' +NIX_LDFLAGS='-rpath /home/nickorlow/programming/personal/anthracite/outputs/out/lib -L/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2/lib -L/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc/lib -L/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0/lib -L/nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8/lib -L/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2/lib -L/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc/lib -L/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0/lib -L/nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8/lib' export NIX_LDFLAGS NIX_NO_SELF_RPATH='1' NIX_PKG_CONFIG_WRAPPER_TARGET_HOST_x86_64_unknown_linux_gnu='1' @@ -64,13 +66,19 @@ OLDPWD='' export OLDPWD OPTERR='1' OSTYPE='linux-gnu' -PATH='/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2/bin:/nix/store/rc5j45brxiyl55fgd3adbcc43pdfr29v-openssl-3.3.2-bin/bin:/nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5/bin:/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0/bin:/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0/bin:/nix/store/62qjb50708fdhb4f2y7zxyqr1afir4fk-gcc-13.3.0/bin:/nix/store/29mb4q8b5306f4gk2wh38h0c1akb0n97-glibc-2.40-36-bin/bin:/nix/store/6wgd8c9vq93mqxzc7jhkl86mv6qbc360-coreutils-9.5/bin:/nix/store/qlzvmgr8w9prdlyys7irqf86p7bndf5b-binutils-wrapper-2.43.1/bin:/nix/store/vk4mlknqk9yjbqa68a7rvpfxfdw3rad7-binutils-2.43.1/bin:/nix/store/6wgd8c9vq93mqxzc7jhkl86mv6qbc360-coreutils-9.5/bin:/nix/store/r99d2m4swgmrv9jvm4l9di40hvanq1aq-findutils-4.10.0/bin:/nix/store/3sln66ij8pg114apkd8p6nr04y37q5z2-diffutils-3.10/bin:/nix/store/yq39xdwm4z0fhx7dsm8mlpgvcz3vbfg3-gnused-4.9/bin:/nix/store/vniy1y5n8g28c55y7788npwc4h09fh7c-gnugrep-3.11/bin:/nix/store/scgfwh3z1s3l2vhvyjsgfgx5ql552sls-gawk-5.3.1/bin:/nix/store/0wqn2k3v5xzrc9rwinijdyr2ywwl82x4-gnutar-1.35/bin:/nix/store/5y240z436gf3rwmkwbhn1a17pqw509w4-gzip-1.13/bin:/nix/store/1yyryxp7mh7zsciapi8f1n0mnxkigmf8-bzip2-1.0.8-bin/bin:/nix/store/hbzw8k8ygv6bfzvsvnd4gb8qmm8xjbvn-gnumake-4.4.1/bin:/nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37/bin:/nix/store/rfrjws98w6scfx7m63grb0m6sg925ahd-patch-2.7.6/bin:/nix/store/4i4mjaf7z6gddspar487grxk5k1j4dcd-xz-5.6.3-bin/bin:/nix/store/9wbpsj6ksd16x1qdqs29xli1dpz3fnl0-file-5.45/bin' +PATH='/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2/bin:/nix/store/rc5j45brxiyl55fgd3adbcc43pdfr29v-openssl-3.3.2-bin/bin:/nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5/bin:/nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8/bin:/nix/store/zps3l0mc26r7bvjqd8x0y1lbc6gmbdvn-gnumake-4.4.1/bin:/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0/bin:/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0/bin:/nix/store/62qjb50708fdhb4f2y7zxyqr1afir4fk-gcc-13.3.0/bin:/nix/store/29mb4q8b5306f4gk2wh38h0c1akb0n97-glibc-2.40-36-bin/bin:/nix/store/6wgd8c9vq93mqxzc7jhkl86mv6qbc360-coreutils-9.5/bin:/nix/store/qlzvmgr8w9prdlyys7irqf86p7bndf5b-binutils-wrapper-2.43.1/bin:/nix/store/vk4mlknqk9yjbqa68a7rvpfxfdw3rad7-binutils-2.43.1/bin:/nix/store/6wgd8c9vq93mqxzc7jhkl86mv6qbc360-coreutils-9.5/bin:/nix/store/r99d2m4swgmrv9jvm4l9di40hvanq1aq-findutils-4.10.0/bin:/nix/store/3sln66ij8pg114apkd8p6nr04y37q5z2-diffutils-3.10/bin:/nix/store/yq39xdwm4z0fhx7dsm8mlpgvcz3vbfg3-gnused-4.9/bin:/nix/store/vniy1y5n8g28c55y7788npwc4h09fh7c-gnugrep-3.11/bin:/nix/store/scgfwh3z1s3l2vhvyjsgfgx5ql552sls-gawk-5.3.1/bin:/nix/store/0wqn2k3v5xzrc9rwinijdyr2ywwl82x4-gnutar-1.35/bin:/nix/store/5y240z436gf3rwmkwbhn1a17pqw509w4-gzip-1.13/bin:/nix/store/1yyryxp7mh7zsciapi8f1n0mnxkigmf8-bzip2-1.0.8-bin/bin:/nix/store/hbzw8k8ygv6bfzvsvnd4gb8qmm8xjbvn-gnumake-4.4.1/bin:/nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37/bin:/nix/store/rfrjws98w6scfx7m63grb0m6sg925ahd-patch-2.7.6/bin:/nix/store/4i4mjaf7z6gddspar487grxk5k1j4dcd-xz-5.6.3-bin/bin:/nix/store/9wbpsj6ksd16x1qdqs29xli1dpz3fnl0-file-5.45/bin' export PATH PKG_CONFIG='pkg-config' export PKG_CONFIG -PKG_CONFIG_PATH='/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/lib/pkgconfig:/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/lib/pkgconfig' +PKG_CONFIG_PATH='/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev/lib/pkgconfig:/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/lib/pkgconfig:/nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8/lib/pkgconfig' export PKG_CONFIG_PATH PS4='+ ' +PYTHONHASHSEED='0' +export PYTHONHASHSEED +PYTHONNOUSERSITE='1' +export PYTHONNOUSERSITE +PYTHONPATH='/nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8/lib/python3.12/site-packages' +export PYTHONPATH RANLIB='ranlib' export RANLIB READELF='readelf' @@ -85,8 +93,12 @@ STRINGS='strings' export STRINGS STRIP='strip' export STRIP -XDG_DATA_DIRS='/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2/share:/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/share:/nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5/share:/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0/share' +XDG_DATA_DIRS='/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2/share:/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev/share:/nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5/share:/nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8/share:/nix/store/zps3l0mc26r7bvjqd8x0y1lbc6gmbdvn-gnumake-4.4.1/share:/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0/share' export XDG_DATA_DIRS +_PYTHON_HOST_PLATFORM='linux-x86_64' +export _PYTHON_HOST_PLATFORM +_PYTHON_SYSCONFIGDATA_NAME='_sysconfigdata__linux_x86_64-linux-gnu' +export _PYTHON_SYSCONFIGDATA_NAME __structuredAttrs='' export __structuredAttrs _substituteStream_has_warned_replace_deprecation='false' @@ -133,14 +145,14 @@ doInstallCheck='' export doInstallCheck dontAddDisableDepTrack='1' export dontAddDisableDepTrack -declare -a envBuildBuildHooks=() -declare -a envBuildHostHooks=() -declare -a envBuildTargetHooks=() +declare -a envBuildBuildHooks=('addPythonPath' 'sysconfigdataHook' ) +declare -a envBuildHostHooks=('addPythonPath' 'sysconfigdataHook' ) +declare -a envBuildTargetHooks=('addPythonPath' 'sysconfigdataHook' ) declare -a envHostHostHooks=('pkgConfigWrapper_addPkgConfigPath' 'addCMakeParams' 'ccWrapper_addCVars' 'bintoolsWrapper_addLDVars' ) declare -a envHostTargetHooks=('pkgConfigWrapper_addPkgConfigPath' 'addCMakeParams' 'ccWrapper_addCVars' 'bintoolsWrapper_addLDVars' ) declare -a envTargetTargetHooks=() declare -a fixupOutputHooks=('if [ -z "${dontPatchELF-}" ]; then patchELF "$prefix"; fi' 'if [[ -z "${noAuditTmpdir-}" && -e "$prefix" ]]; then auditTmpdir "$prefix"; fi' 'if [ -z "${dontGzipMan-}" ]; then compressManPages "$prefix"; fi' '_moveLib64' '_moveSbin' '_moveSystemdUserUnits' 'patchShebangsAuto' '_pruneLibtoolFiles' '_doStrip' ) -flag='-L/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0/lib' +flag='-L/nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8/lib' guess='12' iframework_seen='' initialPath='/nix/store/6wgd8c9vq93mqxzc7jhkl86mv6qbc360-coreutils-9.5 /nix/store/r99d2m4swgmrv9jvm4l9di40hvanq1aq-findutils-4.10.0 /nix/store/3sln66ij8pg114apkd8p6nr04y37q5z2-diffutils-3.10 /nix/store/yq39xdwm4z0fhx7dsm8mlpgvcz3vbfg3-gnused-4.9 /nix/store/vniy1y5n8g28c55y7788npwc4h09fh7c-gnugrep-3.11 /nix/store/scgfwh3z1s3l2vhvyjsgfgx5ql552sls-gawk-5.3.1 /nix/store/0wqn2k3v5xzrc9rwinijdyr2ywwl82x4-gnutar-1.35 /nix/store/5y240z436gf3rwmkwbhn1a17pqw509w4-gzip-1.13 /nix/store/1yyryxp7mh7zsciapi8f1n0mnxkigmf8-bzip2-1.0.8-bin /nix/store/hbzw8k8ygv6bfzvsvnd4gb8qmm8xjbvn-gnumake-4.4.1 /nix/store/gwgqdl0242ymlikq9s9s62gkp5cvyal3-bash-5.2p37 /nix/store/rfrjws98w6scfx7m63grb0m6sg925ahd-patch-2.7.6 /nix/store/4i4mjaf7z6gddspar487grxk5k1j4dcd-xz-5.6.3-bin /nix/store/9wbpsj6ksd16x1qdqs29xli1dpz3fnl0-file-5.45' @@ -149,7 +161,7 @@ mesonFlags='' export mesonFlags name='nix-shell-env' export name -nativeBuildInputs='/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2 /nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev /nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc /nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev /nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5' +nativeBuildInputs='/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2 /nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev /nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc /nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev /nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5 /nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8 /nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5 /nix/store/zps3l0mc26r7bvjqd8x0y1lbc6gmbdvn-gnumake-4.4.1' export nativeBuildInputs out='/home/nickorlow/programming/personal/anthracite/outputs/out' export out @@ -170,7 +182,7 @@ phases='buildPhase' export phases pkg='/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0' declare -a pkgsBuildBuild=() -declare -a pkgsBuildHost=('/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2' '/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev' '/nix/store/rc5j45brxiyl55fgd3adbcc43pdfr29v-openssl-3.3.2-bin' '/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2' '/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc' '/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev' '/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0' '/nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5' '/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0' '/nix/store/ap724yhgv28mpsi1mmqcwypj4rrfhqmg-update-autotools-gnu-config-scripts-hook' '/nix/store/h9lc1dpi14z7is86ffhl3ld569138595-audit-tmpdir.sh' '/nix/store/m54bmrhj6fqz8nds5zcj97w9s9bckc9v-compress-man-pages.sh' '/nix/store/wgrbkkaldkrlrni33ccvm3b6vbxzb656-make-symlinks-relative.sh' '/nix/store/5yzw0vhkyszf2d179m0qfkgxmp5wjjx4-move-docs.sh' '/nix/store/fyaryjvghbkpfnsyw97hb3lyb37s1pd6-move-lib64.sh' '/nix/store/kd4xwxjpjxi71jkm6ka0np72if9rm3y0-move-sbin.sh' '/nix/store/pag6l61paj1dc9sv15l7bm5c17xn5kyk-move-systemd-user-units.sh' '/nix/store/jivxp510zxakaaic7qkrb7v1dd2rdbw9-multiple-outputs.sh' '/nix/store/12lvf0c7xric9cny7slvf9cmhypl1p67-patch-shebangs.sh' '/nix/store/cickvswrvann041nqxb0rxilc46svw1n-prune-libtool-files.sh' '/nix/store/xyff06pkhki3qy1ls77w10s0v79c9il0-reproducible-builds.sh' '/nix/store/aazf105snicrlvyzzbdj85sx4179rpfp-set-source-date-epoch-to-latest.sh' '/nix/store/gps9qrh99j7g02840wv5x78ykmz30byp-strip.sh' '/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0' '/nix/store/qlzvmgr8w9prdlyys7irqf86p7bndf5b-binutils-wrapper-2.43.1' ) +declare -a pkgsBuildHost=('/nix/store/2gc1mdxn6whvjscqibxkfaamfhspbmcd-pkg-config-wrapper-0.29.2' '/nix/store/74z4cw9g5fhzkhydpsaac9d41b073dy6-openssl-3.3.2-dev' '/nix/store/rc5j45brxiyl55fgd3adbcc43pdfr29v-openssl-3.3.2-bin' '/nix/store/zs44kdd3k01schy32fa916pa17gr7y68-openssl-3.3.2' '/nix/store/bmjqxvy53752b3xfvbab6s87xq06hxbs-gcc-13.3.0-libgcc' '/nix/store/cqnj4iwzabn5crvvcigx5cjd2jif7n5y-boost-1.81.0-dev' '/nix/store/jn0mgmpirinjs8ag3iznn8j8jl2dp93s-boost-1.81.0' '/nix/store/yxf0cmyfrar671zqh0ml8pcw15mxk0mh-cmake-3.30.5' '/nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8' '/nix/store/zps3l0mc26r7bvjqd8x0y1lbc6gmbdvn-gnumake-4.4.1' '/nix/store/wnl9qpnhayry14lhcbdafhadsjwsdr6p-patchelf-0.15.0' '/nix/store/ap724yhgv28mpsi1mmqcwypj4rrfhqmg-update-autotools-gnu-config-scripts-hook' '/nix/store/h9lc1dpi14z7is86ffhl3ld569138595-audit-tmpdir.sh' '/nix/store/m54bmrhj6fqz8nds5zcj97w9s9bckc9v-compress-man-pages.sh' '/nix/store/wgrbkkaldkrlrni33ccvm3b6vbxzb656-make-symlinks-relative.sh' '/nix/store/5yzw0vhkyszf2d179m0qfkgxmp5wjjx4-move-docs.sh' '/nix/store/fyaryjvghbkpfnsyw97hb3lyb37s1pd6-move-lib64.sh' '/nix/store/kd4xwxjpjxi71jkm6ka0np72if9rm3y0-move-sbin.sh' '/nix/store/pag6l61paj1dc9sv15l7bm5c17xn5kyk-move-systemd-user-units.sh' '/nix/store/jivxp510zxakaaic7qkrb7v1dd2rdbw9-multiple-outputs.sh' '/nix/store/12lvf0c7xric9cny7slvf9cmhypl1p67-patch-shebangs.sh' '/nix/store/cickvswrvann041nqxb0rxilc46svw1n-prune-libtool-files.sh' '/nix/store/xyff06pkhki3qy1ls77w10s0v79c9il0-reproducible-builds.sh' '/nix/store/aazf105snicrlvyzzbdj85sx4179rpfp-set-source-date-epoch-to-latest.sh' '/nix/store/gps9qrh99j7g02840wv5x78ykmz30byp-strip.sh' '/nix/store/888bkaqdpfpx72dd8bdc69qsqlgbhcvf-gcc-wrapper-13.3.0' '/nix/store/qlzvmgr8w9prdlyys7irqf86p7bndf5b-binutils-wrapper-2.43.1' ) declare -a pkgsBuildTarget=() declare -a pkgsHostHost=() declare -a pkgsHostTarget=() @@ -683,6 +695,11 @@ addEnvHooks () eval "${pkgHookVar}s"'+=("$@")'; done } +addPythonPath () +{ + + addToSearchPathWithCustomDelimiter : PYTHONPATH $1/lib/python3.12/site-packages +} addToSearchPath () { @@ -2072,6 +2089,26 @@ substituteStream () done; printf "%s" "${!var}" } +sysconfigdataHook () +{ + + if [ "$1" = '/nix/store/kjgslpdqchx1sm7a5h9xibi5rrqcqfnl-python3-3.12.8' ]; then + export _PYTHON_HOST_PLATFORM='linux-x86_64'; + export _PYTHON_SYSCONFIGDATA_NAME='_sysconfigdata__linux_x86_64-linux-gnu'; + fi +} +toPythonPath () +{ + + local paths="$1"; + local result=; + for i in $paths; + do + p="$i/lib/python3.12/site-packages"; + result="${result}${result:+:}$p"; + done; + echo $result +} unpackFile () { diff --git a/CHANGELOG.md b/CHANGELOG.md index d34c86f..62590f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - Cleaned up code and seperated most code into headers & source - Revamped build system to use CMake properly - Moved CI/CD over to Forgejo +- Added simple config file system (will be completely replaced by v1.0) - General system stability improvements were made to enhance the user's experience ## HTTP Request Parser Rewrite diff --git a/README.md b/README.md index 629ba55..f5e06da 100644 --- a/README.md +++ b/README.md @@ -4,26 +4,25 @@ Anthracite is an extensible, low-dependency, fast web server. ## Developing -To build/develop Anthracite, you must have C++23, CMake, Make, and Python3 installed. +To build/develop Anthracite, you must have C++20, OpenSSL, CMake, Make, and Python3 installed. Create a `build/` directory, run `cmake ..`, and then `make` to build. -## Todo -- [x] HTTP/1.0 -- [x] Serve HTML Pages -- [x] Properly parse HTTP requests -- [x] Add module-based backend system for handling requests -- [x] Multithreading -- [x] HTTP/1.1 -- [x] Enhance logging -- [x] Create library that can be used to implement custom backends (i.e. webapi, fileserver, etc) -- [x] Faster parsing -- [ ] HTTP/2 -- [ ] Improve benchmarking infrastructure -- [ ] Fix glaring security issues -- [ ] Proper error handling -- [ ] User configuration -- [ ] Cleanup (this one will never truly be done) +## Features + +- HTTP/1.0 & HTTP/1.1 Support +- SSL via OpenSSL +- Event loop thread management +- libanthracite library for reating custom webservers +- Configuration through configuration file +- Minimal dependencies (only OpenSSL & stantart library so far) + +## Roadmap +- HTTP/2 +- HTTP/3 +- More threading modes +- Proxy backend +- Security/Error handling audit ## Screenshots diff --git a/shell.nix b/shell.nix index 201b9ef..4fd6aa6 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,6 @@ { pkgs ? import {} }: pkgs.mkShell { - nativeBuildInputs = [ pkgs.pkg-config pkgs.openssl pkgs.libgcc pkgs.boost pkgs.cmake ]; + nativeBuildInputs = [ pkgs.pkg-config pkgs.openssl pkgs.libgcc pkgs.boost pkgs.cmake pkgs.python312 pkgs.cmake pkgs.gnumake ]; shellHook = '' export OPENSSL_DIR="${pkgs.openssl.dev}" From f48eda81e891109d5a9bf2d7c18acb919be2e2ef Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Mon, 24 Feb 2025 19:41:41 -0500 Subject: [PATCH 13/15] add version.txt fix --- .github/workflows/docker-publish.yml | 2 +- build_supp/version.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 565eda4..6f745d0 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -28,7 +28,7 @@ jobs: - name: Write Version run: | echo Building with version number $RELEASE_VERSION - echo $RELEASE_VERSION > src/build/version.txt + echo $RELEASE_VERSION > src/build_supp/version.txt - name: Log into registry ${{ env.REGISTRY }} if: github.event_name != 'pull_request' diff --git a/build_supp/version.txt b/build_supp/version.txt index d510910..0d91a54 100644 --- a/build_supp/version.txt +++ b/build_supp/version.txt @@ -1 +1 @@ -0.3.0-dev +0.3.0 From 0fc8400066ba6d40a8f17474c8745a4e06c80f65 Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Mon, 24 Feb 2025 19:53:38 -0500 Subject: [PATCH 14/15] add version.txt --- .github/workflows/docker-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 6f745d0..f380c9d 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -28,7 +28,7 @@ jobs: - name: Write Version run: | echo Building with version number $RELEASE_VERSION - echo $RELEASE_VERSION > src/build_supp/version.txt + echo $RELEASE_VERSION > build_supp/version.txt - name: Log into registry ${{ env.REGISTRY }} if: github.event_name != 'pull_request' From efd19d3bbaaa340471b3530ffbaae7557af4360f Mon Sep 17 00:00:00 2001 From: Nicholas Orlowsky Date: Mon, 24 Feb 2025 21:34:54 -0500 Subject: [PATCH 15/15] fix version build step --- CMakeLists.txt | 25 +++++++++++-------------- Dockerfile | 2 +- src/file_main.cpp | 6 ++---- 3 files changed, 14 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9af19e2..4b70101 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,16 +2,19 @@ cmake_minimum_required(VERSION 3.10) project(anthracite) -set(CMAKE_CXX_STANDARD 23) +set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED True) set(CMAKE_CXX_FLAGS_RELEASE "-O3") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) find_package(OpenSSL REQUIRED) -add_custom_target(build-version - COMMAND cd ../build_supp && ./version.sh - DEPENDS build_supp/version.txt +configure_file(build_supp/version.txt version.txt COPYONLY) + +add_custom_command( + COMMAND ../build_supp/version.sh + DEPENDS version.txt + OUTPUT ${CMAKE_BINARY_DIR}/version.cpp COMMENT "Generated supplemental build files (version)" ) @@ -19,19 +22,13 @@ add_custom_target(build-supplemental COMMAND cd ../build_supp && python3 ./error_gen.py COMMAND mkdir -p www && cp -r ../default_www/regular/* ./www/ COMMAND cp ../build_supp/default_config.cfg ./anthracite.cfg - DEPENDS build_supp/version.txt ../default_www/regular/* build_supp/error_gen.py build-version + DEPENDS ../default_www/regular/* build_supp/error_gen.py ${CMAKE_BINARY_DIR}/version.cpp COMMENT "Generated supplemental build files (default www dir + default config + error pages)" ) -add_custom_target(run - COMMAND anthracite-bin - DEPENDS anthracite-bin - WORKING_DIRECTORY ${CMAKE_PROJECT_DIR} -) - -FILE(GLOB LIB_SOURCES lib/*.cpp lib/**/*.cpp build_supp/version.cpp) -add_library(anthracite ${LIB_SOURCES}) -add_dependencies(anthracite build-version) +FILE(GLOB LIB_SOURCES lib/*.cpp lib/**/*.cpp) +add_library(anthracite ${LIB_SOURCES} ${CMAKE_BINARY_DIR}/version.cpp) +add_dependencies(anthracite build-supplemental) target_link_libraries(anthracite OpenSSL::SSL OpenSSL::Crypto) target_include_directories(anthracite PUBLIC ${OPENSSL_INCLUDE_DIR}) diff --git a/Dockerfile b/Dockerfile index 3c5c6c2..449ea5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine as build-env +FROM alpine AS build-env RUN apk add --no-cache build-base python3 cmake openssl-dev COPY ./src ./src diff --git a/src/file_main.cpp b/src/file_main.cpp index 3e7285b..d7e5492 100644 --- a/src/file_main.cpp +++ b/src/file_main.cpp @@ -3,10 +3,8 @@ #include "../lib/socket/openssl_socket.hpp" #include "../lib/socket/socket.hpp" #include "../lib/thread_mgr/event_loop.hpp" -#include "../lib/version.hpp" +#include "signal.h" #include "getopt.h" -#include "string.h" -#include "sys/signal.h" #include #include #include @@ -251,7 +249,7 @@ int main(int argc, char* argv[]) while ((c = getopt_long(argc, argv, "hc:", options, &opt_index)) != -1) { switch (c) { case 'h': { - std::cerr << "Anthracite " << ANTHRACITE_VERSION_STRING << " Help" << std::endl; + std::cerr << "Anthracite Help" << std::endl; std::cerr << std::endl; std::cerr << "-h, --help Prints this help menu " << std::endl; std::cerr << std::endl;