Reworked tools

tools/CMakeLists.txt

@@ -2,10 +2,6 @@ cmake_minimum_required(VERSION 3.24)
 
 project(smoke-api-tools LANGUAGES CXX)
 
-### Install CPM package manager
-
-#include(../KoalaBox/cmake/get_cpm.cmake)
-
 ### Install parser library
 
 CPMAddPackage(
@@ -24,11 +20,11 @@ add_grammar_from_repo(tree-sitter-cpp                  # Library name
 
 ### Install JSON library
 
-CPMAddPackage(
-    NAME nlohmann_json
-    GIT_REPOSITORY https://github.com/nlohmann/json.git
-    GIT_TAG v3.12.0
-)
+#CPMAddPackage(
+#    NAME nlohmann_json
+#    GIT_REPOSITORY https://github.com/nlohmann/json.git
+#    GIT_TAG v3.12.0
+#)
 
 ### Install HTTP client library
 
@@ -43,12 +39,15 @@ CPMAddPackage(
     GIT_TAG 1.12.0
 )
 
-### Install ZIP library https://github.com/richgel999/miniz
 CPMAddPackage(
-    NAME miniz
-    GIT_REPOSITORY https://github.com/richgel999/miniz.git
-    GIT_TAG c883286f1a6443720e7705450f59e579a4bbb8e2
+    NAME BS_thread_pool
+    GITHUB_REPOSITORY bshoshany/thread-pool
+    VERSION 5.0.0
+    EXCLUDE_FROM_ALL
+    SYSTEM
 )
+add_library(BS_thread_pool INTERFACE)
+target_include_directories(BS_thread_pool INTERFACE ${BS_thread_pool_SOURCE_DIR}/include)
 
 ### Steamworks Downloader executable
 
@@ -56,14 +55,16 @@ add_executable(steamworks_downloader steamworks_downloader.cpp)
 target_link_libraries(steamworks_downloader PRIVATE
     KoalaBox
     cpr # HTTP client
-    miniz # ZIP library
+    miniz # ZIP library TODO: Use koalabox instead
 )
 
 ### Steamworks Parser executable
 
 add_executable(steamworks_parser steamworks_parser.cpp)
 target_link_libraries(steamworks_parser PRIVATE
-    nlohmann_json   # JSON parser
-    tree-sitter-cpp # C++ grammar
-    cpp-tree-sitter # C++ bindings for tree-sitter
+    KoalaBox
+    BS_thread_pool
+#    nlohmann_json   # JSON parser
+#    tree-sitter-cpp # C++ grammar
+#    cpp-tree-sitter # C++ bindings for tree-sitter
 )

tools/steamworks_downloader.cpp

@@ -2,6 +2,7 @@
 #include <iostream>
 #include <random>
 #include <string>
+#include <ranges>
 
 #include <cpr/cpr.h>
 
@@ -11,8 +12,7 @@ namespace {
     namespace fs = std::filesystem;
     namespace zip = koalabox::zip;
 
-    // ReSharper disable once CppDFAConstantParameter
-    std::string generate_random_string(const size_t length) {
+    std::string generate_random_string() {
         static constexpr char charset[] = "0123456789"
                                           "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
                                           "abcdefghijklmnopqrstuvwxyz";
@@ -20,6 +20,7 @@ namespace {
         thread_local std::mt19937_64 rng{std::random_device{}()};
         thread_local std::uniform_int_distribution<std::size_t> dist(0, sizeof(charset) - 2);
 
+        constexpr auto length = 16;
         std::string result;
         result.reserve(length);
         for (std::size_t i = 0; i < length; ++i) {
@@ -59,7 +60,7 @@ namespace {
         );
 
         const auto zip_file_path =
-            fs::temp_directory_path() / (generate_random_string(16) + ".zip");
+            fs::temp_directory_path() / (generate_random_string() + ".zip");
 
         std::cout << "Downloading " << download_url << " to " << zip_file_path << std::endl;
 
@@ -87,7 +88,7 @@ namespace {
  * A tool for downloading Steamworks SDK and unpacking its headers and binaries
  * for further processing by other tools.
  */
-int main(const int argc, const char** argv) {
+int main(const int argc, const char** argv) { // NOLINT(*-exception-escape)
     if (argc == 1) {
         print_help();
         return 0;

tools/steamworks_parser.cpp

@@ -1,31 +1,28 @@
-#include <iostream>
-#include <functional>
-#include <filesystem>
+#include "koalabox/crypto.hpp"
+#include "koalabox/io.hpp"
+#include "koalabox/util.hpp"
+
+#include <chrono>
 #include <deque>
+#include <filesystem>
 #include <fstream>
+#include <functional>
 
-#include <nlohmann/json.hpp>
+#include <BS_thread_pool.hpp>
 #include <cpp-tree-sitter.h>
+#include <nlohmann/json.hpp>
 
-extern "C" const TSLanguage* tree_sitter_cpp();
+#include <koalabox/logger.hpp>
+#include <koalabox/parser.hpp>
+#include <koalabox/str.hpp>
 
 namespace {
-    namespace fs = std::filesystem;
     using json = nlohmann::json;
-
-    std::string_view trim(const std::string_view& s) {
-        const auto start =
-            std::ranges::find_if_not(s, [](const unsigned char ch) { return std::isspace(ch); });
-        const auto end =
-            std::find_if_not(
-                s.rbegin(), s.rend(),
-                [](const unsigned char ch) { return std::isspace(ch); }
-            ).base();
-        return start < end ? std::string_view(start, end) : std::string_view();
-    }
+    namespace fs = std::filesystem;
+    namespace parser = koalabox::parser;
 
     std::string_view unquote_if_quoted(const std::string_view& s) {
-        if (s.size() >= 2 && ((s.front() == '"' && s.back() == '"'))) {
+        if (s.size() >= 2 && s.front() == '"' && s.back() == '"') {
             return s.substr(1, s.size() - 2);
         }
 
@@ -33,145 +30,109 @@ namespace {
         return s;
     }
 
-    enum class VisitResult {
-        Continue,
-        SkipChildren,
-        Stop
-    };
-
-    void walk(
-        const ts::Node& root,
-        const std::function<VisitResult(ts::Node)>& visit
-    ) {
-        // DFS traversal
-        std::deque<ts::Node> queue;
-        queue.push_back(root);
-        auto first_visit = true;
-
-        while (not queue.empty()) {
-            const auto node = queue.front();
-            queue.pop_front();
-
-            switch (first_visit ? VisitResult::Continue : visit(node)) {
-            case VisitResult::Continue:
-                break;
-            case VisitResult::SkipChildren:
-                continue;
-            case VisitResult::Stop:
-                return;
-            }
-
-            for (uint32_t i = 0, count = node.getNumNamedChildren(); i < count; ++i) {
-                if (const auto child = node.getNamedChild(i); not child.isNull()) {
-                    queue.push_back(child);
-                }
-            }
-
-            first_visit = false;
-        }
-    }
-
     void parse_header(const std::string_view& source, json& lookup) {
-        const auto language = ts::Language(tree_sitter_cpp());
-        auto parser = ts::Parser(language);
-        const auto tree = parser.parseString(source);
+        const auto tree = parser::parse_source(source);
         const auto root = tree.getRootNode();
 
         json current_lookup = {};
         std::string interface_version;
 
-        walk(root, [&](const auto& current_node) {
+        parser::walk(root, [&](const auto& current_node) {
             const auto current_type = current_node.getType();
             if (current_type == "class_specifier") {
-
                 std::string interface_name;
                 [[maybe_unused]] int vt_idx = 0;
 
-                walk(current_node, [&](const ts::Node& class_node) {
+                koalabox::parser::walk(current_node, [&](const ts::Node& class_node) {
                     const auto type = class_node.getType();
                     const auto value = class_node.getSourceRange(source);
 
                     if (type == "type_identifier" && interface_name.empty()) {
                         interface_name = value;
-                        std::cout << "  Parsing interface: " << interface_name << std::endl;
+                        LOG_DEBUG("Found interface: {}", interface_name);
 
-                        return VisitResult::Continue;
+                        return parser::visit_result::Continue;
                     }
 
                     if (type == "field_declaration" && value.starts_with("virtual ")) {
                         if (value.starts_with("virtual ")) {
-                            walk(class_node, [&](const ts::Node& decl_node) {
+                            koalabox::parser::walk(class_node, [&](const ts::Node& decl_node) {
                                 if (decl_node.getType() == "field_identifier") {
-                                    const auto function_name =
-                                        decl_node.getSourceRange(source);
+                                    const auto function_name = decl_node.getSourceRange(source);
 
-                                    current_lookup[function_name]["vt_idx"] = vt_idx++;
-                                    return VisitResult::Stop;
+                                    current_lookup[function_name] = vt_idx++;
+                                    return parser::visit_result::Stop;
                                 }
-                                return VisitResult::Continue;
+                                return parser::visit_result::Continue;
                             });
                         }
 
-                        return VisitResult::SkipChildren;
+                        return parser::visit_result::SkipChildren;
                     }
 
-                    return VisitResult::Continue;
+                    return parser::visit_result::Continue;
                 });
             } else if (current_type == "preproc_def") {
-                walk(current_node, [&](const ts::Node& preproc_node) {
+                koalabox::parser::walk(current_node, [&](const ts::Node& preproc_node) {
                     if (preproc_node.getType() == "identifier") {
                         const auto identifier = preproc_node.getSourceRange(source);
 
                         return identifier.ends_with("INTERFACE_VERSION")
-                                   ? VisitResult::Continue
-                                   : VisitResult::Stop;
+                                   ? parser::visit_result::Continue
+                                   : parser::visit_result::Stop;
                     }
 
                     if (preproc_node.getType() == "preproc_arg") {
                         const auto quoted_version = preproc_node.getSourceRange(source);
-                        interface_version = unquote_if_quoted(trim(quoted_version));
-                        std::cout << "  Interface version: " << interface_version << std::endl;
+                        const auto trimmed_version = koalabox::str::trim(quoted_version);
+                        interface_version = unquote_if_quoted(trimmed_version);
+                        LOG_DEBUG("Interface version: {}", interface_version);
 
-                        return VisitResult::Stop;
+                        return parser::visit_result::Stop;
                     }
 
-                    return VisitResult::Continue;
+                    return parser::visit_result::Continue;
                 });
             } else if (current_type == "translation_unit" || current_type == "preproc_ifdef") {
-                return VisitResult::Continue;
+                return parser::visit_result::Continue;
             }
 
-            return VisitResult::SkipChildren;
+            return parser::visit_result::SkipChildren;
         });
 
         // Save the findings
+        static std::mutex mutex;
         if (not interface_version.empty()) {
+            const std::lock_guard lock(mutex);
             lookup[interface_version] = current_lookup;
         }
     }
 
-    void parse_sdk(const fs::path& sdk_path, json& lookup) {
+    void parse_sdk(const fs::path& sdk_path, json& lookup, BS::thread_pool<>& pool) {
         const auto headers_dir = sdk_path / "headers";
 
         if (not fs::exists(headers_dir)) {
-            std::cout << "Warning: SDK missing 'headers' directory: " << headers_dir << std::endl;
+            LOG_WARN("Warning: SDK missing 'headers' directory: {}", headers_dir.string());
             return;
         }
 
+        LOG_INFO("Parsing SDK: {}", headers_dir.string());
+
         // Go over each file in headers directory
         for (const auto& entry : fs::directory_iterator(headers_dir)) {
-            if (
-                const auto& header_path = entry.path();
-                header_path.extension() == ".h"
-            ) {
-                std::cout << "Parsing header: " << header_path << std::endl;
+            if (const auto& header_path = entry.path(); header_path.extension() == ".h") {
+                const auto task = pool.submit_task([&, header_path] {
+                    try {
+                        LOG_DEBUG("Parsing header: {}", header_path.string());
 
-                // Read file as text
-                std::ifstream in(header_path, std::ios::binary);
-                const std::string header_contents(std::istreambuf_iterator{in}, {});
+                        // Read file as text
+                        const auto header_contents = koalabox::io::read_file(header_path);
 
-                // Parse it
-                parse_header(header_contents, lookup);
+                        parse_header(header_contents, lookup);
+                    } catch (std::exception& e) {
+                        koalabox::util::panic(std::format("Error parsing header: {}", e.what()));
+                    }
+                });
             }
         }
     }
@@ -179,15 +140,23 @@ namespace {
     void generate_lookup_json(const fs::path& steamworks_dir) {
         json lookup;
 
+        // Create the thread pool
+        const auto thread_count = std::max(2U, std::thread::hardware_concurrency() / 2);
+        LOG_INFO("Creating task pool with {} threads", thread_count);
+        BS::thread_pool pool(thread_count);
+
         // Go over each steamworks sdk version
         for (const auto& entry : fs::directory_iterator(steamworks_dir)) {
             if (not entry.is_directory()) {
                 continue;
             }
 
-            parse_sdk(entry.path(), lookup);
+            parse_sdk(entry.path(), lookup, pool);
         }
 
+        // Wait for all tasks to finish
+        pool.wait();
+
         std::ofstream lookup_output("interface_lookup.json");
         lookup_output << std::setw(4) << lookup << std::endl;
     }
@@ -198,15 +167,22 @@ namespace {
  */
 int main() {
     try {
+        koalabox::logger::init_console_logger();
+
         const auto steamworks_dir = fs::path("steamworks");
 
         if (!fs::exists(steamworks_dir)) {
             throw std::exception("Expected to find 'steamworks' in current working directory.");
         }
 
+        const auto start = std::chrono::steady_clock::now();
         generate_lookup_json(steamworks_dir);
+        const auto end = std::chrono::steady_clock::now();
+        const auto elapsed = duration_cast<std::chrono::seconds>(end - start);
+
+        LOG_INFO("Finished parsing steamworks in {} seconds", elapsed.count());
     } catch (std::exception& e) {
-        std::cerr << "Error: " << e.what() << std::endl;
+        LOG_CRITICAL("Error: {}", e.what());
         return 1;
     }
 }