Implement C++ transpiler backend with C++20 coroutines

- CppCodeBuilder emits a C++ class per script deriving from CppExecutableCode,
  registered by the FNV-1a hash of its source
- scripts and procedures are C++20 coroutines (CppCoroutine)
- run the shared codegen tests with the cpp backend: codegen_test_generator
  collects the generated code which is compiled into codegen_test
- add the codebase generator and scratchcpp-transpiler tool

Co-Authored-By: zemanadam893@gmail.com <zemanadam893@gmail.com>

Author: devin-ai-integration[bot]

CMakeLists.txt

@@ -155,6 +155,7 @@ endif()
 # C++ transpiler
 if (LIBSCRATCHCPP_CODEGEN_BACKEND STREQUAL "cpp")
     target_compile_definitions(scratchcpp PRIVATE CODEGEN_BACKEND_CPP)
+    add_subdirectory(transpiler)
 endif()
 
 # Macros

src/engine/internal/codegen/cpp/CMakeLists.txt

@@ -12,4 +12,8 @@ target_sources(scratchcpp
     cppconstant.h
     cppcodebuilder.cpp
     cppcodebuilder.h
+    cppcoroutine.h
+    cppprelude.h
+    cppcodebasegenerator.cpp
+    cppcodebasegenerator.h
 )

src/engine/internal/codegen/cpp/cppcodebasegenerator.cpp

@@ -0,0 +1,190 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#include <scratchcpp/project.h>
+#include <filesystem>
+#include <fstream>
+#include <sstream>
+#include <iostream>
+#include <iomanip>
+#include <map>
+#include <vector>
+
+#include "cppcodebasegenerator.h"
+#include "cppcompilercontext.h"
+
+using namespace libscratchcpp;
+
+namespace
+{
+
+bool writeFile(const std::filesystem::path &path, const std::string &content)
+{
+    std::ofstream file(path, std::ios::binary);
+
+    if (!file.is_open()) {
+        std::cerr << "error: failed to write " << path.string() << std::endl;
+        return false;
+    }
+
+    file << content;
+    return true;
+}
+
+} // namespace
+
+/*!
+ * Compiles the given project using the C++ backend and writes a CMake project to outputDir:
+ * - project.cpp - the generated code of all scripts
+ * - project_data.cpp - the embedded project file
+ * - main.cpp - loads and runs the embedded project
+ * - CMakeLists.txt
+ */
+bool CppCodebaseGenerator::generate(const std::string &projectFile, const std::string &outputDir, const std::string &projectName)
+{
+    std::ifstream input(projectFile, std::ios::binary);
+
+    if (!input.is_open()) {
+        std::cerr << "error: failed to read " << projectFile << std::endl;
+        return false;
+    }
+
+    std::stringstream data;
+    data << input.rdbuf();
+
+    // The code builder reports the source of every compiled script
+    std::map<uint64_t, std::string> sources;
+    CppCompilerContext::setSourceCallback([&sources](uint64_t hash, const std::string &source) { sources[hash] = source; });
+
+    std::stringstream code;
+    bool ret;
+
+    {
+        Project project(projectFile);
+        ret = project.load();
+    }
+
+    CppCompilerContext::setSourceCallback(nullptr);
+
+    if (ret) {
+        code << CppCompilerContext::sourceHeader();
+
+        for (const auto &[hash, source] : sources)
+            code << source << "\n";
+    }
+
+    if (!ret)
+        return false;
+
+    std::filesystem::path dir(outputDir);
+    std::error_code err;
+    std::filesystem::create_directories(dir, err);
+
+    return writeFile(dir / "project.cpp", code.str()) && writeFile(dir / "project_data.cpp", dataSource(data.str())) && writeFile(dir / "main.cpp", mainSource()) &&
+           writeFile(dir / "CMakeLists.txt", cmakeSource(projectName));
+}
+
+std::string CppCodebaseGenerator::mainSource()
+{
+    return R"(// Generated by libscratchcpp, do not edit
+
+#include <scratchcpp/project.h>
+#include <filesystem>
+#include <fstream>
+#include <string>
+
+extern const unsigned char scratchcpp_project_data[];
+extern const unsigned long long scratchcpp_project_size;
+
+int main(int argc, char **argv)
+{
+    // libscratchcpp loads projects from files, so the embedded project is extracted first
+    std::filesystem::path path = std::filesystem::temp_directory_path() / ("scratchcpp_project_" + std::to_string(std::hash<std::string>()(argv[0])) + ".sb3");
+
+    {
+        std::ofstream file(path, std::ios::binary);
+        file.write(reinterpret_cast<const char *>(scratchcpp_project_data), scratchcpp_project_size);
+    }
+
+    // The C++ backend finds the generated code of each script in project.cpp
+    libscratchcpp::Project project(path.string());
+    bool loaded = project.load();
+    std::error_code err;
+    std::filesystem::remove(path, err);
+
+    if (!loaded)
+        return 1;
+
+    project.run();
+    return 0;
+}
+)";
+}
+
+std::string CppCodebaseGenerator::dataSource(const std::string &data)
+{
+    std::stringstream ss;
+    ss << "// Generated by libscratchcpp, do not edit\n\n";
+    ss << "extern const unsigned long long scratchcpp_project_size = " << data.size() << "ULL;\n\n";
+    ss << "extern const unsigned char scratchcpp_project_data[] = {";
+
+    for (size_t i = 0; i < data.size(); i++) {
+        if (i % 16 == 0)
+            ss << "\n   ";
+
+        ss << " 0x" << std::hex << std::setw(2) << std::setfill('0') << (int)(unsigned char)data[i] << ",";
+    }
+
+    ss << std::dec << "\n    0x00\n};\n";
+    return ss.str();
+}
+
+std::string CppCodebaseGenerator::cmakeSource(const std::string &projectName)
+{
+    std::string ret = R"(# Generated by libscratchcpp
+cmake_minimum_required(VERSION 3.14)
+project(@NAME@ LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Use either libscratchcpp sources (LIBSCRATCHCPP_SOURCE_DIR) or a prebuilt library (LIBSCRATCHCPP_INCLUDE_DIR and LIBSCRATCHCPP_LIBRARY)
+set(LIBSCRATCHCPP_SOURCE_DIR "" CACHE PATH "libscratchcpp source directory")
+set(LIBSCRATCHCPP_INCLUDE_DIR "" CACHE PATH "libscratchcpp include directory")
+set(LIBSCRATCHCPP_LIBRARY "" CACHE FILEPATH "libscratchcpp library")
+
+add_executable(@NAME@ main.cpp project.cpp project_data.cpp)
+
+if(LIBSCRATCHCPP_SOURCE_DIR)
+    set(LIBSCRATCHCPP_BUILD_UNIT_TESTS OFF CACHE BOOL "" FORCE)
+    set(LIBSCRATCHCPP_CODEGEN_BACKEND "cpp" CACHE STRING "" FORCE)
+    add_subdirectory(${LIBSCRATCHCPP_SOURCE_DIR} libscratchcpp)
+    target_link_libraries(@NAME@ PRIVATE scratchcpp)
+elseif(LIBSCRATCHCPP_INCLUDE_DIR AND LIBSCRATCHCPP_LIBRARY)
+    target_include_directories(@NAME@ PRIVATE ${LIBSCRATCHCPP_INCLUDE_DIR})
+    target_link_libraries(@NAME@ PRIVATE ${LIBSCRATCHCPP_LIBRARY})
+else()
+    message(FATAL_ERROR "Set LIBSCRATCHCPP_SOURCE_DIR or LIBSCRATCHCPP_INCLUDE_DIR and LIBSCRATCHCPP_LIBRARY")
+endif()
+)";
+    std::string name = projectName;
+
+    for (char &c : name) {
+        if (!std::isalnum((unsigned char)c) && c != '_' && c != '-')
+            c = '_';
+    }
+
+    size_t pos;
+
+    while ((pos = ret.find("@NAME@")) != std::string::npos)
+        ret.replace(pos, 6, name);
+
+    return ret;
+}
+
+extern "C"
+{
+    LIBSCRATCHCPP_EXPORT bool cpp_generate_codebase(const char *projectFile, const char *outputDir, const char *projectName)
+    {
+        return CppCodebaseGenerator::generate(projectFile, outputDir, projectName);
+    }
+}

src/engine/internal/codegen/cpp/cppcodebasegenerator.h

@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <string>
+
+#include "test_export.h"
+
+namespace libscratchcpp
+{
+
+/*! Generates a standalone C++ codebase (CMake project) from a Scratch project. */
+class LIBSCRATCHCPP_TEST_EXPORT CppCodebaseGenerator
+{
+    public:
+        static bool generate(const std::string &projectFile, const std::string &outputDir, const std::string &projectName = "project");
+
+        static std::string mainSource();
+        static std::string dataSource(const std::string &data);
+        static std::string cmakeSource(const std::string &projectName);
+};
+
+} // namespace libscratchcpp