Add C++ codebase generator and transpiler tool

Co-Authored-By: zemanadam893@gmail.com <zemanadam893@gmail.com>

Author: devin-ai-integration[bot]

CMakeLists.txt

@@ -11,6 +11,7 @@ option(LIBSCRATCHCPP_NETWORK_SUPPORT "Support for downloading projects" ON)
 option(LIBSCRATCHCPP_PRINT_LLVM_IR "Print LLVM IR of compiled Scratch scripts (for debugging)" OFF)
 option(LIBSCRATCHCPP_ENABLE_CODE_ANALYZER "Analyze Scratch scripts to enable various optimizations" ON)
 option(LIBSCRATCHCPP_LLVM_INTEGER_SUPPORT "Use integers when possible to enable various optimizations" ON)
+option(LIBSCRATCHCPP_BUILD_TRANSPILER "Build the Scratch to C++ transpiler tool" ON)
 option(LIBSCRATCHCPP_ENABLE_SANITIZER "Enable sanitizer to detect memory issues" OFF)
 
 if (LIBSCRATCHCPP_ENABLE_SANITIZER)
@@ -153,6 +154,11 @@ target_compile_definitions(scratchcpp PUBLIC LIBSCRATCHCPP_VERSION_MAJOR=${PROJE
 target_compile_definitions(scratchcpp PUBLIC LIBSCRATCHCPP_VERSION_MINOR=${PROJECT_VERSION_MINOR})
 target_compile_definitions(scratchcpp PUBLIC LIBSCRATCHCPP_VERSION_PATCH=${PROJECT_VERSION_PATCH})
 
+# Scratch to C++ transpiler
+if (LIBSCRATCHCPP_BUILD_TRANSPILER)
+    add_subdirectory(transpiler)
+endif()
+
 # Unit tests
 target_link_libraries(scratchcpp PRIVATE scratchcpp-test-export)
 

src/engine/internal/cpp/CMakeLists.txt

@@ -8,7 +8,9 @@ target_sources(scratchcpp
     cppcodebuilderfactory.cpp
     cppcodebuilderfactory.h
     cppregister.h
-    cppcompilercontext.cpp
+  cppcodebasegenerator.cpp
+  cppcodebasegenerator.h
+  cppcompilercontext.cpp
     cppcompilercontext.h
     cppexecutablecode.cpp
     cppexecutablecode.h

src/engine/internal/cpp/cppcodebasegenerator.cpp

@@ -0,0 +1,205 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#include <scratchcpp/project.h>
+#include <filesystem>
+#include <fstream>
+#include <sstream>
+#include <iostream>
+#include <iomanip>
+#include <vector>
+
+#include "cppcodebasegenerator.h"
+#include "cppcodebuilderfactory.h"
+#include "cppcompilercontext.h"
+#include "../../compiler_p.h"
+
+using namespace libscratchcpp;
+
+namespace
+{
+
+// Creates C++ code builders and keeps the compiler contexts to collect the generated code
+class RecordingFactory : public ICodeBuilderFactory
+{
+    public:
+        std::shared_ptr<ICodeBuilder> create(CompilerContext *ctx, BlockPrototype *procedurePrototype, Compiler::CodeType codeType) const override
+        {
+            return CppCodeBuilderFactory::instance()->create(ctx, procedurePrototype, codeType);
+        }
+
+        std::shared_ptr<CompilerContext> createCtx(IEngine *engine, Target *target) const override
+        {
+            auto ctx = CppCodeBuilderFactory::instance()->createCtx(engine, target);
+            contexts.push_back(ctx);
+            return ctx;
+        }
+
+        mutable std::vector<std::shared_ptr<CompilerContext>> contexts;
+};
+
+bool writeFile(const std::filesystem::path &path, const std::string &content)
+{
+    std::ofstream file(path, std::ios::binary);
+
+    if (!file.is_open()) {
+        std::cerr << "error: failed to write " << path.string() << std::endl;
+        return false;
+    }
+
+    file << content;
+    return true;
+}
+
+} // namespace
+
+/*!
+ * Compiles the given project using the C++ backend and writes a CMake project to outputDir:
+ * - project.cpp - the generated code of all scripts
+ * - project_data.cpp - the embedded project file
+ * - main.cpp - loads and runs the embedded project
+ * - CMakeLists.txt
+ */
+bool CppCodebaseGenerator::generate(const std::string &projectFile, const std::string &outputDir, const std::string &projectName)
+{
+    std::ifstream input(projectFile, std::ios::binary);
+
+    if (!input.is_open()) {
+        std::cerr << "error: failed to read " << projectFile << std::endl;
+        return false;
+    }
+
+    std::stringstream data;
+    data << input.rdbuf();
+
+    RecordingFactory factory;
+    ICodeBuilderFactory *oldFactory = CompilerPrivate::builderFactory();
+    CompilerPrivate::setBuilderFactory(&factory);
+
+    std::stringstream code;
+    bool ret;
+
+    {
+        Project project(projectFile);
+        ret = project.load();
+
+        if (ret) {
+            code << "// Generated by libscratchcpp, do not edit\n\n" << CppCompilerContext::prelude() << "\n";
+
+            for (auto ctx : factory.contexts)
+                code << "\n" << static_cast<CppCompilerContext *>(ctx.get())->moduleSource();
+        }
+    }
+
+    CompilerPrivate::setBuilderFactory(oldFactory);
+
+    if (!ret)
+        return false;
+
+    std::filesystem::path dir(outputDir);
+    std::error_code err;
+    std::filesystem::create_directories(dir, err);
+
+    return writeFile(dir / "project.cpp", code.str()) && writeFile(dir / "project_data.cpp", dataSource(data.str())) && writeFile(dir / "main.cpp", mainSource()) &&
+           writeFile(dir / "CMakeLists.txt", cmakeSource(projectName));
+}
+
+std::string CppCodebaseGenerator::mainSource()
+{
+    return R"(// Generated by libscratchcpp, do not edit
+
+#include <scratchcpp/project.h>
+#include <filesystem>
+#include <fstream>
+#include <string>
+
+extern const unsigned char scratchcpp_project_data[];
+extern const unsigned long long scratchcpp_project_size;
+
+extern "C" void cpp_use_cpp_backend();
+
+int main(int argc, char **argv)
+{
+    // libscratchcpp loads projects from files, so the embedded project is extracted first
+    std::filesystem::path path = std::filesystem::temp_directory_path() / ("scratchcpp_project_" + std::to_string(std::hash<std::string>()(argv[0])) + ".sb3");
+
+    {
+        std::ofstream file(path, std::ios::binary);
+        file.write(reinterpret_cast<const char *>(scratchcpp_project_data), scratchcpp_project_size);
+    }
+
+    // Scripts are compiled by the C++ backend which uses the code in project.cpp
+    cpp_use_cpp_backend();
+
+    libscratchcpp::Project project(path.string());
+    bool loaded = project.load();
+    std::error_code err;
+    std::filesystem::remove(path, err);
+
+    if (!loaded)
+        return 1;
+
+    project.run();
+    return 0;
+}
+)";
+}
+
+std::string CppCodebaseGenerator::dataSource(const std::string &data)
+{
+    std::stringstream ss;
+    ss << "// Generated by libscratchcpp, do not edit\n\n";
+    ss << "extern const unsigned long long scratchcpp_project_size = " << data.size() << "ULL;\n\n";
+    ss << "extern const unsigned char scratchcpp_project_data[] = {";
+
+    for (size_t i = 0; i < data.size(); i++) {
+        if (i % 16 == 0)
+            ss << "\n   ";
+
+        ss << " 0x" << std::hex << std::setw(2) << std::setfill('0') << (int)(unsigned char)data[i] << ",";
+    }
+
+    ss << std::dec << "\n    0x00\n};\n";
+    return ss.str();
+}
+
+std::string CppCodebaseGenerator::cmakeSource(const std::string &projectName)
+{
+    std::string ret = R"(# Generated by libscratchcpp
+cmake_minimum_required(VERSION 3.14)
+project(@NAME@ LANGUAGES CXX)
+
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
+# Use either libscratchcpp sources (LIBSCRATCHCPP_SOURCE_DIR) or a prebuilt library (LIBSCRATCHCPP_INCLUDE_DIR and LIBSCRATCHCPP_LIBRARY)
+set(LIBSCRATCHCPP_SOURCE_DIR "" CACHE PATH "libscratchcpp source directory")
+set(LIBSCRATCHCPP_INCLUDE_DIR "" CACHE PATH "libscratchcpp include directory")
+set(LIBSCRATCHCPP_LIBRARY "" CACHE FILEPATH "libscratchcpp library")
+
+add_executable(@NAME@ main.cpp project.cpp project_data.cpp)
+
+if(LIBSCRATCHCPP_SOURCE_DIR)
+    set(LIBSCRATCHCPP_BUILD_UNIT_TESTS OFF CACHE BOOL "" FORCE)
+    add_subdirectory(${LIBSCRATCHCPP_SOURCE_DIR} libscratchcpp)
+    target_link_libraries(@NAME@ PRIVATE scratchcpp)
+elseif(LIBSCRATCHCPP_INCLUDE_DIR AND LIBSCRATCHCPP_LIBRARY)
+    target_include_directories(@NAME@ PRIVATE ${LIBSCRATCHCPP_INCLUDE_DIR})
+    target_link_libraries(@NAME@ PRIVATE ${LIBSCRATCHCPP_LIBRARY})
+else()
+    message(FATAL_ERROR "Set LIBSCRATCHCPP_SOURCE_DIR or LIBSCRATCHCPP_INCLUDE_DIR and LIBSCRATCHCPP_LIBRARY")
+endif()
+)";
+    std::string name = projectName;
+
+    for (char &c : name) {
+        if (!std::isalnum((unsigned char)c) && c != '_' && c != '-')
+            c = '_';
+    }
+
+    size_t pos;
+
+    while ((pos = ret.find("@NAME@")) != std::string::npos)
+        ret.replace(pos, 6, name);
+
+    return ret;
+}

src/engine/internal/cpp/cppcodebasegenerator.h

@@ -0,0 +1,23 @@
+// SPDX-License-Identifier: Apache-2.0
+
+#pragma once
+
+#include <string>
+
+#include "../../../test_export.h"
+
+namespace libscratchcpp
+{
+
+/*! Generates a standalone C++ codebase (CMake project) from a Scratch project. */
+class LIBSCRATCHCPP_TEST_EXPORT CppCodebaseGenerator
+{
+    public:
+        static bool generate(const std::string &projectFile, const std::string &outputDir, const std::string &projectName = "project");
+
+        static std::string mainSource();
+        static std::string dataSource(const std::string &data);
+        static std::string cmakeSource(const std::string &projectName);
+};
+
+} // namespace libscratchcpp

src/engine/internal/cpp/cppcodebuilder.cpp

@@ -6,6 +6,8 @@
 #include <scratchcpp/list.h>
 #include <scratchcpp/target.h>
 #include <scratchcpp/value.h>
+#include <scratchcpp/iengine.h>
+#include <scratchcpp/stage.h>
 #include <iomanip>
 #include <cmath>
 #include <cassert>
@@ -161,8 +163,13 @@ CompilerConstant *CppCodeBuilder::addConstValue(const Value &value)
             break;
 
         case ValueType::Pointer: {
-            // NOTE: Pointers are only valid in the current process
             type = Compiler::StaticType::Pointer;
+            expr = entityPtr(value.toPointer());
+
+            if (!expr.empty())
+                break;
+
+            // NOTE: Other pointers are only valid in the current process
             std::stringstream ss;
             ss << "reinterpret_cast<void *>(0x" << std::hex << reinterpret_cast<uintptr_t>(value.toPointer()) << ")";
             expr = ss.str();
@@ -1009,6 +1016,32 @@ const std::string &CppCodeBuilder::variablePtr(Variable *variable)
     return m_variables[variable] = name;
 }
 
+std::string CppCodeBuilder::entityPtr(const void *ptr)
+{
+    // Lists and variables are resolved at runtime, so generated code doesn't depend on their addresses
+    std::vector<Target *> targets;
+
+    if (m_ctx->target())
+        targets.push_back(m_ctx->target());
+
+    if (m_ctx->engine() && m_ctx->engine()->stage())
+        targets.push_back(m_ctx->engine()->stage());
+
+    for (Target *target : targets) {
+        for (auto list : target->lists()) {
+            if (list.get() == ptr)
+                return "static_cast<void *>(" + listPtr(list.get()) + ")";
+        }
+
+        for (auto var : target->variables()) {
+            if (var.get() == ptr)
+                return "static_cast<void *>(" + variablePtr(var.get()) + ")";
+        }
+    }
+
+    return "";
+}
+
 const std::string &CppCodeBuilder::listPtr(List *list)
 {
     auto it = m_lists.find(list);

src/engine/internal/cpp/cppcodebuilder.h

@@ -164,6 +164,7 @@ class LIBSCRATCHCPP_TEST_EXPORT CppCodeBuilder : public ICodeBuilder
 
         const std::string &variablePtr(Variable *variable);
         const std::string &listPtr(List *list);
+        std::string entityPtr(const void *ptr);
 
         CppCompilerContext *m_ctx = nullptr;
         BlockPrototype *m_procedurePrototype = nullptr;

src/engine/internal/cpp/cppruntime.cpp

@@ -4,6 +4,9 @@
 #include <cassert>
 
 #include "cppruntime.h"
+#include "cppcodebuilderfactory.h"
+#include "cppcodebasegenerator.h"
+#include "../../compiler_p.h"
 #include "cppexecutioncontext.h"
 
 using namespace libscratchcpp;
@@ -46,3 +49,16 @@ extern "C"
         return static_cast<CppExecutionContext *>(ctx)->finished();
     }
 }
+
+extern "C"
+{
+    LIBSCRATCHCPP_EXPORT void cpp_use_cpp_backend()
+    {
+        CompilerPrivate::setBuilderFactory(CppCodeBuilderFactory::instance().get());
+    }
+
+    LIBSCRATCHCPP_EXPORT bool cpp_generate_codebase(const char *projectFile, const char *outputDir, const char *projectName)
+    {
+        return CppCodebaseGenerator::generate(projectFile, outputDir, projectName);
+    }
+}

src/engine/internal/cpp/cppruntime.h

@@ -18,6 +18,12 @@ extern "C"
     LIBSCRATCHCPP_EXPORT void cpp_register_function(uint64_t moduleKey, const char *name, void *function);
     LIBSCRATCHCPP_EXPORT void cpp_mark_thread_as_finished(ExecutionContext *ctx);
     LIBSCRATCHCPP_EXPORT bool cpp_is_thread_finished(ExecutionContext *ctx);
+
+    // Makes the compiler use the C++ backend (used by generated codebases)
+    LIBSCRATCHCPP_EXPORT void cpp_use_cpp_backend();
+
+    // Generates a standalone C++ codebase from a Scratch project
+    LIBSCRATCHCPP_EXPORT bool cpp_generate_codebase(const char *projectFile, const char *outputDir, const char *projectName);
 }
 
 } // namespace libscratchcpp

test/cpp/cases.h

@@ -38,9 +38,9 @@ struct CppTestCase
 {
         std::string name;
         std::function<std::shared_ptr<ExecutableCode>(CppTestEnv &)> build;
-        std::string expectedOutput;  // standard output after running the code
-        int expectedRuns = -1;       // the number of run() calls until the script finishes (-1 to skip)
-        std::string expectedValue;   // the return value of reporters and hat predicates
+        std::string expectedOutput; // standard output after running the code
+        int expectedRuns = -1;      // the number of run() calls until the script finishes (-1 to skip)
+        std::string expectedValue;  // the return value of reporters and hat predicates
         Compiler::CodeType codeType = Compiler::CodeType::Script;
 };