diff --git a/.proj.toml b/.proj.toml index 46d9b37b55..93c92e300f 100644 --- a/.proj.toml +++ b/.proj.toml @@ -112,6 +112,10 @@ cuda = false type = "bin" cuda = false +[targets.import-pt2] +type = "bin" +cuda = false + [targets.substitution-to-dot] type = "bin" cuda = false diff --git a/CMakeLists.txt b/CMakeLists.txt index 0138011ac2..8e627f6c27 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -50,6 +50,7 @@ option(FF_BUILD_SP_IZATION_BENCHMARKING "build sp-ization benchmarking" ON) option(FF_BUILD_ARG_PARSER "build command line argument parser" OFF) option(FF_BUILD_BIN_COMPILE_MODEL "build compile-model binary" ON) option(FF_BUILD_BIN_EXPORT_MODEL_ARCH "build export-model-arch utility" ON) +option(FF_BUILD_BIN_IMPORT_PT2 "build import-pt2 utility" ON) option(FF_BUILD_BIN_RUN_MODEL "build run-model binary" ON) if (FF_USE_CODE_COVERAGE) diff --git a/bin/CMakeLists.txt b/bin/CMakeLists.txt index f62b7d632c..f84f2ac06c 100644 --- a/bin/CMakeLists.txt +++ b/bin/CMakeLists.txt @@ -18,6 +18,10 @@ if(FF_BUILD_BIN_EXPORT_MODEL_ARCH) add_subdirectory(export-model-arch) endif() +if(FF_BUILD_BIN_IMPORT_PT2) + add_subdirectory(import-pt2) +endif() + if(FF_BUILD_BIN_RUN_MODEL) add_subdirectory(run-model) endif() diff --git a/bin/import-pt2/CMakeLists.txt b/bin/import-pt2/CMakeLists.txt new file mode 100644 index 0000000000..bfb988603e --- /dev/null +++ b/bin/import-pt2/CMakeLists.txt @@ -0,0 +1,11 @@ +ff_add_executable( + NAME + import-pt2 + SRC_PATTERNS + src/*.cc + PRIVATE_INCLUDE + include/ + DEPS + utils + pcg +) diff --git a/bin/import-pt2/include/import-pt2/pt2_archive.h b/bin/import-pt2/include/import-pt2/pt2_archive.h new file mode 100644 index 0000000000..820cc102ef --- /dev/null +++ b/bin/import-pt2/include/import-pt2/pt2_archive.h @@ -0,0 +1,51 @@ +#ifndef _FLEXFLOW_BIN_IMPORT_PT2_INCLUDE_IMPORT_PT2_PT2_ARCHIVE_H +#define _FLEXFLOW_BIN_IMPORT_PT2_INCLUDE_IMPORT_PT2_PT2_ARCHIVE_H + +#include +#include +#include +#include + +namespace FlexFlow { + +/** + * @brief Location of one entry inside a zip file. + */ +struct ZipEntryLocation { + uint16_t compression_method; + uint64_t compressed_size; + uint64_t uncompressed_size; + uint64_t local_header_offset; +}; + +/** + * @brief Read the central directory of the zip file at \p path (zip64 aware). + * + * @return a map from entry name to its location. + */ +std::map + read_zip_central_directory(std::string const &path); + +/** + * @brief Read the contents of a stored (uncompressed) zip entry. + * + * PyTorch's archive writer stores every entry uncompressed, so no inflate + * support is needed. Throws if the entry is compressed. + */ +std::string read_stored_zip_entry(std::string const &path, + std::string const &entry_name, + ZipEntryLocation const &location); + +/** + * @brief Read and parse models/{model_name}.json (the serialized + * ExportedProgram) from a torch.export.save .pt2 archive. + * + * The archive's top-level directory is not fixed (PyTorch names it after the + * file), so any single top-level directory is accepted. + */ +nlohmann::json read_pt2_model_json(std::string const &pt2_path, + std::string const &model_name = "model"); + +} // namespace FlexFlow + +#endif diff --git a/bin/import-pt2/include/import-pt2/pt2_to_computation_graph.h b/bin/import-pt2/include/import-pt2/pt2_to_computation_graph.h new file mode 100644 index 0000000000..9cb871a466 --- /dev/null +++ b/bin/import-pt2/include/import-pt2/pt2_to_computation_graph.h @@ -0,0 +1,37 @@ +#ifndef _FLEXFLOW_BIN_IMPORT_PT2_INCLUDE_IMPORT_PT2_PT2_TO_COMPUTATION_GRAPH_H +#define _FLEXFLOW_BIN_IMPORT_PT2_INCLUDE_IMPORT_PT2_PT2_TO_COMPUTATION_GRAPH_H + +#include "pcg/computation_graph.dtg.h" +#include +#include + +namespace FlexFlow { + +/** + * @brief Thrown when a .pt2 graph uses something the importer does not + * support (an unknown ATen op, dynamic shapes, unsupported arguments, ...). + */ +struct Pt2ImportError : public std::runtime_error { + using std::runtime_error::runtime_error; +}; + +/** + * @brief Rebuild a serialized torch.export ExportedProgram (the contents of + * models/model.json in a .pt2 archive) as a FlexFlow + * ComputationGraph, using ComputationGraphBuilder. + * + * Each ATen node becomes the matching builder call. Parameters and buffers + * are not imported as tensors; they are implied by the op that owns them (e.g. + * aten.conv2d's weight determines the conv's output channels and + * kernel size, and its bias determines use_bias). Weight values are + * not imported. + * + * After every op, the shape FlexFlow infers is checked against the shape + * recorded in the .pt2. Any mismatch throws Pt2ImportError naming the node. + */ +ComputationGraph + computation_graph_from_pt2_model_json(nlohmann::json const &model); + +} // namespace FlexFlow + +#endif diff --git a/bin/import-pt2/index.dox b/bin/import-pt2/index.dox new file mode 100644 index 0000000000..bd2c952410 --- /dev/null +++ b/bin/import-pt2/index.dox @@ -0,0 +1,28 @@ +namespace FlexFlow { +/** + +@page import-pt2 import-pt2 + +\brief Imports a PyTorch model saved with torch.export.save() (a .pt2 file) as a FlexFlow ComputationGraph, and outputs it as JSON (see \ref file-format) or DOT. + +The .pt2 archive is read directly (no libtorch dependency). Each ATen op in the exported graph is rebuilt with ComputationGraphBuilder; parameters and buffers are implied by the ops that use them. Weight values are not imported. + +Export a model from PyTorch: + +\verbatim +ep = torch.export.export(model.eval(), (example_input,)) +torch.export.save(ep, "model.pt2") +\endverbatim + +Then build and run \c import-pt2 from the root of the %FlexFlow %Train repository: + +\verbatim +$ proj cmake # if you haven't already +$ proj build +$ ./build/normal/bin/import-pt2/import-pt2 model.pt2 > model.json +$ ./build/normal/bin/import-pt2/import-pt2 --dot model.pt2 > model.dot +\endverbatim + +Unsupported ops, dynamic shapes, and unsupported op arguments are reported as errors naming the offending node. +*/ +} diff --git a/bin/import-pt2/src/import-pt2/main.cc b/bin/import-pt2/src/import-pt2/main.cc new file mode 100644 index 0000000000..bdbb2a6c52 --- /dev/null +++ b/bin/import-pt2/src/import-pt2/main.cc @@ -0,0 +1,77 @@ +#include "import-pt2/pt2_archive.h" +#include "import-pt2/pt2_to_computation_graph.h" +#include "pcg/computation_graph.h" +#include "pcg/file_format/v1/v1_computation_graph.h" +#include "utils/cli/cli_get_help_message.h" +#include "utils/cli/cli_parse.h" +#include "utils/cli/cli_parse_result.h" +#include "utils/cli/cli_spec.h" +#include + +using namespace ::FlexFlow; + +int main(int argc, char **argv) { + CLISpec cli; + + CLIArgumentKey key_help = cli_add_help_flag(cli); + + CLIArgumentKey key_dot = cli.add_flag(CLIFlagSpec{ + /*long_flag=*/"dot", + /*short_flag=*/std::nullopt, + /*description=*/ + "output a dot representation of the imported computation graph " + "instead of JSON", + }); + + CLIArgumentKey key_pt2_path = + cli.add_positional_argument(CLIPositionalArgumentSpec{ + /*name=*/"pt2_file", + /*choices=*/std::nullopt, + /*description=*/"a .pt2 file written by torch.export.save()", + }); + + assert(argc >= 1); + std::string prog_name = argv[0]; + + CLIParseResult parsed = ({ + tl::expected result = + cli_parse(cli, argc, argv); + if (!result.has_value()) { + std::cerr << cli_get_help_message(prog_name, cli); + std::cerr << std::endl; + std::cerr << "error: " << result.error() << std::endl; + return 1; + } + + result.value(); + }); + + if (cli_get_flag(parsed, key_help)) { + std::cerr << cli_get_help_message(prog_name, cli); + return 1; + } + + std::string pt2_path = cli_get_positional_argument(parsed, key_pt2_path); + bool dot = cli_get_flag(parsed, key_dot); + + ComputationGraph cg = ({ + std::optional result; + try { + result = + computation_graph_from_pt2_model_json(read_pt2_model_json(pt2_path)); + } catch (std::exception const &e) { + std::cerr << "error: " << e.what() << std::endl; + return 1; + } + result.value(); + }); + + if (dot) { + std::cout << as_dot(cg) << std::endl; + } else { + nlohmann::json json_output = to_v1(cg); + std::cout << json_output.dump(2) << std::endl; + } + + return 0; +} diff --git a/bin/import-pt2/src/import-pt2/pt2_archive.cc b/bin/import-pt2/src/import-pt2/pt2_archive.cc new file mode 100644 index 0000000000..60a8472d63 --- /dev/null +++ b/bin/import-pt2/src/import-pt2/pt2_archive.cc @@ -0,0 +1,267 @@ +#include "import-pt2/pt2_archive.h" +#include +#include +#include +#include +#include +#include + +namespace FlexFlow { + +namespace { + +std::string join(std::vector const &v) { + std::string result; + for (size_t i = 0; i < v.size(); i++) { + result += (i == 0 ? "" : ", ") + v.at(i); + } + return result; +} + +constexpr uint32_t EOCD_SIGNATURE = 0x06054b50; +constexpr uint32_t ZIP64_EOCD_LOCATOR_SIGNATURE = 0x07064b50; +constexpr uint32_t ZIP64_EOCD_SIGNATURE = 0x06064b50; +constexpr uint32_t CENTRAL_DIR_HEADER_SIGNATURE = 0x02014b50; +constexpr uint32_t LOCAL_HEADER_SIGNATURE = 0x04034b50; +constexpr uint16_t ZIP64_EXTRA_FIELD_ID = 0x0001; +constexpr size_t EOCD_SIZE = 22; +constexpr size_t ZIP64_EOCD_LOCATOR_SIZE = 20; +constexpr size_t ZIP64_EOCD_SIZE = 56; +constexpr size_t CENTRAL_DIR_HEADER_SIZE = 46; +constexpr size_t LOCAL_HEADER_SIZE = 30; +constexpr size_t MAX_ZIP_COMMENT_SIZE = 0xFFFF; + +[[noreturn]] void fail(std::string const &path, std::string const &msg) { + throw std::runtime_error(fmt::format("{}: {}", path, msg)); +} + +uint64_t read_le(std::string const &buf, size_t offset, size_t num_bytes) { + if (offset + num_bytes > buf.size()) { + throw std::out_of_range("zip record truncated"); + } + uint64_t result = 0; + for (size_t i = 0; i < num_bytes; i++) { + result |= static_cast(static_cast(buf[offset + i])) + << (8 * i); + } + return result; +} + +uint16_t read_u16(std::string const &buf, size_t offset) { + return static_cast(read_le(buf, offset, 2)); +} + +uint32_t read_u32(std::string const &buf, size_t offset) { + return static_cast(read_le(buf, offset, 4)); +} + +uint64_t read_u64(std::string const &buf, size_t offset) { + return read_le(buf, offset, 8); +} + +std::ifstream open_binary(std::string const &path) { + std::ifstream f(path, std::ios::binary); + if (!f) { + fail(path, "could not open file"); + } + return f; +} + +std::string read_range(std::ifstream &f, + std::string const &path, + uint64_t offset, + uint64_t size) { + std::string buf(size, '\0'); + f.seekg(static_cast(offset)); + f.read(buf.data(), static_cast(size)); + if (static_cast(f.gcount()) != size) { + fail(path, + fmt::format("unexpected end of file reading {} bytes at offset {}", + size, + offset)); + } + return buf; +} + +} // namespace + +std::map + read_zip_central_directory(std::string const &path) { + std::ifstream f = open_binary(path); + f.seekg(0, std::ios::end); + uint64_t file_size = static_cast(f.tellg()); + if (file_size < EOCD_SIZE) { + fail(path, "not a zip file (too small)"); + } + + // The end-of-central-directory record is at the end of the file, followed + // only by an optional comment of at most 64 KiB. + uint64_t tail_size = + std::min(file_size, EOCD_SIZE + MAX_ZIP_COMMENT_SIZE); + uint64_t tail_offset = file_size - tail_size; + std::string tail = read_range(f, path, tail_offset, tail_size); + + std::optional eocd_pos; + for (size_t i = tail.size() - EOCD_SIZE + 1; i-- > 0;) { + if (read_u32(tail, i) == EOCD_SIGNATURE) { + eocd_pos = i; + break; + } + } + if (!eocd_pos.has_value()) { + fail(path, "not a zip file (no end of central directory record)"); + } + + uint64_t num_entries = read_u16(tail, *eocd_pos + 10); + uint64_t central_dir_size = read_u32(tail, *eocd_pos + 12); + uint64_t central_dir_offset = read_u32(tail, *eocd_pos + 16); + + // zip64: the locator sits immediately before the classic record. + uint64_t eocd_abs = tail_offset + *eocd_pos; + if (eocd_abs >= ZIP64_EOCD_LOCATOR_SIZE) { + std::string locator = read_range( + f, path, eocd_abs - ZIP64_EOCD_LOCATOR_SIZE, ZIP64_EOCD_LOCATOR_SIZE); + if (read_u32(locator, 0) == ZIP64_EOCD_LOCATOR_SIGNATURE) { + uint64_t zip64_eocd_offset = read_u64(locator, 8); + std::string zip64_eocd = + read_range(f, path, zip64_eocd_offset, ZIP64_EOCD_SIZE); + if (read_u32(zip64_eocd, 0) != ZIP64_EOCD_SIGNATURE) { + fail(path, "bad zip64 end of central directory record"); + } + num_entries = read_u64(zip64_eocd, 32); + central_dir_size = read_u64(zip64_eocd, 40); + central_dir_offset = read_u64(zip64_eocd, 48); + } + } + + std::string cd = read_range(f, path, central_dir_offset, central_dir_size); + + std::map entries; + size_t pos = 0; + for (uint64_t i = 0; i < num_entries; i++) { + if (read_u32(cd, pos) != CENTRAL_DIR_HEADER_SIGNATURE) { + fail(path, fmt::format("bad central directory header for entry {}", i)); + } + ZipEntryLocation loc; + loc.compression_method = read_u16(cd, pos + 10); + loc.compressed_size = read_u32(cd, pos + 20); + loc.uncompressed_size = read_u32(cd, pos + 24); + uint16_t name_len = read_u16(cd, pos + 28); + uint16_t extra_len = read_u16(cd, pos + 30); + uint16_t comment_len = read_u16(cd, pos + 32); + loc.local_header_offset = read_u32(cd, pos + 42); + std::string name = cd.substr(pos + CENTRAL_DIR_HEADER_SIZE, name_len); + + // zip64 extended information: only the fields that overflowed are + // present, in this fixed order. + size_t extra_pos = pos + CENTRAL_DIR_HEADER_SIZE + name_len; + size_t extra_end = extra_pos + extra_len; + while (extra_pos + 4 <= extra_end) { + uint16_t id = read_u16(cd, extra_pos); + uint16_t size = read_u16(cd, extra_pos + 2); + if (id == ZIP64_EXTRA_FIELD_ID) { + size_t p = extra_pos + 4; + if (loc.uncompressed_size == 0xFFFFFFFF) { + loc.uncompressed_size = read_u64(cd, p); + p += 8; + } + if (loc.compressed_size == 0xFFFFFFFF) { + loc.compressed_size = read_u64(cd, p); + p += 8; + } + if (loc.local_header_offset == 0xFFFFFFFF) { + loc.local_header_offset = read_u64(cd, p); + p += 8; + } + } + extra_pos += 4 + size; + } + + entries.emplace(name, loc); + pos += CENTRAL_DIR_HEADER_SIZE + name_len + extra_len + comment_len; + } + return entries; +} + +std::string read_stored_zip_entry(std::string const &path, + std::string const &entry_name, + ZipEntryLocation const &location) { + if (location.compression_method != 0) { + fail(path, + fmt::format("entry {} is compressed (method {}); only stored " + "(uncompressed) entries are supported", + entry_name, + location.compression_method)); + } + std::ifstream f = open_binary(path); + std::string header = + read_range(f, path, location.local_header_offset, LOCAL_HEADER_SIZE); + if (read_u32(header, 0) != LOCAL_HEADER_SIGNATURE) { + fail(path, fmt::format("bad local header for entry {}", entry_name)); + } + uint16_t name_len = read_u16(header, 26); + uint16_t extra_len = read_u16(header, 28); + uint64_t data_offset = + location.local_header_offset + LOCAL_HEADER_SIZE + name_len + extra_len; + return read_range(f, path, data_offset, location.uncompressed_size); +} + +nlohmann::json read_pt2_model_json(std::string const &pt2_path, + std::string const &model_name) { + std::map entries = + read_zip_central_directory(pt2_path); + + auto strip_root = [](std::string const &name) -> std::optional { + size_t slash = name.find('/'); + if (slash == std::string::npos) { + return std::nullopt; + } + return name.substr(slash + 1); + }; + + std::string wanted = fmt::format("models/{}.json", model_name); + std::vector matches; + std::vector available_models; + for (auto const &[name, loc] : entries) { + std::optional rel = strip_root(name); + if (!rel.has_value()) { + continue; + } + if (*rel == wanted) { + matches.push_back(name); + } else if (rel->rfind("models/", 0) == 0 && + rel->find('/', 7) == std::string::npos) { + available_models.push_back(*rel); + } + if (*rel == "archive_format") { + std::string format = read_stored_zip_entry(pt2_path, name, loc); + if (format != "pt2") { + fail(pt2_path, + fmt::format("archive_format is \"{}\", expected \"pt2\"", format)); + } + } + } + + if (matches.empty()) { + fail(pt2_path, + fmt::format("no {} in archive (found: [{}]). Is this a " + "torch.export.save() file?", + wanted, + join(available_models))); + } + if (matches.size() > 1) { + fail(pt2_path, + fmt::format("multiple {} entries: [{}]", wanted, join(matches))); + } + + std::string contents = + read_stored_zip_entry(pt2_path, matches.at(0), entries.at(matches.at(0))); + try { + return nlohmann::json::parse(contents); + } catch (nlohmann::json::parse_error const &e) { + fail(pt2_path, + fmt::format("could not parse {}: {}", matches.at(0), e.what())); + } +} + +} // namespace FlexFlow diff --git a/bin/import-pt2/src/import-pt2/pt2_to_computation_graph.cc b/bin/import-pt2/src/import-pt2/pt2_to_computation_graph.cc new file mode 100644 index 0000000000..3c6dbb3e17 --- /dev/null +++ b/bin/import-pt2/src/import-pt2/pt2_to_computation_graph.cc @@ -0,0 +1,786 @@ +#include "import-pt2/pt2_to_computation_graph.h" +#include "op-attrs/tensor_dims.h" +#include "pcg/computation_graph_builder.h" +#include +#include +#include +#include +#include +#include +#include + +namespace FlexFlow { + +namespace { + +using json = nlohmann::json; +using Shape = std::vector; + +// torch/_export/serde/schema.py: ScalarType +DataType datatype_from_torch_scalar_type(int scalar_type, + std::string const &tensor_name) { + switch (scalar_type) { + case 4: + return DataType::INT32; + case 5: + return DataType::INT64; + case 6: + return DataType::HALF; + case 7: + return DataType::FLOAT; + case 8: + return DataType::DOUBLE; + case 12: + return DataType::BOOL; + default: + throw Pt2ImportError( + fmt::format("tensor {}: unsupported torch dtype (ScalarType {})", + tensor_name, + scalar_type)); + } +} + +std::string to_str(std::vector const &v) { + std::string result = "["; + for (size_t i = 0; i < v.size(); i++) { + result += (i == 0 ? "" : ", ") + std::to_string(v.at(i)); + } + return result + "]"; +} + +int64_t normalize_dim(int64_t dim, size_t rank) { + return dim < 0 ? dim + static_cast(rank) : dim; +} + +/** + * @brief The arguments of one serialized node, with typed accessors. + */ +struct NodeArgs { + std::string node_name; + std::string target; + std::map args; + + [[noreturn]] void fail(std::string const &msg) const { + throw Pt2ImportError( + fmt::format("node {} ({}): {}", this->node_name, this->target, msg)); + } + + bool has(std::string const &name) const { + auto it = this->args.find(name); + return it != this->args.end() && !it->second.contains("as_none"); + } + + json const &raw(std::string const &name) const { + auto it = this->args.find(name); + if (it == this->args.end()) { + this->fail(fmt::format("missing argument '{}'", name)); + } + return it->second; + } + + bool is_tensor(std::string const &name) const { + return this->has(name) && this->raw(name).contains("as_tensor"); + } + + std::string tensor(std::string const &name) const { + json const &a = this->raw(name); + if (!a.contains("as_tensor")) { + this->fail( + fmt::format("argument '{}' is not a tensor: {}", name, a.dump())); + } + return a.at("as_tensor").at("name").get(); + } + + std::optional optional_tensor(std::string const &name) const { + if (!this->has(name)) { + return std::nullopt; + } + return this->tensor(name); + } + + std::vector tensors(std::string const &name) const { + json const &a = this->raw(name); + if (!a.contains("as_tensors")) { + this->fail(fmt::format("argument '{}' is not a tensor list", name)); + } + std::vector result; + for (json const &t : a.at("as_tensors")) { + result.push_back(t.at("name").get()); + } + return result; + } + + int64_t get_int(std::string const &name, + std::optional default_value = std::nullopt) const { + if (!this->has(name)) { + if (default_value.has_value()) { + return *default_value; + } + this->fail(fmt::format("missing int argument '{}'", name)); + } + json const &a = this->raw(name); + if (!a.contains("as_int")) { + this->fail(fmt::format( + "argument '{}' is not a constant int: {}", name, a.dump())); + } + return a.at("as_int").get(); + } + + std::vector get_ints( + std::string const &name, + std::optional> default_value = std::nullopt) const { + if (!this->has(name)) { + if (default_value.has_value()) { + return *default_value; + } + this->fail(fmt::format("missing int list argument '{}'", name)); + } + json const &a = this->raw(name); + if (!a.contains("as_ints")) { + this->fail(fmt::format( + "argument '{}' is not a constant int list: {}", name, a.dump())); + } + return a.at("as_ints").get>(); + } + + double get_float(std::string const &name, + std::optional default_value = std::nullopt) const { + if (!this->has(name)) { + if (default_value.has_value()) { + return *default_value; + } + this->fail(fmt::format("missing float argument '{}'", name)); + } + json const &a = this->raw(name); + if (a.contains("as_float")) { + return a.at("as_float").get(); + } + if (a.contains("as_int")) { + return static_cast(a.at("as_int").get()); + } + this->fail(fmt::format( + "argument '{}' is not a constant number: {}", name, a.dump())); + } + + bool get_bool(std::string const &name, + std::optional default_value = std::nullopt) const { + if (!this->has(name)) { + if (default_value.has_value()) { + return *default_value; + } + this->fail(fmt::format("missing bool argument '{}'", name)); + } + json const &a = this->raw(name); + if (!a.contains("as_bool")) { + this->fail(fmt::format("argument '{}' is not a constant bool", name)); + } + return a.at("as_bool").get(); + } + + std::optional get_string(std::string const &name) const { + if (!this->has(name)) { + return std::nullopt; + } + json const &a = this->raw(name); + if (!a.contains("as_string")) { + this->fail(fmt::format("argument '{}' is not a constant string", name)); + } + return a.at("as_string").get(); + } + + // Expands 1-element int lists (e.g. stride=[2]) to 2 elements. + std::pair get_int_pair( + std::string const &name, + std::optional> default_value = std::nullopt) const { + std::vector v = this->get_ints(name, default_value); + if (v.size() == 1) { + return {v.at(0), v.at(0)}; + } + if (v.size() != 2) { + this->fail( + fmt::format("argument '{}' should have 1 or 2 elements, got {}", + name, + to_str(v))); + } + return {v.at(0), v.at(1)}; + } +}; + +class Pt2GraphImporter { +public: + explicit Pt2GraphImporter(json const &model) + : graph(model.at("graph_module").at("graph")), + signature(model.at("graph_module").at("signature")) {} + + ComputationGraph run() { + this->load_tensor_values(); + this->load_signature_inputs(); + for (json const &node : this->graph.at("nodes")) { + this->import_node(node); + } + for (json const &spec : this->signature.at("output_specs")) { + if (!spec.contains("user_output")) { + throw Pt2ImportError(fmt::format( + "unsupported output spec (only user outputs are supported, e.g. " + "no buffer mutations): {}", + spec.dump())); + } + json const &arg = spec.at("user_output").at("arg"); + if (!arg.contains("as_tensor")) { + throw Pt2ImportError( + fmt::format("unsupported non-tensor graph output: {}", arg.dump())); + } + std::string name = arg.at("as_tensor").at("name").get(); + if (this->tensors.count(name) == 0) { + throw Pt2ImportError(fmt::format( + "graph output {} was not produced by the import", name)); + } + } + return this->cgb.computation_graph; + } + +private: + json const &graph; + json const &signature; + ComputationGraphBuilder cgb; + std::map shapes; + std::map dtypes; + std::map requires_grad; + std::set constants; // parameters, buffers, lifted constants + std::map tensors; + + void load_tensor_values() { + for (auto const &[name, meta] : this->graph.at("tensor_values").items()) { + Shape shape; + for (json const &s : meta.at("sizes")) { + if (!s.contains("as_int")) { + throw Pt2ImportError(fmt::format( + "tensor {} has a symbolic (dynamic) size; only static shapes " + "are supported. Export without dynamic_shapes.", + name)); + } + shape.push_back(s.at("as_int").get()); + } + this->shapes.emplace(name, shape); + this->dtypes.emplace(name, meta.at("dtype").get()); + this->requires_grad.emplace(name, meta.value("requires_grad", false)); + } + } + + void load_signature_inputs() { + for (json const &spec : this->signature.at("input_specs")) { + auto const &[kind, body] = *spec.items().begin(); + if (kind == "user_input") { + json const &arg = body.at("arg"); + if (!arg.contains("as_tensor")) { + throw Pt2ImportError( + fmt::format("unsupported non-tensor user input: {}", arg.dump())); + } + std::string name = arg.at("as_tensor").at("name").get(); + this->create_input(name); + } else if (kind == "parameter" || kind == "buffer" || + kind == "tensor_constant") { + this->constants.insert(body.at("arg").at("name").get()); + } else { + throw Pt2ImportError( + fmt::format("unsupported input spec kind '{}'", kind)); + } + } + } + + void create_input(std::string const &name) { + TensorShape shape = TensorShape{ + TensorDims{FFOrdered{this->positive_dims(name)}}, + datatype_from_torch_scalar_type(this->dtypes.at(name), name), + }; + CreateGrad create_grad = + this->requires_grad.at(name) ? CreateGrad::YES : CreateGrad::NO; + this->tensors.emplace(name, + this->cgb.create_input(shape, create_grad, name)); + } + + Shape const &shape_of(std::string const &name) const { + auto it = this->shapes.find(name); + if (it == this->shapes.end()) { + throw Pt2ImportError(fmt::format("no tensor_values entry for {}", name)); + } + return it->second; + } + + std::vector positive_dims(std::string const &name) const { + Shape const &shape = this->shape_of(name); + if (shape.empty()) { + throw Pt2ImportError(fmt::format( + "tensor {} is 0-dimensional; FlexFlow tensors need at least 1 dim", + name)); + } + std::vector dims; + for (int64_t d : shape) { + if (d <= 0) { + throw Pt2ImportError(fmt::format( + "tensor {} has a non-positive dim: {}", name, to_str(shape))); + } + dims.push_back(positive_int{static_cast(d)}); + } + return dims; + } + + tensor_guid_t data_tensor(NodeArgs const &n, std::string const &name) const { + if (this->constants.count(name)) { + n.fail(fmt::format("parameter/buffer {} is used as a data input, which " + "is not supported for this op", + name)); + } + auto it = this->tensors.find(name); + if (it == this->tensors.end()) { + n.fail(fmt::format("input tensor {} was not produced by a supported op", + name)); + } + return it->second; + } + + tensor_guid_t data_arg(NodeArgs const &n, std::string const &arg) const { + return this->data_tensor(n, n.tensor(arg)); + } + + std::string const &constant_arg(NodeArgs const &n, + std::string const &name) const { + if (!this->constants.count(name)) { + n.fail(fmt::format("expected {} to be a parameter or buffer", name)); + } + return name; + } + + static positive_int + pos(NodeArgs const &n, int64_t v, std::string const &what) { + if (v <= 0) { + n.fail(fmt::format("{} must be positive, got {}", what, v)); + } + return positive_int{static_cast(v)}; + } + + static nonnegative_int + nonneg(NodeArgs const &n, int64_t v, std::string const &what) { + if (v < 0) { + n.fail(fmt::format("{} must be non-negative, got {}", what, v)); + } + return nonnegative_int{static_cast(v)}; + } + + static std::vector output_names(NodeArgs const &n, + json const &node) { + std::vector names; + for (json const &o : node.at("outputs")) { + if (o.contains("as_tensor")) { + names.push_back(o.at("as_tensor").at("name").get()); + } else if (o.contains("as_tensors")) { + for (json const &t : o.at("as_tensors")) { + names.push_back(t.at("name").get()); + } + } else if (o.contains("as_none")) { + continue; + } else { + n.fail(fmt::format("unsupported output kind: {}", o.dump())); + } + } + return names; + } + + void import_node(json const &node) { + NodeArgs n; + n.node_name = node.at("name").get(); + n.target = node.at("target").get(); + for (json const &input : node.at("inputs")) { + n.args.emplace(input.at("name").get(), input.at("arg")); + } + std::vector outs = output_names(n, node); + + std::vector> results = + this->translate(n, outs); + + if (results.size() != outs.size()) { + n.fail(fmt::format("importer produced {} outputs, node has {}", + results.size(), + outs.size())); + } + for (size_t i = 0; i < outs.size(); i++) { + if (!results.at(i).has_value()) { + continue; // unused auxiliary output (e.g. batch norm saved stats) + } + tensor_guid_t t = results.at(i).value(); + this->check_shape(n, outs.at(i), t); + this->tensors.emplace(outs.at(i), t); + } + } + + void check_shape(NodeArgs const &n, + std::string const &name, + tensor_guid_t const &t) const { + // Keep the TensorShape alive: ff_ordered() returns a reference into it. + TensorShape ff_tensor_shape = this->cgb.get_shape(t); + Shape ff_shape; + for (positive_int d : ff_ordered(ff_tensor_shape.dims)) { + ff_shape.push_back(d.int_from_positive_int()); + } + Shape const &torch_shape = this->shape_of(name); + if (ff_shape != torch_shape) { + n.fail(fmt::format("output {}: FlexFlow inferred shape {} but the .pt2 " + "records {}", + name, + to_str(ff_shape), + to_str(torch_shape))); + } + } + + // Size of each output along `dim`; used for chunk/split. + std::vector + output_sizes_along(NodeArgs const &n, + std::vector const &outs, + int64_t dim) const { + std::vector sizes; + for (std::string const &o : outs) { + sizes.push_back(pos(n, this->shape_of(o).at(dim), "split size")); + } + return sizes; + } + + using Results = std::vector>; + + Results translate(NodeArgs const &n, std::vector const &outs) { + std::string const prefix = "torch.ops.aten."; + if (n.target.rfind(prefix, 0) != 0) { + n.fail("unsupported op (not an ATen op)"); + } + std::string op = n.target.substr(prefix.size()); + std::optional name = n.node_name; + ComputationGraphBuilder &b = this->cgb; + + auto single = [](tensor_guid_t t) { return Results{t}; }; + + // ---- no-ops and pass-throughs + if (op == "_assert_tensor_metadata.default") { + return {}; + } + if (op == "detach.default" || op == "detach_.default" || + op == "clone.default" || op == "alias.default" || + op == "contiguous.default" || op == "lift_fresh_copy.default") { + return single(this->data_arg(n, "self")); + } + if (op == "dropout.default") { + if (n.get_bool("train")) { + n.fail("dropout in training mode is not supported; export the model " + "in eval() mode"); + } + return single(this->data_arg(n, "input")); + } + if (op == "to.dtype" || op == "_to_copy.default") { + std::string in = n.tensor("self"); + if (this->dtypes.at(in) != this->dtypes.at(outs.at(0))) { + n.fail("dtype conversion is not supported yet"); + } + return single(this->data_tensor(n, in)); + } + + // ---- convolution / normalization / linear + if (op == "conv2d.default") { + Shape const &w = + this->shape_of(this->constant_arg(n, n.tensor("weight"))); + if (w.size() != 4) { + n.fail(fmt::format("expected a 4D conv weight, got {}", to_str(w))); + } + auto [sh, sw] = n.get_int_pair("stride", std::vector{1}); + auto [ph, pw] = n.get_int_pair("padding", std::vector{0}); + auto [dh, dw] = n.get_int_pair("dilation", std::vector{1}); + if (dh != 1 || dw != 1) { + n.fail("dilated convolution is not supported"); + } + std::optional bias = n.optional_tensor("bias"); + if (bias.has_value()) { + this->constant_arg(n, *bias); + } + return single( + b.conv2d(this->data_arg(n, "input"), + /*outChannels=*/pos(n, w.at(0), "out_channels"), + /*kernelH=*/pos(n, w.at(2), "kernel_h"), + /*kernelW=*/pos(n, w.at(3), "kernel_w"), + /*strideH=*/pos(n, sh, "stride_h"), + /*strideW=*/pos(n, sw, "stride_w"), + /*paddingH=*/nonneg(n, ph, "padding_h"), + /*paddingW=*/nonneg(n, pw, "padding_w"), + /*activation=*/std::nullopt, + /*groups=*/pos(n, n.get_int("groups", 1), "groups"), + /*use_bias=*/bias.has_value(), + /*kernel_initializer=*/std::nullopt, + /*bias_initializer=*/std::nullopt, + /*kernel_regularizer=*/std::nullopt, + name)); + } + if (op == "batch_norm.default" || + op == "_native_batch_norm_legit_no_training.default") { + bool has_weight = n.has("weight"); + if (has_weight != n.has("bias")) { + n.fail("batch norm with only one of weight/bias is not supported"); + } + float eps = static_cast(n.get_float("eps")); + float momentum = static_cast(n.get_float("momentum")); + tensor_guid_t out = b.batch_norm(this->data_arg(n, "input"), + /*affine=*/has_weight, + /*activation=*/std::nullopt, + eps, + momentum, + name); + Results r{out}; + // The decomposed form also returns (save_mean, save_rstd), unused at + // inference. + while (r.size() < outs.size()) { + r.push_back(std::nullopt); + } + return r; + } + if (op == "linear.default") { + Shape const &w = + this->shape_of(this->constant_arg(n, n.tensor("weight"))); + if (w.size() != 2) { + n.fail(fmt::format("expected a 2D linear weight, got {}", to_str(w))); + } + std::optional bias = n.optional_tensor("bias"); + if (bias.has_value()) { + this->constant_arg(n, *bias); + } + std::string in = n.tensor("input"); + return single( + b.dense(this->data_tensor(n, in), + /*outDim=*/pos(n, w.at(0), "out_features"), + /*activation=*/std::nullopt, + /*use_bias=*/bias.has_value(), + /*data_type=*/ + datatype_from_torch_scalar_type(this->dtypes.at(in), in), + /*projection_initializer=*/std::nullopt, + /*bias_initializer=*/std::nullopt, + name)); + } + + // ---- element-wise unary + if (op == "silu.default" || op == "silu_.default") { + return single(b.silu(this->data_arg(n, "self"), name)); + } + if (op == "relu.default" || op == "relu_.default") { + return single(b.relu(this->data_arg(n, "self"), name)); + } + if (op == "sigmoid.default" || op == "sigmoid_.default") { + return single(b.sigmoid(this->data_arg(n, "self"), name)); + } + if (op == "tanh.default" || op == "tanh_.default") { + return single(b.tanh(this->data_arg(n, "self"), name)); + } + if (op == "gelu.default") { + std::optional approx = n.get_string("approximate"); + if (approx.has_value() && *approx != "none") { + n.fail(fmt::format("gelu approximate='{}' is not supported", *approx)); + } + return single(b.gelu(this->data_arg(n, "self"), name)); + } + if (op == "exp.default") { + return single(b.exp(this->data_arg(n, "self"), name)); + } + if (op == "rsqrt.default") { + return single(b.rsqrt(this->data_arg(n, "self"), name)); + } + + // ---- element-wise binary (tensor-tensor or tensor-scalar) + if (op == "add.Tensor" || op == "sub.Tensor") { + if (n.has("alpha") && n.get_float("alpha") != 1.0) { + n.fail("alpha != 1 is not supported"); + } + } + if (op == "add.Tensor" || op == "sub.Tensor" || op == "mul.Tensor" || + op == "div.Tensor") { + tensor_guid_t self = this->data_arg(n, "self"); + if (n.is_tensor("other")) { + tensor_guid_t other = this->data_arg(n, "other"); + if (op == "add.Tensor") { + return single(b.add(self, other, name)); + } else if (op == "sub.Tensor") { + return single(b.subtract(self, other, name)); + } else if (op == "mul.Tensor") { + return single(b.multiply(self, other, name)); + } else { + return single(b.divide(self, other, name)); + } + } + float s = static_cast(n.get_float("other")); + if (op == "add.Tensor") { + return single(b.scalar_add(self, s, name)); + } else if (op == "sub.Tensor") { + return single(b.scalar_sub(self, s, name)); + } else if (op == "mul.Tensor") { + return single(b.scalar_multiply(self, s, name)); + } else { + return single(b.scalar_truediv(self, s, name)); + } + } + + // ---- shape ops + if (op == "cat.default") { + std::vector ins = n.tensors("tensors"); + std::vector ts; + for (std::string const &t : ins) { + ts.push_back(this->data_tensor(n, t)); + } + int64_t dim = n.get_int("dim", 0); + return single( + b.concat(ts, relative_ff_dim_t{static_cast(dim)}, name)); + } + if (op == "chunk.default" || op == "split.Tensor" || + op == "split_with_sizes.default") { + std::string in = n.tensor("self"); + int64_t dim = + normalize_dim(n.get_int("dim", 0), this->shape_of(in).size()); + tensor_guid_t input = this->data_tensor(n, in); + if (outs.size() == 1) { + return single(input); + } + std::vector parts = + b.split(input, + this->output_sizes_along(n, outs, dim), + relative_ff_dim_t{static_cast(dim)}, + name); + return Results(parts.begin(), parts.end()); + } + if (op == "view.default" || op == "reshape.default" || + op == "_unsafe_view.default" || op == "flatten.using_ints" || + op == "unsqueeze.default" || op == "squeeze.dim" || + op == "squeeze.dims") { + // The target shape (with -1 resolved) is the recorded output shape. + return single(b.reshape( + this->data_arg(n, "self"), this->positive_dims(outs.at(0)), name)); + } + if (op == "transpose.int" || op == "permute.default") { + std::string in = n.tensor("self"); + size_t rank = this->shape_of(in).size(); + std::vector perm(rank); + for (size_t i = 0; i < rank; i++) { + perm.at(i) = static_cast(i); + } + if (op == "transpose.int") { + int64_t d0 = normalize_dim(n.get_int("dim0"), rank); + int64_t d1 = normalize_dim(n.get_int("dim1"), rank); + std::swap(perm.at(d0), perm.at(d1)); + } else { + std::vector dims = n.get_ints("dims"); + if (dims.size() != rank) { + n.fail("permute dims has the wrong length"); + } + for (size_t i = 0; i < rank; i++) { + perm.at(i) = normalize_dim(dims.at(i), rank); + } + } + std::vector ff_perm; + for (int64_t p : perm) { + ff_perm.push_back(nonneg(n, p, "permutation entry")); + } + return single(b.transpose(this->data_tensor(n, in), ff_perm, name)); + } + + // ---- pooling / resampling + if (op == "max_pool2d.default") { + auto [kh, kw] = n.get_int_pair("kernel_size"); + // PyTorch: an empty stride means stride = kernel_size. + auto [sh, sw] = n.get_ints("stride", std::vector{}).empty() + ? std::pair{kh, kw} + : n.get_int_pair("stride"); + auto [ph, pw] = n.get_int_pair("padding", std::vector{0}); + auto [dh, dw] = n.get_int_pair("dilation", std::vector{1}); + if (dh != 1 || dw != 1) { + n.fail("dilated max_pool2d is not supported"); + } + if (n.get_bool("ceil_mode", false)) { + n.fail("max_pool2d ceil_mode=True is not supported"); + } + return single(b.pool2d(this->data_arg(n, "self"), + pos(n, kh, "kernel_h"), + pos(n, kw, "kernel_w"), + pos(n, sh, "stride_h"), + pos(n, sw, "stride_w"), + nonneg(n, ph, "padding_h"), + nonneg(n, pw, "padding_w"), + PoolOp::MAX, + /*activation=*/std::nullopt, + name)); + } + if (op == "adaptive_avg_pool2d.default") { + auto [oh, ow] = n.get_int_pair("output_size"); + return single(b.adaptive_pool2d(this->data_arg(n, "self"), + pos(n, oh, "output_h"), + pos(n, ow, "output_w"), + PoolOp::AVG, + /*activation=*/std::nullopt, + name)); + } + if (op == "upsample_nearest2d.vec" || op == "upsample_nearest2d.default") { + std::string in = n.tensor("input"); + Shape const &is = this->shape_of(in); + Shape const &os = this->shape_of(outs.at(0)); + if (is.size() != 4 || os.size() != 4 || os.at(2) % is.at(2) != 0 || + os.at(3) % is.at(3) != 0 || + os.at(2) / is.at(2) != os.at(3) / is.at(3) || + os.at(2) / is.at(2) < 2) { + n.fail(fmt::format("only equal integer scale factors >= 2 are " + "supported (input {}, output {})", + to_str(is), + to_str(os))); + } + int factor = static_cast(os.at(2) / is.at(2)); + return single(b.upsample(this->data_tensor(n, in), + int_ge_two{factor}, + UpsampleMode::NEAREST, + name)); + } + + // ---- matmul / softmax + if (op == "matmul.default" || op == "bmm.default") { + std::string lhs = n.tensor("self"); + std::string rhs = + n.is_tensor("other") ? n.tensor("other") : n.tensor("mat2"); + Shape const &ls = this->shape_of(lhs); + Shape const &rs = this->shape_of(rhs); + if (ls.size() < 3 || ls.size() != rs.size() || + !std::equal(ls.begin(), ls.end() - 2, rs.begin())) { + n.fail(fmt::format("only batched matmul with equal batch dims is " + "supported (lhs {}, rhs {})", + to_str(ls), + to_str(rs))); + } + return single(b.batch_matmul( + this->data_tensor(n, lhs), this->data_tensor(n, rhs), name)); + } + if (op == "softmax.int" || op == "_softmax.default") { + if (n.has("half_to_float") && n.get_bool("half_to_float")) { + n.fail("half_to_float softmax is not supported"); + } + if (n.has("dtype")) { + n.fail("softmax with dtype= is not supported"); + } + return single( + b.softmax(this->data_arg(n, "self"), + relative_ff_dim_t{static_cast(n.get_int("dim"))}, + name)); + } + + n.fail("unsupported op"); + } +}; + +} // namespace + +ComputationGraph + computation_graph_from_pt2_model_json(nlohmann::json const &model) { + try { + return Pt2GraphImporter{model}.run(); + } catch (json::exception const &e) { + throw Pt2ImportError( + fmt::format("unexpected .pt2 model.json structure: {}", e.what())); + } +} + +} // namespace FlexFlow diff --git a/bin/index.dox b/bin/index.dox index 0ac3e9f0b2..bc241d3c41 100644 --- a/bin/index.dox +++ b/bin/index.dox @@ -7,6 +7,7 @@ This directory contains command-line interfaces for %FlexFlow %Train and associa - \subpage compile-model "": \copybrief compile-model - \subpage export-model-arch "": \copybrief export-model-arch +- \subpage import-pt2 "": \copybrief import-pt2 - \subpage protobuf-to-json "": \copybrief protobuf-to-json - \subpage run-model "": \copybrief run-model - \subpage sp-ization-benchmarking "": \copybrief sp-ization-benchmarking