Introduce a new NUMBER_VARINT_TUPLE encoding

DEPENDENCIES

@@ -1,4 +1,4 @@
 vendorpull https://github.com/sourcemeta/vendorpull 1dcbac42809cf87cb5b045106b863e17ad84ba02
-core https://github.com/sourcemeta/core 614dd74eb014f50b4b83d81db3a39de43ffe83d0
-blaze https://github.com/sourcemeta/blaze 87b62693c610f7a07efb63d1a8baf22e99571aa9
+core https://github.com/sourcemeta/core af2b932ef6284c880f47012749b78bb75526f7a4
+blaze https://github.com/sourcemeta/blaze a5a019aa49139e7cd21accd52ad2c65f66afff16
 bootstrap https://github.com/twbs/bootstrap 1a6fdfae6be09b09eaced8f0e442ca6f7680a61e

src/compiler/mapper/number_arbitrary.h

@@ -25,7 +25,7 @@ class NumberArbitrary final : public sourcemeta::blaze::SchemaTransformRule {
       sourcemeta::core::JSON &schema,
       [[maybe_unused]] const sourcemeta::blaze::SchemaTransformRule::Result
           &result) const -> void override {
-    make_encoding(schema, "DOUBLE_VARINT_TUPLE",
+    make_encoding(schema, "NUMBER_VARINT_TUPLE",
                   sourcemeta::core::JSON::make_object());
   }
 };

src/runtime/decoder_any.cc

@@ -65,6 +65,8 @@ auto Decoder::ANY_PACKED_TYPE_TAG_BYTE_PREFIX(
         return sourcemeta::core::JSON{true};
       case SUBTYPE_NUMBER:
         return this->DOUBLE_VARINT_TUPLE({});
+      case SUBTYPE_DECIMAL:
+        return this->NUMBER_VARINT_TUPLE({});
       case SUBTYPE_POSITIVE_REAL_INTEGER_BYTE:
         return sourcemeta::core::JSON{static_cast<double>(this->get_byte())};
       case SUBTYPE_POSITIVE_INTEGER:

src/runtime/decoder_common.cc

@@ -36,6 +36,7 @@ auto Decoder::read(const Encoding &encoding) -> sourcemeta::core::JSON {
     HANDLE_DECODING(19, ROOF_TYPED_ARRAY)
     HANDLE_DECODING(20, FIXED_TYPED_ARBITRARY_OBJECT)
     HANDLE_DECODING(21, VARINT_TYPED_ARBITRARY_OBJECT)
+    HANDLE_DECODING(22, NUMBER_VARINT_TUPLE)
 #undef HANDLE_DECODING
     default:
       // We should never get here. If so, it is definitely a bug

src/runtime/decoder_number.cc

@@ -1,5 +1,8 @@
 #include <sourcemeta/jsonbinpack/runtime_decoder.h>
 
+#include <sourcemeta/core/numeric.h>
+
+#include <cmath>   // std::isfinite
 #include <cstdint> // std::int64_t, std::uint64_t
 
 #if defined(__GNUC__) && !defined(__clang__)
@@ -23,4 +26,38 @@ auto Decoder::DOUBLE_VARINT_TUPLE(
   return sourcemeta::core::JSON{static_cast<double>(digits) / divisor};
 }
 
+auto Decoder::NUMBER_VARINT_TUPLE(
+    [[maybe_unused]] const struct NUMBER_VARINT_TUPLE &options)
+    -> sourcemeta::core::JSON {
+#ifdef __clang__
+#pragma clang fp reciprocal(off)
+#endif
+  const std::int64_t coefficient{this->get_varint_zigzag()};
+  const std::uint64_t tag{this->get_varint()};
+  const std::int64_t exponent{
+      sourcemeta::core::zigzag_decode<std::uint64_t>(tag >> 1)};
+
+  if ((tag & 1) != 0) {
+    return sourcemeta::core::JSON{
+        sourcemeta::core::Decimal{coefficient}.scale_by(
+            sourcemeta::core::Decimal{exponent})};
+  }
+
+  // A scale that has grown past the range of the type can only keep growing,
+  // so the loop stops there rather than counting out an exponent that a
+  // corrupt stream may have made arbitrarily large
+  const std::uint64_t shifts{exponent < 0
+                                 ? static_cast<std::uint64_t>(-exponent)
+                                 : static_cast<std::uint64_t>(exponent)};
+  double scale{1.0};
+  for (std::uint64_t index = 0; index < shifts && std::isfinite(scale);
+       ++index) {
+    scale *= 10.0;
+  }
+
+  return sourcemeta::core::JSON{exponent < 0
+                                    ? static_cast<double>(coefficient) / scale
+                                    : static_cast<double>(coefficient) * scale};
+}
+
 } // namespace sourcemeta::jsonbinpack

src/runtime/encoder_any.cc

@@ -114,6 +114,10 @@ auto Encoder::ANY_PACKED_TYPE_TAG_BYTE_PREFIX(
                      static_cast<std::uint8_t>(subtype << TYPE_SIZE));
       this->put_varint(absolute);
     }
+  } else if (document.is_decimal()) {
+    this->put_byte(TYPE_OTHER |
+                   static_cast<std::uint8_t>(SUBTYPE_DECIMAL << TYPE_SIZE));
+    this->NUMBER_VARINT_TUPLE(document, {});
   } else if (document.is_string()) {
     const sourcemeta::core::JSON::String &value{document.to_string()};
     const auto size{document.byte_size()};

src/runtime/encoder_common.cc

@@ -38,6 +38,7 @@ auto Encoder::write(const sourcemeta::core::JSON &document,
     HANDLE_ENCODING(19, ROOF_TYPED_ARRAY)
     HANDLE_ENCODING(20, FIXED_TYPED_ARBITRARY_OBJECT)
     HANDLE_ENCODING(21, VARINT_TYPED_ARBITRARY_OBJECT)
+    HANDLE_ENCODING(22, NUMBER_VARINT_TUPLE)
 #undef HANDLE_ENCODING
     default:
       // We should never get here. If so, it is definitely a bug

src/runtime/encoder_number.cc

@@ -2,8 +2,79 @@
 
 #include <sourcemeta/core/numeric.h>
 
-#include <cassert> // assert
-#include <cstdint> // std::uint64_t
+#include <cassert>      // assert
+#include <charconv>     // std::from_chars
+#include <cstddef>      // std::size_t
+#include <cstdint>      // std::int64_t, std::uint64_t
+#include <limits>       // std::numeric_limits
+#include <string_view>  // std::string_view
+#include <system_error> // std::errc
+
+namespace {
+
+#ifndef NDEBUG
+// The widest coefficient that still fits in a signed 64-bit integer
+constexpr std::size_t MAXIMUM_COEFFICIENT_DIGITS{19};
+#endif
+
+// The coefficient and the base-10 exponent of an arbitrary precision decimal.
+// Core keeps both of them private and only spells them out through the
+// scientific form, as `[-]D[.DDDD]e(+|-)ADJUSTED`, whose exponent is adjusted
+// to count from the leading digit rather than from the last one
+auto decimal_coefficient(const sourcemeta::core::Decimal &value,
+                         std::int64_t &exponent) -> std::int64_t {
+  const auto scientific{value.reduce().to_scientific_string()};
+  const std::string_view view{scientific};
+  const auto marker{view.find('e')};
+  assert(marker != std::string_view::npos);
+
+  auto mantissa{view.substr(0, marker)};
+  const auto negative{mantissa.front() == '-'};
+  if (negative) {
+    mantissa.remove_prefix(1);
+  }
+
+  // Every digit past the leading one sits after a decimal mark that the
+  // adjusted exponent already accounts for
+  const auto point{mantissa.find('.')};
+  const auto leading{mantissa.substr(
+      0, point == std::string_view::npos ? mantissa.size() : point)};
+  const auto trailing{point == std::string_view::npos
+                          ? std::string_view{}
+                          : mantissa.substr(point + 1)};
+
+  const auto digits{leading.size() + trailing.size()};
+  assert(digits >= 1);
+  assert(digits <= MAXIMUM_COEFFICIENT_DIGITS);
+
+  std::uint64_t magnitude{0};
+  for (const auto digit : leading) {
+    magnitude = (magnitude * 10) + static_cast<std::uint64_t>(digit - '0');
+  }
+  for (const auto digit : trailing) {
+    magnitude = (magnitude * 10) + static_cast<std::uint64_t>(digit - '0');
+  }
+
+  assert(magnitude <=
+         static_cast<std::uint64_t>(std::numeric_limits<std::int64_t>::max()));
+
+  auto adjusted_exponent{view.substr(marker + 1)};
+  if (adjusted_exponent.front() == '+') {
+    adjusted_exponent.remove_prefix(1);
+  }
+
+  std::int64_t adjusted{0};
+  [[maybe_unused]] const auto conversion{std::from_chars(
+      adjusted_exponent.data(),
+      adjusted_exponent.data() + adjusted_exponent.size(), adjusted)};
+  assert(conversion.ec == std::errc{});
+
+  exponent = adjusted - static_cast<std::int64_t>(digits - 1);
+  const auto coefficient{static_cast<std::int64_t>(magnitude)};
+  return negative ? -coefficient : coefficient;
+}
+
+} // namespace
 
 namespace sourcemeta::jsonbinpack {
 
@@ -19,4 +90,26 @@ auto Encoder::DOUBLE_VARINT_TUPLE(
   this->put_varint(point_position);
 }
 
+auto Encoder::NUMBER_VARINT_TUPLE(
+    const sourcemeta::core::JSON &document,
+    [[maybe_unused]] const struct NUMBER_VARINT_TUPLE &options) -> void {
+  assert(document.is_real() || document.is_decimal());
+  const auto decimal{document.is_decimal()};
+  std::int64_t coefficient{0};
+  std::int64_t exponent{0};
+
+  if (decimal) {
+    coefficient = decimal_coefficient(document.to_decimal(), exponent);
+  } else {
+    std::uint64_t point_position;
+    coefficient = sourcemeta::core::real_digits<std::int64_t>(
+        document.to_real(), point_position);
+    exponent = -static_cast<std::int64_t>(point_position);
+  }
+
+  this->put_varint_zigzag(coefficient);
+  this->put_varint((sourcemeta::core::zigzag_encode(exponent) << 1) |
+                   static_cast<std::uint64_t>(decimal));
+}
+
 } // namespace sourcemeta::jsonbinpack

src/runtime/include/sourcemeta/jsonbinpack/runtime_decoder.h

@@ -34,6 +34,7 @@ class SOURCEMETA_JSONBINPACK_RUNTIME_EXPORT Decoder : private InputStream {
 
   // Number
   DECLARE_ENCODING(DOUBLE_VARINT_TUPLE)
+  DECLARE_ENCODING(NUMBER_VARINT_TUPLE)
 
   // Any
   DECLARE_ENCODING(BYTE_CHOICE_INDEX)

src/runtime/include/sourcemeta/jsonbinpack/runtime_encoder.h

@@ -36,6 +36,7 @@ class SOURCEMETA_JSONBINPACK_RUNTIME_EXPORT Encoder : private OutputStream {
 
   // Number
   DECLARE_ENCODING(DOUBLE_VARINT_TUPLE)
+  DECLARE_ENCODING(NUMBER_VARINT_TUPLE)
 
   // Any
   DECLARE_ENCODING(BYTE_CHOICE_INDEX)

src/runtime/include/sourcemeta/jsonbinpack/runtime_encoding.h

@@ -23,6 +23,7 @@ struct FLOOR_MULTIPLE_ENUM_VARINT;
 struct ROOF_MULTIPLE_MIRROR_ENUM_VARINT;
 struct ARBITRARY_MULTIPLE_ZIGZAG_VARINT;
 struct DOUBLE_VARINT_TUPLE;
+struct NUMBER_VARINT_TUPLE;
 struct BYTE_CHOICE_INDEX;
 struct LARGE_CHOICE_INDEX;
 struct TOP_LEVEL_BYTE_CHOICE_INDEX;
@@ -55,7 +56,8 @@ using Encoding = std::variant<
     BOUNDED_8BIT_PREFIX_UTF8_STRING_SHARED, RFC3339_DATE_INTEGER_TRIPLET,
     PREFIX_VARINT_LENGTH_STRING_SHARED, FIXED_TYPED_ARRAY,
     BOUNDED_8BITS_TYPED_ARRAY, FLOOR_TYPED_ARRAY, ROOF_TYPED_ARRAY,
-    FIXED_TYPED_ARBITRARY_OBJECT, VARINT_TYPED_ARBITRARY_OBJECT>;
+    FIXED_TYPED_ARBITRARY_OBJECT, VARINT_TYPED_ARBITRARY_OBJECT,
+    NUMBER_VARINT_TUPLE>;
 
 // The encoding names are the ones the JSON BinPack specification
 // defines, so they keep their casing rather than following the C++
@@ -265,6 +267,57 @@ struct ARBITRARY_MULTIPLE_ZIGZAG_VARINT {
 // clang-format on
 struct DOUBLE_VARINT_TUPLE {};
 
+// clang-format off
+/// @brief The encoding consists of a sequence of two integers. The first is
+/// the signed coefficient of the number as a ZigZag-encoded Base-128 64-bit
+/// Little Endian variable-length unsigned integer. The second is the base-10
+/// exponent that the coefficient is scaled by, ZigZag-encoded and shifted left
+/// by one bit, whose least significant bit is set when the number is an
+/// arbitrary-precision decimal rather than a double, as a Base-128 64-bit
+/// Little Endian variable-length unsigned integer.
+///
+/// Unlike DOUBLE_VARINT_TUPLE, this encoding covers both of the numeric
+/// representations that a JSON document may hold, and the schema-driven mode
+/// relies on that: a schema that permits a number cannot tell which of the two
+/// an instance will carry, so the distinction travels with the data. The
+/// exponent is signed so that magnitudes a decimal mark cannot reach, such as
+/// 1e300, remain within the encoding.
+///
+/// ### Options
+///
+/// None
+///
+/// ### Conditions
+///
+/// The coefficient must fit in a signed 64-bit integer, which holds every
+/// number of up to 19 significant digits
+///
+/// ### Examples
+///
+/// Given the decimal input value 2.1, the encoding results in the
+/// variable-length integer 42 (the ZigZag encoding of 21) followed by the
+/// variable-length unsigned integer 3 (the ZigZag encoding of the exponent -1,
+/// shifted left by one bit, with the decimal bit set).
+///
+/// ```
+/// +------+------+
+/// | 0x2a | 0x03 |
+/// +------+------+
+/// ```
+///
+/// Given the double input value 3.14, the encoding results in the
+/// variable-length integer 628 (the ZigZag encoding of 314) followed by the
+/// variable-length unsigned integer 6 (the ZigZag encoding of the exponent -2,
+/// shifted left by one bit, with the decimal bit clear).
+///
+/// ```
+/// +------+------+------+
+/// | 0xf4 | 0x04 | 0x06 |
+/// +------+------+------+
+/// ```
+// clang-format on
+struct NUMBER_VARINT_TUPLE {};
+
 /// @}
 
 /// @ingroup runtime
@@ -445,6 +498,7 @@ constexpr std::uint8_t SUBTYPE_LONG_STRING_BASE_EXPONENT_7 = 0b00000111;
 constexpr std::uint8_t SUBTYPE_LONG_STRING_BASE_EXPONENT_8 = 0b00001000;
 constexpr std::uint8_t SUBTYPE_LONG_STRING_BASE_EXPONENT_9 = 0b00001001;
 constexpr std::uint8_t SUBTYPE_LONG_STRING_BASE_EXPONENT_10 = 0b00001010;
+constexpr std::uint8_t SUBTYPE_DECIMAL = 0b00001011;
 
 static_assert(SUBTYPE_FALSE <= sourcemeta::core::uint_max<SUBTYPE_SIZE>);
 static_assert(SUBTYPE_TRUE <= sourcemeta::core::uint_max<SUBTYPE_SIZE>);
@@ -464,6 +518,7 @@ static_assert(SUBTYPE_LONG_STRING_BASE_EXPONENT_9 <=
               sourcemeta::core::uint_max<SUBTYPE_SIZE>);
 static_assert(SUBTYPE_LONG_STRING_BASE_EXPONENT_10 <=
               sourcemeta::core::uint_max<SUBTYPE_SIZE>);
+static_assert(SUBTYPE_DECIMAL <= sourcemeta::core::uint_max<SUBTYPE_SIZE>);
 
 // Note that the binary values actually match the declared exponents
 static_assert(SUBTYPE_LONG_STRING_BASE_EXPONENT_7 == 7);

src/runtime/loader.cc

@@ -29,6 +29,7 @@ auto load(const sourcemeta::core::JSON &input) -> Encoding {
   PARSE_ENCODING(v1, ARBITRARY_MULTIPLE_ZIGZAG_VARINT)
   // Numbers
   PARSE_ENCODING(v1, DOUBLE_VARINT_TUPLE)
+  PARSE_ENCODING(v1, NUMBER_VARINT_TUPLE)
   // Any
   PARSE_ENCODING(v1, BYTE_CHOICE_INDEX)
   PARSE_ENCODING(v1, LARGE_CHOICE_INDEX)

src/runtime/loader_v1_number.h

@@ -16,6 +16,11 @@ auto DOUBLE_VARINT_TUPLE([[maybe_unused]] const sourcemeta::core::JSON &options)
   return sourcemeta::jsonbinpack::DOUBLE_VARINT_TUPLE{};
 }
 
+auto NUMBER_VARINT_TUPLE([[maybe_unused]] const sourcemeta::core::JSON &options)
+    -> Encoding {
+  return sourcemeta::jsonbinpack::NUMBER_VARINT_TUPLE{};
+}
+
 // NOLINTEND(readability-identifier-naming)
 } // namespace sourcemeta::jsonbinpack::v1
 

test/compiler/2020_12_compiler_number_test.cc

@@ -13,7 +13,7 @@ TEST(arbitrary) {
 
   const auto expected = sourcemeta::core::parse_json(R"JSON({
     "$schema": "tag:sourcemeta.com,2024:jsonbinpack/encoding/v1",
-    "binpackEncoding": "DOUBLE_VARINT_TUPLE",
+    "binpackEncoding": "NUMBER_VARINT_TUPLE",
     "binpackOptions": {}
   })JSON");
 

test/e2e/circlecimatrix/schema-less/output.bin

@@ -1,2 +1,2 @@
-version/*

+version_*
 workflowstestjobsm1matrixparametersa$%

test/runtime/decode_any_test.cc

@@ -1,5 +1,6 @@
 #include <sourcemeta/core/io.h>
 #include <sourcemeta/core/json.h>
+#include <sourcemeta/core/numeric.h>
 #include <sourcemeta/core/test.h>
 #include <sourcemeta/jsonbinpack/runtime.h>
 
@@ -1182,3 +1183,23 @@ TEST(ANY_PACKED_TYPE_TAG_BYTE_PREFIX__object_62_xs_shared) {
   EXPECT_TRUE(baz.is_string());
   EXPECT_EQ(baz.to_string(), std::string(62, 'x'));
 }
+
+TEST(ANY_PACKED_TYPE_TAG_BYTE_PREFIX__decimal_2_point_1) {
+  using namespace sourcemeta::jsonbinpack;
+  sourcemeta::core::InputByteStream stream{0x5f, 0x2a, 0x03};
+  Decoder decoder{stream};
+  const auto result = decoder.ANY_PACKED_TYPE_TAG_BYTE_PREFIX({});
+  EXPECT_TRUE(result.is_decimal());
+  const sourcemeta::core::JSON expected{sourcemeta::core::Decimal{"2.1"}};
+  EXPECT_EQ(result, expected);
+}
+
+TEST(ANY_PACKED_TYPE_TAG_BYTE_PREFIX__decimal_minus_122_point_08) {
+  using namespace sourcemeta::jsonbinpack;
+  sourcemeta::core::InputByteStream stream{0x5f, 0xdf, 0xbe, 0x01, 0x07};
+  Decoder decoder{stream};
+  const auto result = decoder.ANY_PACKED_TYPE_TAG_BYTE_PREFIX({});
+  EXPECT_TRUE(result.is_decimal());
+  const sourcemeta::core::JSON expected{sourcemeta::core::Decimal{"-122.08"}};
+  EXPECT_EQ(result, expected);
+}