JSON: Add opt-in comments and trailing comma flags

ext/json/json.c

@@ -351,8 +351,11 @@ PHP_FUNCTION(json_validate)
 	ZEND_PARSE_PARAMETERS_END();
 
 
-	if ((options != 0) && (options != PHP_JSON_INVALID_UTF8_IGNORE)) {
-		zend_argument_value_error(3, "must be a valid flag (allowed flags: JSON_INVALID_UTF8_IGNORE)");
+	if ((options & ~(PHP_JSON_INVALID_UTF8_IGNORE
+			| PHP_JSON_ALLOW_COMMENTS
+			| PHP_JSON_ALLOW_TRAILING_COMMAS)) != 0) {
+		zend_argument_value_error(3, "must be a valid flag (allowed flags: "
+			"JSON_INVALID_UTF8_IGNORE, JSON_ALLOW_COMMENTS, JSON_ALLOW_TRAILING_COMMAS)");
 		RETURN_THROWS();
 	}
 

ext/json/json.stub.php

@@ -90,6 +90,17 @@
  */
 const JSON_THROW_ON_ERROR = UNKNOWN;
 
+/**
+ * @var int
+ * @cvalue PHP_JSON_ALLOW_COMMENTS
+ */
+const JSON_ALLOW_COMMENTS = UNKNOWN;
+/**
+ * @var int
+ * @cvalue PHP_JSON_ALLOW_TRAILING_COMMAS
+ */
+const JSON_ALLOW_TRAILING_COMMAS = UNKNOWN;
+
 /**
  * @var int
  * @cvalue PHP_JSON_ERROR_NONE

ext/json/json_parser.y

@@ -119,6 +119,17 @@ members:
 				}
 			}
 	|	member
+	|	member ','
+			{
+				if (!(parser->scanner.options & PHP_JSON_ALLOW_TRAILING_COMMAS)) {
+					if (!parser->scanner.errcode) {
+						parser->scanner.errcode = PHP_JSON_ERROR_SYNTAX;
+					}
+					zval_ptr_dtor_nogc(&$1);
+					YYERROR;
+				}
+				ZVAL_COPY_VALUE(&$$, &$1);
+			}
 ;
 
 member:
@@ -175,6 +186,17 @@ elements:
 				}
 			}
 	|	element
+	|	element ','
+			{
+				if (!(parser->scanner.options & PHP_JSON_ALLOW_TRAILING_COMMAS)) {
+					if (!parser->scanner.errcode) {
+						parser->scanner.errcode = PHP_JSON_ERROR_SYNTAX;
+					}
+					zval_ptr_dtor_nogc(&$1);
+					YYERROR;
+				}
+				ZVAL_COPY_VALUE(&$$, &$1);
+			}
 ;
 
 element:

ext/json/json_scanner.re

@@ -102,6 +102,11 @@ void php_json_scanner_init(php_json_scanner *s, const char *str, size_t str_len,
 
 int php_json_scan(php_json_scanner *s)
 {
+	/* Comments are fully consumed before returning, so their opener location can remain local. */
+	php_json_ctype *comment_token = s->cursor;
+	php_json_ctype *comment_line_start = s->line_start;
+	uint64_t comment_line = s->line;
+
 	ZVAL_NULL(&s->value);
 
 std:
@@ -203,6 +208,23 @@ std:
 		goto std;
 	}
 	<JS>WS                   { goto std; }
+	<JS>"//"                 {
+		if (!(s->options & PHP_JSON_ALLOW_COMMENTS)) {
+			s->errcode = PHP_JSON_ERROR_SYNTAX;
+			return PHP_JSON_T_ERROR;
+		}
+		PHP_JSON_CONDITION_SET_AND_GOTO(COMMENT_LINE);
+	}
+	<JS>"/*"                 {
+		if (!(s->options & PHP_JSON_ALLOW_COMMENTS)) {
+			s->errcode = PHP_JSON_ERROR_SYNTAX;
+			return PHP_JSON_T_ERROR;
+		}
+		comment_token = s->token;
+		comment_line_start = s->line_start;
+		comment_line = s->line;
+		PHP_JSON_CONDITION_SET_AND_GOTO(COMMENT_BLOCK);
+	}
 	<JS>EOI                  {
 		if (s->limit < s->cursor) {
 			return PHP_JSON_T_EOI;
@@ -230,6 +252,68 @@ std:
 		s->errcode = PHP_JSON_ERROR_UTF8;
 		return PHP_JSON_T_ERROR;
 	}
+	<COMMENT_LINE>"\r\n"    {
+		s->line++;
+		s->line_start = s->cursor;
+		PHP_JSON_CONDITION_SET(JS);
+		goto std;
+	}
+	<COMMENT_LINE>[\r\n]     {
+		s->line++;
+		s->line_start = s->cursor;
+		PHP_JSON_CONDITION_SET(JS);
+		goto std;
+	}
+	<COMMENT_LINE>EOI        {
+		if (s->limit < s->cursor) {
+			s->token = s->limit;
+			PHP_JSON_CONDITION_SET(JS);
+			return PHP_JSON_T_EOI;
+		}
+		PHP_JSON_CONDITION_GOTO(COMMENT_LINE);
+	}
+	<COMMENT_LINE>UTF8       { PHP_JSON_CONDITION_GOTO(COMMENT_LINE); }
+	<COMMENT_LINE>ANY        {
+		if (s->options & (PHP_JSON_INVALID_UTF8_IGNORE | PHP_JSON_INVALID_UTF8_SUBSTITUTE)) {
+			PHP_JSON_CONDITION_GOTO(COMMENT_LINE);
+		}
+		s->token = s->cursor - 1;
+		s->errcode = PHP_JSON_ERROR_UTF8;
+		return PHP_JSON_T_ERROR;
+	}
+	<COMMENT_BLOCK>"*/"      {
+		PHP_JSON_CONDITION_SET(JS);
+		goto std;
+	}
+	<COMMENT_BLOCK>"\r\n"    {
+		s->line++;
+		s->line_start = s->cursor;
+		PHP_JSON_CONDITION_GOTO(COMMENT_BLOCK);
+	}
+	<COMMENT_BLOCK>[\r\n]    {
+		s->line++;
+		s->line_start = s->cursor;
+		PHP_JSON_CONDITION_GOTO(COMMENT_BLOCK);
+	}
+	<COMMENT_BLOCK>EOI       {
+		if (s->limit < s->cursor) {
+			s->token = comment_token;
+			s->line_start = comment_line_start;
+			s->line = comment_line;
+			s->errcode = PHP_JSON_ERROR_SYNTAX;
+			return PHP_JSON_T_ERROR;
+		}
+		PHP_JSON_CONDITION_GOTO(COMMENT_BLOCK);
+	}
+	<COMMENT_BLOCK>UTF8      { PHP_JSON_CONDITION_GOTO(COMMENT_BLOCK); }
+	<COMMENT_BLOCK>ANY       {
+		if (s->options & (PHP_JSON_INVALID_UTF8_IGNORE | PHP_JSON_INVALID_UTF8_SUBSTITUTE)) {
+			PHP_JSON_CONDITION_GOTO(COMMENT_BLOCK);
+		}
+		s->token = s->cursor - 1;
+		s->errcode = PHP_JSON_ERROR_UTF8;
+		return PHP_JSON_T_ERROR;
+	}
 	<STR_P1>EOI             {
 		if (s->limit < s->cursor) {
 			s->errcode = PHP_JSON_ERROR_SYNTAX;

ext/json/php_json.h

@@ -89,6 +89,10 @@ static inline void php_json_error_details_clear(php_json_error_details *out) {
 #define PHP_JSON_INVALID_UTF8_SUBSTITUTE    (1<<21)
 #define PHP_JSON_THROW_ON_ERROR             (1<<22)
 
+/* json_validate() and json_decode() common options */
+#define PHP_JSON_ALLOW_COMMENTS             (1<<23)
+#define PHP_JSON_ALLOW_TRAILING_COMMAS      (1<<24)
+
 /* default depth */
 #define PHP_JSON_PARSER_DEFAULT_DEPTH 512
 

ext/json/tests/json_decode_allow_comments.phpt

@@ -0,0 +1,70 @@
+--TEST--
+json_decode() and json_validate() with JSON_ALLOW_COMMENTS
+--FILE--
+<?php
+
+function decode_and_validate(string $json): void {
+    $value = json_decode($json, true, 512, JSON_ALLOW_COMMENTS);
+    echo json_encode($value, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES), "\n";
+    var_dump(json_validate($json, 512, JSON_ALLOW_COMMENTS));
+}
+
+/* Exercise every whitespace position in objects and arrays. */
+decode_and_validate('/* before root */ {
+    /* after object opener */
+    "a" /* after key */ : /* after colon */
+    [ /* after array opener */ 1 /* before comma */,
+      /* after comma */ 2 /* before array closer */ ]
+    /* after value */, /* after member comma */
+    "b"/* before colon */:/* after colon */3/* before object closer */
+} /* after root */');
+
+decode_and_validate('[1 // before comma
+, 2 // before closer
+]');
+decode_and_validate('/* object */{/* only trivia */}');
+decode_and_validate('/* array */[/* only trivia */]');
+
+/* Line comments end at LF, CR, CRLF, or end-of-input. */
+decode_and_validate("// LF\n1");
+decode_and_validate("// CR\r2");
+decode_and_validate("// CRLF\r\n3");
+decode_and_validate("4// EOF");
+decode_and_validate("// U+2028 \u{2028} and U+2029 \u{2029}\n6");
+decode_and_validate("// embedded NUL: a\x00b\n7");
+
+/* Comment delimiters inside strings are ordinary string content. */
+decode_and_validate('["//", "/* not a comment */", "https://php.net/a/*b*/"]');
+
+/* Valid Unicode, Unicode line separators, controls, and NUL are comment text. */
+decode_and_validate("/* héllo \u{2028} world \u{2029} \x01\x0c\x00 */ 5");
+decode_and_validate('/* ' . str_repeat('x', 65536) . ' */ 8');
+
+?>
+--EXPECT--
+{"a":[1,2],"b":3}
+bool(true)
+[1,2]
+bool(true)
+[]
+bool(true)
+[]
+bool(true)
+1
+bool(true)
+2
+bool(true)
+3
+bool(true)
+4
+bool(true)
+6
+bool(true)
+7
+bool(true)
+["//","/* not a comment */","https://php.net/a/*b*/"]
+bool(true)
+5
+bool(true)
+8
+bool(true)

ext/json/tests/json_decode_allow_comments_errors.phpt

@@ -0,0 +1,67 @@
+--TEST--
+JSON_ALLOW_COMMENTS rejects malformed or misplaced comments
+--FILE--
+<?php
+
+function decode_error(string $json, int $flags = JSON_ALLOW_COMMENTS): void {
+    var_dump(json_decode($json, true, 512, $flags));
+    echo json_last_error(), ": ", json_last_error_msg(), "\n";
+}
+
+/* Strict parsing is unchanged. */
+decode_error('// comment\n1', 0);
+decode_error('/* comment */ 1', 0);
+
+/* Comments are trivia, not values or token fragments. */
+decode_error('// comment only');
+decode_error('/* comment only */');
+decode_error('# comment\n1');
+decode_error('/');
+decode_error('tr/* comment */ue');
+decode_error('1e/* comment */2');
+decode_error('-/* comment */1');
+decode_error('{"a" /* missing colon */ 1}');
+
+/* Block comments do not nest; the first closing delimiter wins. */
+decode_error('/* outer /* inner */ */ 1');
+
+/* Unterminated block comments report their opening delimiter. */
+decode_error('/* unterminated');
+decode_error("{\n  \"a\": 1,\n  /* unterminated\n  still open");
+
+try {
+    json_decode("[1,\n /* unterminated", true, 512,
+        JSON_ALLOW_COMMENTS | JSON_THROW_ON_ERROR);
+} catch (JsonException $e) {
+    echo get_class($e), ': ', $e->getCode(), ': ', $e->getMessage(), "\n";
+}
+
+?>
+--EXPECT--
+NULL
+4: Syntax error near location 1:1
+NULL
+4: Syntax error near location 1:1
+NULL
+4: Syntax error near location 1:16
+NULL
+4: Syntax error near location 1:19
+NULL
+4: Syntax error near location 1:1
+NULL
+4: Syntax error near location 1:1
+NULL
+4: Syntax error near location 1:1
+NULL
+4: Syntax error near location 1:2
+NULL
+4: Syntax error near location 1:1
+NULL
+4: Syntax error near location 1:26
+NULL
+4: Syntax error near location 1:22
+NULL
+4: Syntax error near location 1:1
+NULL
+4: Syntax error near location 3:3
+JsonException: 4: Syntax error near location 2:2

ext/json/tests/json_decode_allow_comments_utf8.phpt

@@ -0,0 +1,57 @@
+--TEST--
+JSON_ALLOW_COMMENTS interactions with malformed UTF-8 flags
+--FILE--
+<?php
+
+function show_decode(string $json, int $flags): void {
+    var_dump(json_decode($json, true, 512, JSON_ALLOW_COMMENTS | $flags));
+    echo json_last_error(), "\n";
+}
+
+$badBlockComment = "/* bad: \x80 */ [\"a\x80b\"]";
+$badLineComment = "// bad: \x80\n1";
+
+show_decode($badBlockComment, 0);
+show_decode($badBlockComment, JSON_INVALID_UTF8_IGNORE);
+show_decode($badBlockComment, JSON_INVALID_UTF8_SUBSTITUTE);
+show_decode($badLineComment, 0);
+show_decode($badLineComment, JSON_INVALID_UTF8_IGNORE);
+
+var_dump(json_validate($badBlockComment, 512, JSON_ALLOW_COMMENTS));
+echo json_last_error(), "\n";
+var_dump(json_validate(
+    $badBlockComment,
+    512,
+    JSON_ALLOW_COMMENTS | JSON_INVALID_UTF8_IGNORE
+));
+echo json_last_error(), "\n";
+
+try {
+    json_decode($badLineComment, flags: JSON_ALLOW_COMMENTS | JSON_THROW_ON_ERROR);
+} catch (JsonException $e) {
+    echo $e->getCode(), ': ', $e->getMessage(), "\n";
+}
+
+?>
+--EXPECT--
+NULL
+5
+array(1) {
+  [0]=>
+  string(2) "ab"
+}
+0
+array(1) {
+  [0]=>
+  string(5) "a�b"
+}
+0
+NULL
+5
+int(1)
+0
+bool(false)
+5
+bool(true)
+0
+5: Malformed UTF-8 characters, possibly incorrectly encoded near location 1:9