diff --git a/NEWS b/NEWS index 6bb8496271f8..2bfb41e8d274 100644 --- a/NEWS +++ b/NEWS @@ -22,6 +22,10 @@ PHP NEWS . Fixed mhash_keygen_s2k() truncating a $length above INT_MAX instead of throwing a ValueError. (Marc Bennewitz) +- JSON: + . Improved performance of json_decode(), json_validate() and json_encode() + for input containing strings. (zeynallow) + - Intl: . Added IntlRelativeDateTimeFormatter for locale-aware relative date and time formatting, with enums for styles, capitalization contexts, and units. diff --git a/UPGRADING b/UPGRADING index baa930594bd9..1a6926c07e4f 100644 --- a/UPGRADING +++ b/UPGRADING @@ -180,6 +180,11 @@ PHP 8.7 UPGRADE NOTES . htmlspecialchars() returns its input without copying it when nothing needs encoding. +- JSON: + . Improved performance of json_decode() and json_validate() when scanning + strings, and of json_encode() when escaping strings, notably UTF-8 text + with JSON_UNESCAPED_UNICODE. + - MBString: . Improved performance of mb_strlen for UTF-8 strings. diff --git a/ext/json/json_encoder.c b/ext/json/json_encoder.c index 9d46ceab3ccd..a1a050039b7a 100644 --- a/ext/json/json_encoder.c +++ b/ext/json/json_encoder.c @@ -27,6 +27,11 @@ #include "zend_enum.h" #include "zend_property_hooks.h" #include "zend_lazy_objects.h" +#include "zend_bitset.h" + +#ifdef __SSE2__ +# include +#endif static const char digits[] = "0123456789abcdef"; @@ -358,11 +363,76 @@ static zend_result php_json_encode_array(smart_str *buf, zval *val, int options, } /* }}} */ +/* Number of leading bytes of s that are copied verbatim by php_json_escape_string() + * regardless of options, i.e. printable ASCII except " & ' / < > and backslash. */ +static zend_always_inline size_t php_json_verbatim_ascii_len(const char *s, size_t len) +{ + size_t i = 0; +#ifdef __SSE2__ + /* As signed bytes, both control characters (< 0x20) and non-ASCII bytes (>= 0x80) are < 0x20. */ + const __m128i ctrl_or_high = _mm_set1_epi8(0x20); + while (len - i >= 16) { + const __m128i chunk = _mm_loadu_si128((const __m128i *) (s + i)); + __m128i special = _mm_cmpgt_epi8(ctrl_or_high, chunk); + special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('"'))); + special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('&'))); + special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('\''))); + special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('/'))); + special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('<'))); + special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('>'))); + special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('\\'))); + const int mask = _mm_movemask_epi8(special); + if (mask) { + return i + zend_ulong_ntz((zend_ulong) mask); + } + i += 16; + } +#endif + while (i < len) { + const unsigned char c = (unsigned char) s[i]; + if (c < 0x20 || c >= 0x80 || c == '"' || c == '&' || c == '\'' || c == '/' + || c == '<' || c == '>' || c == '\\') { + break; + } + i++; + } + return i; +} + +/* With JSON_UNESCAPED_UNICODE, a well-formed UTF-8 sequence is copied verbatim unless it encodes + * U+2028/U+2029 and JSON_UNESCAPED_LINE_TERMINATORS is not set. Returns the length of such a + * sequence at s (whose first byte is >= 0x80), or 0 if it needs the generic code path. */ +static zend_always_inline size_t php_json_verbatim_utf8_len(const unsigned char *s, size_t len, int options) +{ + const unsigned char c = s[0]; + if (c >= 0xc2 && c < 0xe0) { + if (len >= 2 && (s[1] & 0xc0) == 0x80) { + return 2; + } + } else if (c >= 0xe0 && c < 0xf0) { + if (len >= 3 && (s[1] & 0xc0) == 0x80 && (s[2] & 0xc0) == 0x80) { + const unsigned int cp = ((c & 0x0f) << 12) | ((s[1] & 0x3f) << 6) | (s[2] & 0x3f); + if (cp >= 0x800 && (cp < 0xd800 || cp > 0xdfff) + && (cp < 0x2028 || cp > 0x2029 || (options & PHP_JSON_UNESCAPED_LINE_TERMINATORS))) { + return 3; + } + } + } else if (c >= 0xf0 && c < 0xf5) { + if (len >= 4 && (s[1] & 0xc0) == 0x80 && (s[2] & 0xc0) == 0x80 && (s[3] & 0xc0) == 0x80) { + const unsigned int cp = ((c & 0x07) << 18) | ((s[1] & 0x3f) << 12) | ((s[2] & 0x3f) << 6) | (s[3] & 0x3f); + if (cp >= 0x10000 && cp <= 0x10ffff) { + return 4; + } + } + } + return 0; +} + zend_result php_json_escape_string( smart_str *buf, const char *s, size_t len, int options, php_json_encoder *encoder) /* {{{ */ { - size_t pos, checkpoint; + size_t pos, checkpoint, verbatim; char *dst; if (len == 0) { @@ -403,8 +473,18 @@ zend_result php_json_escape_string( unsigned int us = (unsigned char)s[pos]; if (EXPECTED(!ZEND_BIT_TEST(charmap, us))) { - pos++; - len--; + verbatim = 1 + php_json_verbatim_ascii_len(s + pos + 1, len - 1); + pos += verbatim; + len -= verbatim; + if (len == 0) { + smart_str_appendl(buf, s, pos); + break; + } + } else if (us >= 0x80 && (options & PHP_JSON_UNESCAPED_UNICODE) + && (verbatim = php_json_verbatim_utf8_len((const unsigned char *) s + pos, len, options))) { + /* Keep well-formed UTF-8 in the pending run instead of copying it on its own. */ + pos += verbatim; + len -= verbatim; if (len == 0) { smart_str_appendl(buf, s, pos); break; diff --git a/ext/json/json_scanner.re b/ext/json/json_scanner.re index 91c5c0024390..f3239eafcfdd 100644 --- a/ext/json/json_scanner.re +++ b/ext/json/json_scanner.re @@ -17,6 +17,11 @@ #include "php_json_scanner_defs.h" #include "php_json_parser.h" #include "json_parser.tab.h" +#include "zend_bitset.h" + +#ifdef __SSE2__ +# include +#endif #define YYCTYPE php_json_ctype #define YYCURSOR s->cursor @@ -34,6 +39,13 @@ #define PHP_JSON_CONDITION_SET_AND_GOTO(condition) \ PHP_JSON_CONDITION_SET(condition); \ PHP_JSON_CONDITION_GOTO(condition) +/* Skip the run of bytes inside a string that need no handling by the state machine + * before (re)entering the first string pass. */ +#define PHP_JSON_CONDITION_GOTO_STR_P1() \ + do { \ + s->cursor = php_json_skip_plain_string_bytes(s->cursor, s->limit); \ + PHP_JSON_CONDITION_GOTO(STR_P1); \ + } while (0) #define PHP_JSON_CONDITION_GOTO_STR_P2() \ do { \ if (s->utf8_invalid) { \ @@ -61,6 +73,36 @@ static void php_json_scanner_copy_string(php_json_scanner *s, size_t esc_size) } } +/* Returns a pointer to the first byte at or after p (and before limit) that is not printable + * ASCII other than '"' and '\\'. Those are the only bytes the first string pass has to look + * at; everything else is accepted as is. */ +static zend_always_inline php_json_ctype *php_json_skip_plain_string_bytes( + php_json_ctype *p, const php_json_ctype *limit) +{ +#ifdef __SSE2__ + const __m128i quote = _mm_set1_epi8('"'); + const __m128i backslash = _mm_set1_epi8('\\'); + /* As signed bytes, both control characters (< 0x20) and non-ASCII bytes (>= 0x80) are < 0x20. */ + const __m128i ctrl_or_high = _mm_set1_epi8(0x20); + + while (limit - p >= 16) { + const __m128i chunk = _mm_loadu_si128((const __m128i *) p); + const __m128i special = _mm_or_si128( + _mm_or_si128(_mm_cmpeq_epi8(chunk, quote), _mm_cmpeq_epi8(chunk, backslash)), + _mm_cmpgt_epi8(ctrl_or_high, chunk)); + const int mask = _mm_movemask_epi8(special); + if (mask) { + return p + zend_ulong_ntz((zend_ulong) mask); + } + p += 16; + } +#endif + while (p < limit && *p >= 0x20 && *p < 0x80 && *p != '"' && *p != '\\') { + p++; + } + return p; +} + static int php_json_hex_to_int(unsigned char code) { if (code >= '0' && code <= '9') { @@ -216,7 +258,8 @@ std: s->str_esc = 0; s->utf8_invalid = 0; s->utf8_invalid_count = 0; - PHP_JSON_CONDITION_SET_AND_GOTO(STR_P1); + PHP_JSON_CONDITION_SET(STR_P1); + PHP_JSON_CONDITION_GOTO_STR_P1(); } CTRL { s->errcode = PHP_JSON_ERROR_CTRL_CHAR; @@ -244,19 +287,19 @@ std: } UTF16_1 { s->str_esc += 5; - PHP_JSON_CONDITION_GOTO(STR_P1); + PHP_JSON_CONDITION_GOTO_STR_P1(); } UTF16_2 { s->str_esc += 4; - PHP_JSON_CONDITION_GOTO(STR_P1); + PHP_JSON_CONDITION_GOTO_STR_P1(); } UTF16_3 { s->str_esc += 3; - PHP_JSON_CONDITION_GOTO(STR_P1); + PHP_JSON_CONDITION_GOTO_STR_P1(); } UTF16_4 { s->str_esc += 8; - PHP_JSON_CONDITION_GOTO(STR_P1); + PHP_JSON_CONDITION_GOTO_STR_P1(); } UCS2 { s->errcode = PHP_JSON_ERROR_UTF16; @@ -264,7 +307,7 @@ std: } ESC { s->str_esc++; - PHP_JSON_CONDITION_GOTO(STR_P1); + PHP_JSON_CONDITION_GOTO_STR_P1(); } ESCPREF { s->errcode = PHP_JSON_ERROR_SYNTAX; @@ -295,7 +338,7 @@ std: return PHP_JSON_T_STRING; } } - UTF8 { PHP_JSON_CONDITION_GOTO(STR_P1); } + UTF8 { PHP_JSON_CONDITION_GOTO_STR_P1(); } ANY { if (s->options & (PHP_JSON_INVALID_UTF8_IGNORE | PHP_JSON_INVALID_UTF8_SUBSTITUTE)) { if (s->options & PHP_JSON_INVALID_UTF8_SUBSTITUTE) { diff --git a/ext/json/tests/string_scan_boundaries.phpt b/ext/json/tests/string_scan_boundaries.phpt new file mode 100644 index 000000000000..16ff9115baf2 --- /dev/null +++ b/ext/json/tests/string_scan_boundaries.phpt @@ -0,0 +1,90 @@ +--TEST-- +json_decode()/json_encode() handle special bytes at every position of a string, across vectorized chunk boundaries +--FILE-- + JSON_ERROR_CTRL_CHAR, + "\x1f" => JSON_ERROR_CTRL_CHAR, + "\xff" => JSON_ERROR_UTF8, + "\xc3" => JSON_ERROR_UTF8, + "\xed\xa0\x80" => JSON_ERROR_UTF8, + "\x20" => JSON_ERROR_NONE, + "\x7f" => JSON_ERROR_NONE, + "é" => JSON_ERROR_NONE, + "\u{2028}" => JSON_ERROR_NONE, + "😀" => JSON_ERROR_NONE, +]; +for ($len = 0; $len <= strlen($plain); $len++) { + for ($pos = 0; $pos <= $len; $pos++) { + $prefix = substr($plain, 0, $pos); + $suffix = substr($plain, $pos, $len - $pos); + foreach ($raw as $byte => $error) { + $value = $prefix . $byte . $suffix; + $decoded = json_decode('"' . $value . '"'); + if (json_last_error() !== $error || ($error === JSON_ERROR_NONE && $decoded !== $value)) { + echo "decode failed: len=$len pos=$pos byte=", bin2hex($byte), "\n"; + $failures++; + } + if (json_validate('"' . $value . '"') !== ($error === JSON_ERROR_NONE)) { + echo "validate failed: len=$len pos=$pos byte=", bin2hex($byte), "\n"; + $failures++; + } + } + /* Escape sequences and a lone quote that terminates the string early. */ + foreach (['\\"' => '"', '\\\\' => '\\', '\\n' => "\n", '\\u00e9' => 'é', '\\ud83d\\ude00' => '😀'] as $esc => $char) { + if (json_decode('"' . $prefix . $esc . $suffix . '"') !== $prefix . $char . $suffix) { + echo "escape failed: len=$len pos=$pos esc=$esc\n"; + $failures++; + } + } + json_decode('"' . $prefix . '"' . $suffix . '"'); + if (json_last_error() !== JSON_ERROR_SYNTAX) { + echo "early quote failed: len=$len pos=$pos\n"; + $failures++; + } + } +} + +/* Encoding: each byte that may need escaping, with and without JSON_UNESCAPED_UNICODE. */ +$special = ['"', '\\', '/', '<', '>', '&', "'", "\n", "\x00", "\x1f", "\x7f", "é", "€", "😀", "\u{2028}", "\u{2029}"]; +$flagsets = [0, JSON_UNESCAPED_UNICODE, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_LINE_TERMINATORS, + JSON_UNESCAPED_SLASHES | JSON_HEX_TAG | JSON_HEX_AMP | JSON_HEX_APOS | JSON_HEX_QUOT]; +for ($len = 0; $len <= strlen($plain); $len++) { + for ($pos = 0; $pos <= $len; $pos++) { + foreach ($special as $char) { + $value = substr($plain, 0, $pos) . $char . substr($plain, $pos, $len - $pos); + foreach ($flagsets as $flags) { + $encoded = json_encode($value, $flags); + if (json_decode($encoded) !== $value) { + echo "encode roundtrip failed: len=$len pos=$pos char=", bin2hex($char), " flags=$flags\n"; + $failures++; + } + $escapes_unicode = !($flags & JSON_UNESCAPED_UNICODE); + $escapes_lt = $escapes_unicode || !($flags & JSON_UNESCAPED_LINE_TERMINATORS); + $verbatim = strlen($char) > 1 && ($char === "\u{2028}" || $char === "\u{2029}" ? !$escapes_lt : !$escapes_unicode); + if (strlen($char) > 1 && $verbatim !== str_contains($encoded, $char)) { + echo "encode escaping failed: len=$len pos=$pos char=", bin2hex($char), " flags=$flags\n"; + $failures++; + } + } + } + /* Invalid UTF-8 at every position. */ + $value = substr($plain, 0, $pos) . "\xff" . substr($plain, $pos, $len - $pos); + if (json_encode($value, JSON_UNESCAPED_UNICODE) !== false || json_last_error() !== JSON_ERROR_UTF8 + || json_encode($value, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_IGNORE) !== json_encode(str_replace("\xff", '', $value))) { + echo "encode invalid UTF-8 failed: len=$len pos=$pos\n"; + $failures++; + } + } +} + +echo $failures === 0 ? "OK\n" : "$failures failures\n"; + +?> +--EXPECT-- +OK