Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions NEWS
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@ PHP NEWS
. Fixed mhash_keygen_s2k() truncating a $length above INT_MAX instead of
throwing a ValueError. (Marc Bennewitz)

- JSON:
. Improved performance of json_decode(), json_validate() and json_encode()
for input containing strings. (zeynallow)

- Intl:
. Added IntlRelativeDateTimeFormatter for locale-aware relative date and time
formatting, with enums for styles, capitalization contexts, and units.
Expand Down
5 changes: 5 additions & 0 deletions UPGRADING
Original file line number Diff line number Diff line change
Expand Up @@ -180,6 +180,11 @@ PHP 8.7 UPGRADE NOTES
. htmlspecialchars() returns its input without copying it when nothing needs
encoding.

- JSON:
. Improved performance of json_decode() and json_validate() when scanning
strings, and of json_encode() when escaping strings, notably UTF-8 text
with JSON_UNESCAPED_UNICODE.

- MBString:
. Improved performance of mb_strlen for UTF-8 strings.

Expand Down
86 changes: 83 additions & 3 deletions ext/json/json_encoder.c
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,11 @@
#include "zend_enum.h"
#include "zend_property_hooks.h"
#include "zend_lazy_objects.h"
#include "zend_bitset.h"

#ifdef __SSE2__
# include <emmintrin.h>
#endif

static const char digits[] = "0123456789abcdef";

Expand Down Expand Up @@ -358,11 +363,76 @@ static zend_result php_json_encode_array(smart_str *buf, zval *val, int options,
}
/* }}} */

/* Number of leading bytes of s that are copied verbatim by php_json_escape_string()
* regardless of options, i.e. printable ASCII except " & ' / < > and backslash. */
static zend_always_inline size_t php_json_verbatim_ascii_len(const char *s, size_t len)
{
size_t i = 0;
#ifdef __SSE2__
/* As signed bytes, both control characters (< 0x20) and non-ASCII bytes (>= 0x80) are < 0x20. */
const __m128i ctrl_or_high = _mm_set1_epi8(0x20);
while (len - i >= 16) {
const __m128i chunk = _mm_loadu_si128((const __m128i *) (s + i));
__m128i special = _mm_cmpgt_epi8(ctrl_or_high, chunk);
special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('"')));
special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('&')));
special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('\'')));
special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('/')));
special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('<')));
special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('>')));
special = _mm_or_si128(special, _mm_cmpeq_epi8(chunk, _mm_set1_epi8('\\')));
const int mask = _mm_movemask_epi8(special);
if (mask) {
return i + zend_ulong_ntz((zend_ulong) mask);
}
i += 16;
}
#endif
while (i < len) {
const unsigned char c = (unsigned char) s[i];
if (c < 0x20 || c >= 0x80 || c == '"' || c == '&' || c == '\'' || c == '/'
|| c == '<' || c == '>' || c == '\\') {
break;
}
i++;
}
return i;
}

/* With JSON_UNESCAPED_UNICODE, a well-formed UTF-8 sequence is copied verbatim unless it encodes
* U+2028/U+2029 and JSON_UNESCAPED_LINE_TERMINATORS is not set. Returns the length of such a
* sequence at s (whose first byte is >= 0x80), or 0 if it needs the generic code path. */
static zend_always_inline size_t php_json_verbatim_utf8_len(const unsigned char *s, size_t len, int options)
{
const unsigned char c = s[0];
if (c >= 0xc2 && c < 0xe0) {
if (len >= 2 && (s[1] & 0xc0) == 0x80) {
return 2;
}
} else if (c >= 0xe0 && c < 0xf0) {
if (len >= 3 && (s[1] & 0xc0) == 0x80 && (s[2] & 0xc0) == 0x80) {
const unsigned int cp = ((c & 0x0f) << 12) | ((s[1] & 0x3f) << 6) | (s[2] & 0x3f);
if (cp >= 0x800 && (cp < 0xd800 || cp > 0xdfff)
&& (cp < 0x2028 || cp > 0x2029 || (options & PHP_JSON_UNESCAPED_LINE_TERMINATORS))) {
return 3;
}
}
} else if (c >= 0xf0 && c < 0xf5) {
if (len >= 4 && (s[1] & 0xc0) == 0x80 && (s[2] & 0xc0) == 0x80 && (s[3] & 0xc0) == 0x80) {
const unsigned int cp = ((c & 0x07) << 18) | ((s[1] & 0x3f) << 12) | ((s[2] & 0x3f) << 6) | (s[3] & 0x3f);
if (cp >= 0x10000 && cp <= 0x10ffff) {
return 4;
}
}
}
return 0;
}

zend_result php_json_escape_string(
smart_str *buf, const char *s, size_t len,
int options, php_json_encoder *encoder) /* {{{ */
{
size_t pos, checkpoint;
size_t pos, checkpoint, verbatim;
char *dst;

if (len == 0) {
Expand Down Expand Up @@ -403,8 +473,18 @@ zend_result php_json_escape_string(

unsigned int us = (unsigned char)s[pos];
if (EXPECTED(!ZEND_BIT_TEST(charmap, us))) {
pos++;
len--;
verbatim = 1 + php_json_verbatim_ascii_len(s + pos + 1, len - 1);
pos += verbatim;
len -= verbatim;
if (len == 0) {
smart_str_appendl(buf, s, pos);
break;
}
} else if (us >= 0x80 && (options & PHP_JSON_UNESCAPED_UNICODE)
&& (verbatim = php_json_verbatim_utf8_len((const unsigned char *) s + pos, len, options))) {
/* Keep well-formed UTF-8 in the pending run instead of copying it on its own. */
pos += verbatim;
len -= verbatim;
if (len == 0) {
smart_str_appendl(buf, s, pos);
break;
Expand Down
57 changes: 50 additions & 7 deletions ext/json/json_scanner.re
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,11 @@
#include "php_json_scanner_defs.h"
#include "php_json_parser.h"
#include "json_parser.tab.h"
#include "zend_bitset.h"

#ifdef __SSE2__
# include <emmintrin.h>
#endif

#define YYCTYPE php_json_ctype
#define YYCURSOR s->cursor
Expand All @@ -34,6 +39,13 @@
#define PHP_JSON_CONDITION_SET_AND_GOTO(condition) \
PHP_JSON_CONDITION_SET(condition); \
PHP_JSON_CONDITION_GOTO(condition)
/* Skip the run of bytes inside a string that need no handling by the state machine
* before (re)entering the first string pass. */
#define PHP_JSON_CONDITION_GOTO_STR_P1() \
do { \
s->cursor = php_json_skip_plain_string_bytes(s->cursor, s->limit); \
PHP_JSON_CONDITION_GOTO(STR_P1); \
} while (0)
#define PHP_JSON_CONDITION_GOTO_STR_P2() \
do { \
if (s->utf8_invalid) { \
Expand Down Expand Up @@ -61,6 +73,36 @@ static void php_json_scanner_copy_string(php_json_scanner *s, size_t esc_size)
}
}

/* Returns a pointer to the first byte at or after p (and before limit) that is not printable
* ASCII other than '"' and '\\'. Those are the only bytes the first string pass has to look
* at; everything else is accepted as is. */
static zend_always_inline php_json_ctype *php_json_skip_plain_string_bytes(
php_json_ctype *p, const php_json_ctype *limit)
{
#ifdef __SSE2__
const __m128i quote = _mm_set1_epi8('"');
const __m128i backslash = _mm_set1_epi8('\\');
/* As signed bytes, both control characters (< 0x20) and non-ASCII bytes (>= 0x80) are < 0x20. */
const __m128i ctrl_or_high = _mm_set1_epi8(0x20);

while (limit - p >= 16) {
const __m128i chunk = _mm_loadu_si128((const __m128i *) p);
const __m128i special = _mm_or_si128(
_mm_or_si128(_mm_cmpeq_epi8(chunk, quote), _mm_cmpeq_epi8(chunk, backslash)),
_mm_cmpgt_epi8(ctrl_or_high, chunk));
const int mask = _mm_movemask_epi8(special);
if (mask) {
return p + zend_ulong_ntz((zend_ulong) mask);
}
p += 16;
}
#endif
while (p < limit && *p >= 0x20 && *p < 0x80 && *p != '"' && *p != '\\') {
p++;
}
return p;
}

static int php_json_hex_to_int(unsigned char code)
{
if (code >= '0' && code <= '9') {
Expand Down Expand Up @@ -216,7 +258,8 @@ std:
s->str_esc = 0;
s->utf8_invalid = 0;
s->utf8_invalid_count = 0;
PHP_JSON_CONDITION_SET_AND_GOTO(STR_P1);
PHP_JSON_CONDITION_SET(STR_P1);
PHP_JSON_CONDITION_GOTO_STR_P1();
}
<JS>CTRL {
s->errcode = PHP_JSON_ERROR_CTRL_CHAR;
Expand Down Expand Up @@ -244,27 +287,27 @@ std:
}
<STR_P1>UTF16_1 {
s->str_esc += 5;
PHP_JSON_CONDITION_GOTO(STR_P1);
PHP_JSON_CONDITION_GOTO_STR_P1();
}
<STR_P1>UTF16_2 {
s->str_esc += 4;
PHP_JSON_CONDITION_GOTO(STR_P1);
PHP_JSON_CONDITION_GOTO_STR_P1();
}
<STR_P1>UTF16_3 {
s->str_esc += 3;
PHP_JSON_CONDITION_GOTO(STR_P1);
PHP_JSON_CONDITION_GOTO_STR_P1();
}
<STR_P1>UTF16_4 {
s->str_esc += 8;
PHP_JSON_CONDITION_GOTO(STR_P1);
PHP_JSON_CONDITION_GOTO_STR_P1();
}
<STR_P1>UCS2 {
s->errcode = PHP_JSON_ERROR_UTF16;
return PHP_JSON_T_ERROR;
}
<STR_P1>ESC {
s->str_esc++;
PHP_JSON_CONDITION_GOTO(STR_P1);
PHP_JSON_CONDITION_GOTO_STR_P1();
}
<STR_P1>ESCPREF {
s->errcode = PHP_JSON_ERROR_SYNTAX;
Expand Down Expand Up @@ -295,7 +338,7 @@ std:
return PHP_JSON_T_STRING;
}
}
<STR_P1>UTF8 { PHP_JSON_CONDITION_GOTO(STR_P1); }
<STR_P1>UTF8 { PHP_JSON_CONDITION_GOTO_STR_P1(); }
<STR_P1>ANY {
if (s->options & (PHP_JSON_INVALID_UTF8_IGNORE | PHP_JSON_INVALID_UTF8_SUBSTITUTE)) {
if (s->options & PHP_JSON_INVALID_UTF8_SUBSTITUTE) {
Expand Down
90 changes: 90 additions & 0 deletions ext/json/tests/string_scan_boundaries.phpt
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
--TEST--
json_decode()/json_encode() handle special bytes at every position of a string, across vectorized chunk boundaries
--FILE--
<?php

$plain = str_repeat('abcdefghijklmnop', 3); /* 48 bytes, several 16-byte chunks */
$failures = 0;

/* Decoding: raw bytes placed inside a JSON string literal. */
$raw = [
"\x00" => JSON_ERROR_CTRL_CHAR,
"\x1f" => JSON_ERROR_CTRL_CHAR,
"\xff" => JSON_ERROR_UTF8,
"\xc3" => JSON_ERROR_UTF8,
"\xed\xa0\x80" => JSON_ERROR_UTF8,
"\x20" => JSON_ERROR_NONE,
"\x7f" => JSON_ERROR_NONE,
"é" => JSON_ERROR_NONE,
"\u{2028}" => JSON_ERROR_NONE,
"😀" => JSON_ERROR_NONE,
];
for ($len = 0; $len <= strlen($plain); $len++) {
for ($pos = 0; $pos <= $len; $pos++) {
$prefix = substr($plain, 0, $pos);
$suffix = substr($plain, $pos, $len - $pos);
foreach ($raw as $byte => $error) {
$value = $prefix . $byte . $suffix;
$decoded = json_decode('"' . $value . '"');
if (json_last_error() !== $error || ($error === JSON_ERROR_NONE && $decoded !== $value)) {
echo "decode failed: len=$len pos=$pos byte=", bin2hex($byte), "\n";
$failures++;
}
if (json_validate('"' . $value . '"') !== ($error === JSON_ERROR_NONE)) {
echo "validate failed: len=$len pos=$pos byte=", bin2hex($byte), "\n";
$failures++;
}
}
/* Escape sequences and a lone quote that terminates the string early. */
foreach (['\\"' => '"', '\\\\' => '\\', '\\n' => "\n", '\\u00e9' => 'é', '\\ud83d\\ude00' => '😀'] as $esc => $char) {
if (json_decode('"' . $prefix . $esc . $suffix . '"') !== $prefix . $char . $suffix) {
echo "escape failed: len=$len pos=$pos esc=$esc\n";
$failures++;
}
}
json_decode('"' . $prefix . '"' . $suffix . '"');
if (json_last_error() !== JSON_ERROR_SYNTAX) {
echo "early quote failed: len=$len pos=$pos\n";
$failures++;
}
}
}

/* Encoding: each byte that may need escaping, with and without JSON_UNESCAPED_UNICODE. */
$special = ['"', '\\', '/', '<', '>', '&', "'", "\n", "\x00", "\x1f", "\x7f", "é", "€", "😀", "\u{2028}", "\u{2029}"];
$flagsets = [0, JSON_UNESCAPED_UNICODE, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_LINE_TERMINATORS,
JSON_UNESCAPED_SLASHES | JSON_HEX_TAG | JSON_HEX_AMP | JSON_HEX_APOS | JSON_HEX_QUOT];
for ($len = 0; $len <= strlen($plain); $len++) {
for ($pos = 0; $pos <= $len; $pos++) {
foreach ($special as $char) {
$value = substr($plain, 0, $pos) . $char . substr($plain, $pos, $len - $pos);
foreach ($flagsets as $flags) {
$encoded = json_encode($value, $flags);
if (json_decode($encoded) !== $value) {
echo "encode roundtrip failed: len=$len pos=$pos char=", bin2hex($char), " flags=$flags\n";
$failures++;
}
$escapes_unicode = !($flags & JSON_UNESCAPED_UNICODE);
$escapes_lt = $escapes_unicode || !($flags & JSON_UNESCAPED_LINE_TERMINATORS);
$verbatim = strlen($char) > 1 && ($char === "\u{2028}" || $char === "\u{2029}" ? !$escapes_lt : !$escapes_unicode);
if (strlen($char) > 1 && $verbatim !== str_contains($encoded, $char)) {
echo "encode escaping failed: len=$len pos=$pos char=", bin2hex($char), " flags=$flags\n";
$failures++;
}
}
}
/* Invalid UTF-8 at every position. */
$value = substr($plain, 0, $pos) . "\xff" . substr($plain, $pos, $len - $pos);
if (json_encode($value, JSON_UNESCAPED_UNICODE) !== false || json_last_error() !== JSON_ERROR_UTF8
|| json_encode($value, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_IGNORE) !== json_encode(str_replace("\xff", '', $value))) {
echo "encode invalid UTF-8 failed: len=$len pos=$pos\n";
$failures++;
}
}
}

echo $failures === 0 ? "OK\n" : "$failures failures\n";

?>
--EXPECT--
OK
Loading