add JSON_UNESCAPED_UNICODE

This is giving warnings on wikimedia and was implemented in 5.4

hphp/runtime/base/util/string_buffer.cpp

@@ -19,6 +19,7 @@
 #include "hphp/runtime/base/zend/zend_functions.h"
 #include "hphp/runtime/base/zend/utf8_decode.h"
 #include "hphp/runtime/ext/ext_json.h"
+#include "hphp/runtime/ext/JSON_parser.h"
 
 #include "hphp/util/alloc.h"
 
@@ -311,7 +312,9 @@ void StringBuffer::appendJsonEscape(const char *s, int len, int options) {
         }
         break;
       default:
-        if (us >= ' ' && (us & 127) == us) {
+        if (us >= ' ' && options & k_JSON_UNESCAPED_UNICODE) {
+          utf16_to_utf8(*this, us);
+        } else if (us >= ' ' && (us & 127) == us) {
           append((char)us);
         } else {
           append("\\u", 2);

hphp/runtime/ext/JSON_parser.cpp

@@ -28,7 +28,6 @@ SOFTWARE.
 
 
 #include "hphp/runtime/ext/JSON_parser.h"
-#include "hphp/runtime/base/util/string_buffer.h"
 #include "hphp/runtime/base/complex_types.h"
 #include "hphp/runtime/base/type_conversions.h"
 #include "hphp/runtime/base/builtin_functions.h"
@@ -402,7 +401,7 @@ static void json_create_zval(Variant &z, StringBuffer &buf, int type) {
   }
 }
 
-static void utf16_to_utf8(StringBuffer &buf, unsigned short utf16) {
+void utf16_to_utf8(StringBuffer &buf, unsigned short utf16) {
   if (utf16 < 0x80) {
     buf += (char)utf16;
   } else if (utf16 < 0x800) {

hphp/runtime/ext/JSON_parser.h

@@ -14,9 +14,15 @@
    | license@php.net so we can mail you a copy immediately.               |
    +----------------------------------------------------------------------+
 */
-/* JSON_checker.h */
+
+#ifndef incl_HPHP_JSON_PARSER_H_
+#define incl_HPHP_JSON_PARSER_H_
 
 #include "hphp/runtime/base/complex_types.h"
+#include "hphp/runtime/base/util/string_buffer.h"
 
+void utf16_to_utf8(HPHP::StringBuffer &buf, unsigned short utf16);
 bool JSON_parser(HPHP::Variant &z, const char *p, int length,
                  bool assoc/*<fb>*/, bool loose/*</fb>*/);
+
+#endif // incl_HPHP_JSON_PARSER_H_

hphp/runtime/ext/ext_json.cpp

@@ -23,18 +23,19 @@
 namespace HPHP {
 IMPLEMENT_DEFAULT_EXTENSION(json);
 ///////////////////////////////////////////////////////////////////////////////
-const int64_t k_JSON_HEX_TAG       = 1<<0;
-const int64_t k_JSON_HEX_AMP       = 1<<1;
-const int64_t k_JSON_HEX_APOS      = 1<<2;
-const int64_t k_JSON_HEX_QUOT      = 1<<3;
-const int64_t k_JSON_FORCE_OBJECT  = 1<<4;
-const int64_t k_JSON_NUMERIC_CHECK = 1<<5;
+const int64_t k_JSON_HEX_TAG           = 1<<0;
+const int64_t k_JSON_HEX_AMP           = 1<<1;
+const int64_t k_JSON_HEX_APOS          = 1<<2;
+const int64_t k_JSON_HEX_QUOT          = 1<<3;
+const int64_t k_JSON_FORCE_OBJECT      = 1<<4;
+const int64_t k_JSON_NUMERIC_CHECK     = 1<<5;
 const int64_t k_JSON_UNESCAPED_SLASHES = 1<<6;
-const int64_t k_JSON_PRETTY_PRINT  = 1<<7;
+const int64_t k_JSON_PRETTY_PRINT      = 1<<7;
+const int64_t k_JSON_UNESCAPED_UNICODE = 1<<8;
 // intentionally higher so when PHP adds more options we're fine
-const int64_t k_JSON_FB_LOOSE      = 1<<20;
-const int64_t k_JSON_FB_UNLIMITED  = 1<<21;
-const int64_t k_JSON_FB_EXTRA_ESCAPES = 1<<22;
+const int64_t k_JSON_FB_LOOSE          = 1<<20;
+const int64_t k_JSON_FB_UNLIMITED      = 1<<21;
+const int64_t k_JSON_FB_EXTRA_ESCAPES  = 1<<22;
 
 ///////////////////////////////////////////////////////////////////////////////
 

hphp/runtime/ext/ext_json.h

@@ -35,6 +35,7 @@ extern const int64_t k_JSON_FORCE_OBJECT;
 extern const int64_t k_JSON_NUMERIC_CHECK;
 extern const int64_t k_JSON_UNESCAPED_SLASHES;
 extern const int64_t k_JSON_PRETTY_PRINT;
+extern const int64_t k_JSON_UNESCAPED_UNICODE;
 extern const int64_t k_JSON_FB_LOOSE;
 extern const int64_t k_JSON_FB_LOOSE;
 extern const int64_t k_JSON_FB_EXTRA_ESCAPES;

hphp/system/idl/json.idl.json

@@ -33,6 +33,10 @@
             "name": "JSON_PRETTY_PRINT",
             "type": "Int64"
         },
+        {
+            "name": "JSON_UNESCAPED_UNICODE",
+            "type": "Int64"
+        },
         {
             "name": "JSON_FB_LOOSE",
             "type": "Int64"
@@ -104,4 +108,4 @@
     ],
     "classes": [
     ]
-}
+}

hphp/test/slow/json/JSON_UNESCAPED_UNICODE.php

@@ -0,0 +1,11 @@
+<?php
+
+$tests = array(
+  html_entity_decode('&#xd800;&#xdc00;', ENT_NOQUOTES, 'utf-8'),
+  htmlspecialchars_decode('&#x1D11E;')
+);
+foreach ($tests as $s) {
+  var_dump($s);
+  var_dump(json_encode($s));
+  var_dump(json_encode($s, JSON_UNESCAPED_UNICODE));
+}

hphp/test/slow/json/JSON_UNESCAPED_UNICODE.php.expect

@@ -0,0 +1,6 @@
+string(6) "������"
+string(4) "null"
+string(4) "null"
+string(4) "𝄞"
+string(14) ""\ud834\udd1e""
+string(6) ""𝄞""