1
0
mirror of https://github.com/ultrajson/ultrajson.git synced 2024-12-04 06:38:23 +01:00

Fix unchecked buffer overflows (CVE-2021-45958).

Add a few extra memory reserve calls to account for the extra space that
indentation needs.

These kinds of memory issues are hard to spot because the buffer is resized in
powers of 2 meaning that a miscalculation would only show any symptoms if the
required buffer size is estimated to be just below a 2 power but is actually
just above. Add a debug mode which replaces the 2 power scheme with reserving
only the memory explicitly requested and adds some overflow checks.
This commit is contained in:
Brénainn Woodsend 2022-02-09 20:44:01 +00:00
parent 881ee9317b
commit 61dd6f19e8
4 changed files with 944 additions and 9 deletions

@ -31,9 +31,17 @@ jobs:
python -m pip install -U pip
python -m pip install -U pytest
python -m pip install .
env:
CFLAGS: '-DDEBUG'
- name: Tests
run: |
pytest -s
- name: Test without debug mode
run: |
git clean -Xfd
python -m pip install --force-reinstall .
pytest
- name: Test with coverage

@ -41,6 +41,7 @@ https://opensource.apple.com/source/tcl/tcl-14/tcl/license.terms
#include <assert.h>
#include <string.h>
#include <stdlib.h>
#include <stddef.h>
#include <math.h>
#include <float.h>
@ -113,14 +114,25 @@ FIXME: Keep track of how big these get across several encoder calls and try to m
That way we won't run our head into the wall each call */
static void Buffer_Realloc (JSONObjectEncoder *enc, size_t cbNeeded)
{
size_t free_space = enc->end - enc->offset;
if (free_space >= cbNeeded)
{
return;
}
size_t curSize = enc->end - enc->start;
size_t newSize = curSize * 2;
size_t newSize = curSize;
size_t offset = enc->offset - enc->start;
#ifdef DEBUG
// In debug mode, allocate only what is requested so that any miscalculation
// shows up plainly as a crash.
newSize = (enc->offset - enc->start) + cbNeeded;
#else
while (newSize < curSize + cbNeeded)
{
newSize *= 2;
}
#endif
if (enc->heap)
{
@ -147,6 +159,12 @@ static void Buffer_Realloc (JSONObjectEncoder *enc, size_t cbNeeded)
enc->end = enc->start + newSize;
}
#define Buffer_Reserve(__enc, __len) \
if ( (size_t) ((__enc)->end - (__enc)->offset) < (size_t) (__len)) \
{ \
Buffer_Realloc((__enc), (__len));\
} \
static FASTCALL_ATTR INLINE_PREFIX void FASTCALL_MSVC Buffer_AppendShortHexUnchecked (char *outputOffset, unsigned short value)
{
*(outputOffset++) = g_hexChars[(value & 0xf000) >> 12];
@ -261,11 +279,19 @@ static int Buffer_EscapeStringUnvalidated (JSONObjectEncoder *enc, const char *i
static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, const char *io, const char *end)
{
Buffer_Reserve(enc, RESERVE_STRING(end - io));
JSUTF32 ucs;
char *of = (char *) enc->offset;
for (;;)
{
#ifdef DEBUG
if ((io < end) && (enc->end - of < RESERVE_STRING(1))) {
fprintf(stderr, "Ran out of buffer space during Buffer_EscapeStringValidated()\n");
abort();
}
#endif
JSUINT8 utflen = g_asciiOutputTable[(unsigned char) *io];
switch (utflen)
@ -487,15 +513,28 @@ static int Buffer_EscapeStringValidated (JSOBJ obj, JSONObjectEncoder *enc, cons
}
}
#define Buffer_Reserve(__enc, __len) \
if ( (size_t) ((__enc)->end - (__enc)->offset) < (size_t) (__len)) \
{ \
Buffer_Realloc((__enc), (__len));\
} \
#define Buffer_AppendCharUnchecked(__enc, __chr) \
*((__enc)->offset++) = __chr; \
static FASTCALL_ATTR INLINE_PREFIX void FASTCALL_MSVC Buffer_AppendCharUnchecked(JSONObjectEncoder *enc, char chr)
{
#ifdef DEBUG
if (enc->end <= enc->offset)
{
fprintf(stderr, "Overflow writing byte %d '%c'. The last few characters were:\n'''", chr, chr);
char * recent = enc->offset - 1000;
if (enc->start > recent)
{
recent = enc->start;
}
for (; recent < enc->offset; recent++)
{
fprintf(stderr, "%c", *recent);
}
fprintf(stderr, "'''\n");
abort();
}
#endif
*(enc->offset++) = chr;
}
static FASTCALL_ATTR INLINE_PREFIX void FASTCALL_MSVC strreverse(char* begin, char* end)
{
@ -670,6 +709,7 @@ static void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, size_t c
iterObj = enc->iterGetValue(obj, &tc);
enc->level ++;
Buffer_Reserve (enc, enc->indent * enc->level);
Buffer_AppendIndentUnchecked (enc, enc->level);
encode (iterObj, enc, NULL, 0);
if (enc->errorMsg)
@ -685,6 +725,9 @@ static void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, size_t c
enc->iterEnd(obj, &tc);
if (count > 0) {
// Reserve space for the indentation plus the newline and the closing
// bracket.
Buffer_Reserve (enc, enc->indent * enc->level + 2);
Buffer_AppendIndentNewlineUnchecked (enc);
Buffer_AppendIndentUnchecked (enc, enc->level);
}
@ -700,6 +743,7 @@ static void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, size_t c
while ((res = enc->iterNext(obj, &tc)))
{
Buffer_Reserve (enc, 3 + (enc->indent * (enc->level + 1)));
if(res < 0)
{
enc->iterEnd(obj, &tc);
@ -721,6 +765,7 @@ static void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, size_t c
objName = enc->iterGetName(obj, &tc, &szlen);
enc->level ++;
Buffer_Reserve (enc, enc->indent * enc->level);
Buffer_AppendIndentUnchecked (enc, enc->level);
encode (iterObj, enc, objName, szlen);
if (enc->errorMsg)
@ -736,6 +781,7 @@ static void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, size_t c
enc->iterEnd(obj, &tc);
if (count > 0) {
Buffer_Reserve (enc, enc->indent * enc->level + 4);
Buffer_AppendIndentNewlineUnchecked (enc);
Buffer_AppendIndentUnchecked (enc, enc->level);
}

857
tests/334-reproducer.json Normal file

@ -0,0 +1,857 @@
{
"ak.somestring.internal.Shadow": {
"id": 33300002,
"init_state": "(bk.action.array.Make, (bk.action.i32.Const, 0))",
"child": {
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "stretch",
"children": [
{
"ak.somestring.Collection": {
"id": 33300001,
"snap": "center",
"direction": "row",
"children": [
{
"ak.somestring.Flexbox": {
"decoration": {
"ak.somestring.BoxDecoration": {
"background": {
"ak.somestring.ColorDrawable": {
"color": "#2c8932"
}
}
}
},
"children": [
{
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "stretch",
"children": [
{
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "stretch",
"children": [
{
"ak.somestring.Flexbox": {
"children": [
{
"ls.components.Image": {
"media_id": "10156403921218138",
"preview_url": "https://scontent.xx.whoaa.net/v/t1.0-9/51099660_10156403921233138_3677795704043995136_n.jpg?_nc_cat=102&_nc_log=1&_nc_oc=AQk3Td-w9KpopLL2N1jgZ4WDMuxUyuGY3ZvY4mDSCk8W9-GjsFPi2S4gVQk0Y3A5ZaaQf7ASvQ2s_eR85kTmFvr0&_nc_ad=z-m&_nc_cid=0&_nc_zor=9&_nc_ht=scontent.xx&oh=fb16b0d60b13817a505f583cc9dad1eb&oe=5CBCDB46",
"height": 278,
"width": 156
}
}
],
"_style": {
"flex": {
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "stretch",
"children": [
{
"ak.somestring.Flexbox": {
"flex_direction": "row",
"align_items": "stretch",
"children": [
{
"ak.somestring.Flexbox": {
"decoration": {
"ak.somestring.BoxDecoration": {
"background": {
"ak.somestring.ColorDrawable": {
"color": "#ffffff"
}
}
}
},
"_style": {
"flex": {
"margin_right": "4dp",
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"decoration": {
"ak.somestring.BoxDecoration": {
"background": {
"ak.somestring.ColorDrawable": {
"color": "#ffffff"
}
}
}
},
"_style": {
"flex": {
"margin_right": "4dp",
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"flex_direction": "row",
"align_items": "stretch",
"decoration": {
"ak.somestring.BoxDecoration": {
"background": {
"ak.somestring.ColorDrawable": {
"color": "#ffffff"
}
}
}
},
"children": [
{
"ak.somestring.Flexbox": {
"id": 33300004,
"_style": {
"flex": {
"grow": 1
}
}
}
}
],
"_style": {
"flex": {
"margin_right": "4dp",
"grow": 1
}
}
}
}
],
"_style": {
"flex": {
"height": "2dp",
"margin_left": "4dp"
}
}
}
}
],
"_style": {
"flex": {
"position_type": "absolute",
"left": "0dp",
"top": "10dp",
"margin_top": "10dp",
"right": "0dp",
"height": "2dp",
"width": "100%"
}
}
}
}
],
"_style": {
"flex": {
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"align_items": "flex_start",
"children": [
{
"ak.somestring.Flexbox": {
"decoration": {
"ak.somestring.BoxDecoration": {
"corner_radius": "17dp"
}
},
"children": [
{
"ls.components.Image": {
"media_id": "10156403921218138",
"preview_url": "https://scontent.xx.whoaa.net/v/t1.0-9/51099660_10156403921233138_3677795704043995136_n.jpg?_nc_cat=102&_nc_log=1&_nc_oc=AQk3Td-w9KpopLL2N1jgZ4WDMuxUyuGY3ZvY4mDSCk8W9-GjsFPi2S4gVQk0Y3A5ZaaQf7ASvQ2s_eR85kTmFvr0&_nc_ad=z-m&_nc_cid=0&_nc_zor=9&_nc_ht=scontent.xx&oh=fb16b0d60b13817a505f583cc9dad1eb&oe=5CBCDB46",
"height": 34,
"width": 34,
"_style": {
"flex": {
"width": "34dp",
"height": "34dp"
}
}
}
}
],
"_style": {
"flex": {
"margin_right": "12dp",
"width": "34dp",
"height": "34dp"
}
}
}
},
{
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "flex_start",
"children": [
{
"ak.somestring.RichText": {
"children": [
{
"ak.somestring.TextSpan": {
"text": "eric",
"text_size": "15sp",
"text_style": "bold",
"text_color": "#ffffff"
}
}
],
"_style": {
"flex": {
"margin_bottom": "2dp",
"width": "100%"
}
}
}
},
{
"ak.somestring.RichText": {
"children": [
{
"ak.somestring.TextSpan": {
"text": "8h",
"text_size": "13sp",
"text_style": "normal",
"text_color": "#ffffff"
}
}
],
"_style": {
"flex": {
"width": "100%"
}
}
}
}
],
"_style": {
"flex": {
"width": "100%",
"height": "100%"
}
}
}
}
],
"_style": {
"flex": {
"position_type": "absolute",
"top": "30dp",
"left": "10dp",
"height": "48dp"
}
}
}
},
{
"ak.somestring.Flexbox": {
"children": [
{
"ls.components.StoriesReplyBar": {}
}
],
"_style": {
"flex": {
"width": "100%",
"height": "45dp",
"margin_top": "auto",
"margin_bottom": "auto"
}
}
}
}
],
"_style": {
"flex": {
"position_type": "absolute",
"width": "100%",
"height": "100%",
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "stretch",
"children": [
{
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "stretch",
"children": [
{
"ak.somestring.Flexbox": {
"children": [
{
"ls.components.Image": {
"media_id": "10101230968216658",
"preview_url": "https://scontent.xx.whoaa.net/v/t1.0-9/50800535_10101230968226638_6755212111762161664_n.jpg?_nc_cat=101&_nc_log=1&_nc_oc=AQmKcqYvt6DI7aeGk3k_oF6RHSVZkUg7f9hnBCWilyaOGdCWO0-u9_zssC5qGvca6wqsrz3AP0y1RPLPiZj8ycCv&_nc_ad=z-m&_nc_cid=0&_nc_zor=9&_nc_ht=scontent.xx&oh=2fffbab8f0a102d196454ee0138c1850&oe=5CC15206",
"height": 278,
"width": 156
}
}
],
"_style": {
"flex": {
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "stretch",
"children": [
{
"ak.somestring.Flexbox": {
"flex_direction": "row",
"align_items": "stretch",
"children": [
{
"ak.somestring.Flexbox": {
"decoration": {
"ak.somestring.BoxDecoration": {
"background": {
"ak.somestring.ColorDrawable": {
"color": "#ffffff"
}
}
}
},
"_style": {
"flex": {
"margin_right": "4dp",
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"flex_direction": "row",
"align_items": "stretch",
"decoration": {
"ak.somestring.BoxDecoration": {
"background": {
"ak.somestring.ColorDrawable": {
"color": "#ffffff"
}
}
}
},
"children": [
{
"ak.somestring.Flexbox": {
"id": 33300005,
"_style": {
"flex": {
"grow": 1
}
}
}
}
],
"_style": {
"flex": {
"margin_right": "4dp",
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"decoration": {
"ak.somestring.BoxDecoration": {
"background": {
"ak.somestring.ColorDrawable": {
"color": "#cccccc"
}
}
}
},
"_style": {
"flex": {
"margin_right": "4dp",
"grow": 1
}
}
}
}
],
"_style": {
"flex": {
"height": "2dp",
"margin_left": "4dp"
}
}
}
}
],
"_style": {
"flex": {
"position_type": "absolute",
"left": "0dp",
"top": "10dp",
"margin_top": "10dp",
"right": "0dp",
"height": "2dp",
"width": "100%"
}
}
}
}
],
"_style": {
"flex": {
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"align_items": "flex_start",
"children": [
{
"ak.somestring.Flexbox": {
"decoration": {
"ak.somestring.BoxDecoration": {
"corner_radius": "17dp"
}
},
"children": [
{
"ls.components.Image": {
"media_id": "10101230968216658",
"preview_url": "https://scontent.xx.whoaa.net/v/t1.0-9/50800535_10101230968226638_6755212111762161664_n.jpg?_nc_cat=101&_nc_log=1&_nc_oc=AQmKcqYvt6DI7aeGk3k_oF6RHSVZkUg7f9hnBCWilyaOGdCWO0-u9_zssC5qGvca6wqsrz3AP0y1RPLPiZj8ycCv&_nc_ad=z-m&_nc_cid=0&_nc_zor=9&_nc_ht=scontent.xx&oh=2fffbab8f0a102d196454ee0138c1850&oe=5CC15206",
"height": 34,
"width": 34,
"_style": {
"flex": {
"width": "34dp",
"height": "34dp"
}
}
}
}
],
"_style": {
"flex": {
"margin_right": "12dp",
"width": "34dp",
"height": "34dp"
}
}
}
},
{
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "flex_start",
"children": [
{
"ak.somestring.RichText": {
"children": [
{
"ak.somestring.TextSpan": {
"text": "eric",
"text_size": "15sp",
"text_style": "bold",
"text_color": "#ffffff"
}
}
],
"_style": {
"flex": {
"margin_bottom": "2dp",
"width": "100%"
}
}
}
},
{
"ak.somestring.RichText": {
"children": [
{
"ak.somestring.TextSpan": {
"text": "2h",
"text_size": "13sp",
"text_style": "normal",
"text_color": "#ffffff"
}
}
],
"_style": {
"flex": {
"width": "100%"
}
}
}
}
],
"_style": {
"flex": {
"width": "100%",
"height": "100%"
}
}
}
}
],
"_style": {
"flex": {
"position_type": "absolute",
"top": "30dp",
"left": "10dp",
"height": "48dp"
}
}
}
},
{
"ak.somestring.Flexbox": {
"children": [
{
"ls.components.StoriesReplyBar": {}
}
],
"_style": {
"flex": {
"width": "100%",
"height": "45dp",
"margin_top": "auto",
"margin_bottom": "auto"
}
}
}
}
],
"_style": {
"flex": {
"position_type": "absolute",
"width": "100%",
"height": "100%",
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "stretch",
"children": [
{
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "stretch",
"children": [
{
"ak.somestring.Flexbox": {
"children": [
{
"ls.components.Video": {
"media_id": "10156395664922983",
"video_url": "https://video.xx.whoaa.net/v/t42.9040-2/51636103_316525608877874_407931582842667008_n.mp4?_nc_cat=109&efg=eyJ2ZW5jb2RlX3RhZyI6InN2ZV9oZCJ9&_nc_log=1&_nc_oc=AQm6aMctRAFdMe3C66upF2JulQP4mV3Hd4THkueZex952PR389F6Ay9XHm1S40dV1x7M1I-fAW5y3iH7JlQ3MgDM&_nc_ht=video.xx&oh=e17b1f7ec67619d57a5b1cda5e076fef&oe=5C587F7D",
"preview_url": "https://scontent.xx.whoaa.net/v/t15.5256-10/s960x960/51767715_10156395667952983_4168426706077483008_n.jpg?_nc_cat=104&_nc_log=1&_nc_oc=AQnVwEZk2vG8Q3TcoR0SxdXSi8rL_GaST2aH3i9auDcDnJNTRKvuYEFfd_qKGBhmD4-bo-f8BY5j9jHyit765O7P&_nc_ad=z-m&_nc_cid=0&_nc_zor=9&_nc_ht=scontent.xx&oh=9a17e4bcf8a2a9aabc21d2ecf9f8611b&oe=5CB3D14B",
"show_media_play_button": false,
"media_height": 960,
"media_width": 540
}
}
],
"_style": {
"flex": {
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "stretch",
"children": [
{
"ak.somestring.Flexbox": {
"flex_direction": "row",
"align_items": "stretch",
"children": [
{
"ak.somestring.Flexbox": {
"flex_direction": "row",
"align_items": "stretch",
"decoration": {
"ak.somestring.BoxDecoration": {
"background": {
"ak.somestring.ColorDrawable": {
"color": "#ffffff"
}
}
}
},
"children": [
{
"ak.somestring.Flexbox": {
"id": 33300006,
"_style": {
"flex": {
"grow": 1
}
}
}
}
],
"_style": {
"flex": {
"margin_right": "4dp",
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"decoration": {
"ak.somestring.BoxDecoration": {
"background": {
"ak.somestring.ColorDrawable": {
"color": "#cccccc"
}
}
}
},
"_style": {
"flex": {
"margin_right": "4dp",
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"decoration": {
"ak.somestring.BoxDecoration": {
"background": {
"ak.somestring.ColorDrawable": {
"color": "#cccccc"
}
}
}
},
"_style": {
"flex": {
"margin_right": "4dp",
"grow": 1
}
}
}
}
],
"_style": {
"flex": {
"height": "2dp",
"margin_left": "4dp"
}
}
}
}
],
"_style": {
"flex": {
"position_type": "absolute",
"left": "0dp",
"top": "10dp",
"margin_top": "10dp",
"right": "0dp",
"height": "2dp",
"width": "100%"
}
}
}
}
],
"_style": {
"flex": {
"grow": 1
}
}
}
},
{
"ak.somestring.Flexbox": {
"align_items": "flex_start",
"children": [
{
"ak.somestring.Flexbox": {
"decoration": {
"ak.somestring.BoxDecoration": {
"corner_radius": "17dp"
}
},
"children": [
{
"ls.components.Image": {
"media_id": "10156395664922983",
"height": 34,
"width": 34,
"_style": {
"flex": {
"width": "34dp",
"height": "34dp"
}
}
}
}
],
"_style": {
"flex": {
"margin_right": "12dp",
"width": "34dp",
"height": "34dp"
}
}
}
},
{
"ak.somestring.Flexbox": {
"flex_direction": "column",
"align_items": "flex_start",
"children": [
{
"ak.somestring.RichText": {
"children": [
{
"ak.somestring.TextSpan": {
"text": "eric",
"text_size": "15sp",
"text_style": "bold",
"text_color": "#ffffff"
}
}
],
"_style": {
"flex": {
"margin_bottom": "2dp",
"width": "100%"
}
}
}
},
{
"ak.somestring.RichText": {
"children": [
{
"ak.somestring.TextSpan": {
"text": "20h",
"text_size": "13sp",
"text_style": "normal",
"text_color": "#ffffff"
}
}
],
"_style": {
"flex": {
"width": "100%"
}
}
}
}
],
"_style": {
"flex": {
"width": "100%",
"height": "100%"
}
}
}
}
],
"_style": {
"flex": {
"position_type": "absolute",
"top": "30dp",
"left": "10dp",
"height": "48dp"
}
}
}
},
{
"ak.somestring.Flexbox": {
"children": [
{
"ls.components.StoriesReplyBar": {}
}
],
"_style": {
"flex": {
"width": "100%",
"height": "45dp",
"margin_top": "auto",
"margin_bottom": "auto"
}
}
}
}
],
"_style": {
"flex": {
"position_type": "absolute",
"width": "100%",
"height": "100%",
"grow": 1
}
}
}
}
],
"_style": {
"flex": {
"width": "100%",
"height": "100%"
}
}
}
}
],
"_style": {
"flex": {
"height": "100%"
}
}
}
}
]
}
}
}
}

@ -7,6 +7,7 @@ import re
import sys
import uuid
from collections import OrderedDict
from pathlib import Path
import pytest
import ujson
@ -947,6 +948,29 @@ def test_default_function():
ujson.dumps(unjsonable_obj, default=default)
def test_dump_huge_indent():
ujson.encode({"a": True}, indent=65539)
def test_dump_long_string():
ujson.dumps(["aaaa", "\x00" * 10921])
def test_dump_indented_nested_list():
a = _a = []
for i in range(20):
_a.append(list(range(i)))
_a = _a[-1]
ujson.dumps(a, indent=i)
@pytest.mark.parametrize("indent", [0, 1, 2, 4, 5, 8, 49])
def test_issue_334(indent):
path = Path(__file__).with_name("334-reproducer.json")
a = ujson.loads(path.read_bytes())
ujson.dumps(a, indent=indent)
"""
def test_decode_numeric_int_frc_overflow():
input = "X.Y"