rtmp: fix multiple broken overflow checks

Sanity checks like `data + size >= data_end || data + size < data' are
broken, because `data + size < data' assumes pointer overflow, which is
undefined behavior in C.  Many compilers such as gcc/clang optimize such
checks away.

Use `size < 0 || size >= data_end - data' instead.

Signed-off-by: Xi Wang <xi.wang@gmail.com>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit 902cfe2f74d777a7dc20ac68f2393b9f84b790c1)

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
This commit is contained in:
Xi Wang 2013-01-22 17:49:29 -05:00 committed by Michael Niedermayer
parent 165f783235
commit 69b3fedc09

View File

@ -279,11 +279,11 @@ int ff_amf_tag_size(const uint8_t *data, const uint8_t *data_end)
data++; data++;
break; break;
} }
if (data + size >= data_end || data + size < data) if (size < 0 || size >= data_end - data)
return -1; return -1;
data += size; data += size;
t = ff_amf_tag_size(data, data_end); t = ff_amf_tag_size(data, data_end);
if (t < 0 || data + t >= data_end) if (t < 0 || t >= data_end - data)
return -1; return -1;
data += t; data += t;
} }
@ -312,7 +312,7 @@ int ff_amf_get_field_value(const uint8_t *data, const uint8_t *data_end,
int size = bytestream_get_be16(&data); int size = bytestream_get_be16(&data);
if (!size) if (!size)
break; break;
if (data + size >= data_end || data + size < data) if (size < 0 || size >= data_end - data)
return -1; return -1;
data += size; data += size;
if (size == namelen && !memcmp(data-size, name, namelen)) { if (size == namelen && !memcmp(data-size, name, namelen)) {
@ -333,7 +333,7 @@ int ff_amf_get_field_value(const uint8_t *data, const uint8_t *data_end,
return 0; return 0;
} }
len = ff_amf_tag_size(data, data_end); len = ff_amf_tag_size(data, data_end);
if (len < 0 || data + len >= data_end || data + len < data) if (len < 0 || len >= data_end - data)
return -1; return -1;
data += len; data += len;
} }
@ -404,13 +404,13 @@ static void ff_amf_tag_contents(void *ctx, const uint8_t *data, const uint8_t *d
data++; data++;
break; break;
} }
if (data + size >= data_end || data + size < data) if (size < 0 || size >= data_end - data)
return; return;
data += size; data += size;
av_log(ctx, AV_LOG_DEBUG, " %s: ", buf); av_log(ctx, AV_LOG_DEBUG, " %s: ", buf);
ff_amf_tag_contents(ctx, data, data_end); ff_amf_tag_contents(ctx, data, data_end);
t = ff_amf_tag_size(data, data_end); t = ff_amf_tag_size(data, data_end);
if (t < 0 || data + t >= data_end) if (t < 0 || t >= data_end - data)
return; return;
data += t; data += t;
} }