Fix possible stack overflows in decoder for illegal bit streams

Fixes CVE-2018-0429
A vulnerability in the Thor decoder (available at:
https://github.com/cisco/thor) could allow an authenticated, local
attacker to cause segmentation faults and stack overflows when using a
non-conformant Thor bitstream as input.
The vulnerability is due to lack of input validation when parsing the
bitstream. A successful exploit could allow the attacker to cause a
stack overflow and potentially inject and execute arbitrary code.

Author: stemidts

dec/decode_block.c

@@ -650,7 +650,7 @@ void TEMPLATE(process_block_dec)(decoder_info_t *decoder_info,int size,int yposY
 
   decoder_info->bit_count.super_mode[decoder_info->bit_count.stat_frame_type] += (stream->bitcnt - bit_start);
 
-  if (split_flag){
+  if (split_flag && size >= MIN_BLOCK_SIZE){
     int new_size = size/2;
     TEMPLATE(process_block_dec)(decoder_info,new_size,yposY+0*new_size,xposY+0*new_size,sub);
     TEMPLATE(process_block_dec)(decoder_info,new_size,yposY+1*new_size,xposY+0*new_size,sub);

dec/read_bits.c

@@ -50,6 +50,7 @@ void read_sequence_header(decoder_info_t *decoder_info, stream_t *stream) {
   decoder_info->width = get_flc(16, stream);
   decoder_info->height = get_flc(16, stream);
   decoder_info->log2_sb_size = get_flc(3, stream);
+  decoder_info->log2_sb_size = clip(decoder_info->log2_sb_size, log2i(MIN_BLOCK_SIZE), log2i(MAX_SB_SIZE));
   decoder_info->pb_split = get_flc(1, stream);
   decoder_info->tb_split_enable = get_flc(1, stream);
   decoder_info->max_num_ref = get_flc(2, stream) + 1;