libtheora: cherry-picked some fixes from upstream:

- Sanity improvements to oc_dec_headerin() error returns. (50fd339b4)
- Fix mismatched oc_mb_fill_cmapping11 signature. (12f20c7a5)
- Correct an operator precedence error in the visualization code.  (6b94bf905)
- Remove some redundant NULL checks (de386d3ea)

media/libtheora/lib/decinfo.c

@@ -174,7 +174,23 @@ static int oc_dec_headerin(oc_pack_buf *_opb,th_info *_info,
   packtype=(int)val;
   /*If we're at a data packet and we have received all three headers, we're
      done.*/
-  if(!(packtype&0x80)&&_info->frame_width>0&&_tc->vendor!=NULL&&*_setup!=NULL){
+  /*If we're at a data packet...*/
+  if(!(packtype&0x80)){
+    /*Check to make sure we received all three headers...
+      If we haven't seen any valid headers, assume this is not actually
+       Theora.*/
+    if(_info->frame_width<=0)return TH_ENOTFORMAT;
+    /*Follow our documentation, which says we'll return TH_EFAULT if this
+       are NULL (_info was checked by our caller).*/
+    if(_tc==NULL)return TH_EFAULT;
+    /*And if any other headers were missing, declare this packet "out of
+       sequence" instead.*/
+    if(_tc->vendor==NULL)return TH_EBADHEADER;
+    /*Don't check this until it's needed, since we allow passing NULL for the
+       arguments that we're not expecting the next header to fill in yet.*/
+    if(_setup==NULL)return TH_EFAULT;
+    if(*_setup==NULL)return TH_EBADHEADER;
+    /*If we got everything, we're done.*/
     return 0;
   }
   /*Check the codec string.*/

media/libtheora/lib/decode.c

@@ -2705,8 +2705,8 @@ int th_decode_ycbcr_out(th_dec_ctx *_dec,th_ycbcr_buffer _ycbcr){
                 }break;
                 /*Plus:*/
                 case 1:{
-                  if(bi&2==0)yp-=2;
-                  if(bi&1==0)xp-=2;
+                  if((bi&2)==0)yp-=2;
+                  if((bi&1)==0)xp-=2;
                   cairo_move_to(c,xp+4.5,yp+2.5);
                   cairo_line_to(c,xp+4.5,yp+6.5);
                   cairo_move_to(c,xp+2.5,yp+4.5);

media/libtheora/lib/internal.c

@@ -131,7 +131,6 @@ void **oc_malloc_2d(size_t _height,size_t _width,size_t _sz){
   datsz=rowsz*_height;
   /*Alloc array and row pointers.*/
   ret=(char *)_ogg_malloc(datsz+colsz);
-  if(ret==NULL)return NULL;
   /*Initialize the array.*/
   if(ret!=NULL){
     size_t   i;
@@ -154,7 +153,6 @@ void **oc_calloc_2d(size_t _height,size_t _width,size_t _sz){
   datsz=rowsz*_height;
   /*Alloc array and row pointers.*/
   ret=(char *)_ogg_calloc(datsz+colsz,1);
-  if(ret==NULL)return NULL;
   /*Initialize the array.*/
   if(ret!=NULL){
     size_t   i;

media/libtheora/lib/state.c

@@ -253,10 +253,14 @@ static void oc_mb_fill_cmapping10(oc_mb_map_plane _mb_map[3],
   This version is for use with no chroma decimation (4:4:4).
   This uses the already filled-in luma plane values.
   _mb_map:  The macro block map to fill.
-  _fplanes: The descriptions of the fragment planes.*/
+  _fplanes: The descriptions of the fragment planes.
+  _xfrag0:  The X location of the upper-left hand fragment in the luma plane.
+  _yfrag0:  The Y location of the upper-left hand fragment in the luma plane.*/
 static void oc_mb_fill_cmapping11(oc_mb_map_plane _mb_map[3],
- const oc_fragment_plane _fplanes[3]){
+ const oc_fragment_plane _fplanes[3],int _xfrag0,int _yfrag0){
   int k;
+  (void)_xfrag0;
+  (void)_yfrag0;
   for(k=0;k<4;k++){
     _mb_map[1][k]=_mb_map[0][k]+_fplanes[1].froffset;
     _mb_map[2][k]=_mb_map[0][k]+_fplanes[2].froffset;
@@ -278,7 +282,7 @@ static const oc_mb_fill_cmapping_func OC_MB_FILL_CMAPPING_TABLE[4]={
   oc_mb_fill_cmapping00,
   oc_mb_fill_cmapping01,
   oc_mb_fill_cmapping10,
-  (oc_mb_fill_cmapping_func)oc_mb_fill_cmapping11
+  oc_mb_fill_cmapping11
 };
 
 /*Fills in the mapping from macro blocks to their corresponding fragment