fix bug when rotating phone in MediaCodec H264 encoder, resulting in video...

fix bug when rotating phone in MediaCodec H264 encoder, resulting in video being sent within a green rectangle.

src/android/mediacodech264dec.c

@@ -276,7 +276,7 @@ static void dec_flush(DecData *d, bool_t with_reset){
 static void dec_process(MSFilter *f) {
 	DecData *d = (DecData *)f->data;
 	MSPicture pic = {0};
-	mblk_t *im, *om = NULL;
+	mblk_t *im = NULL;
 	ssize_t oBufidx = -1;
 	size_t bufsize;
 	bool_t need_reinit = FALSE;
@@ -387,7 +387,8 @@ static void dec_process(MSFilter *f) {
 	/*secondly try to get decoded frames from the decoder, this is performed every tick*/
 	while (d->buffer_queued && (oBufidx = AMediaCodec_dequeueOutputBuffer(d->codec, &info, TIMEOUT_US)) >= 0) {
 		AMediaFormat *format;
-		int width = 0, height = 0, color = 0;
+		mblk_t *om = NULL;
+		int color = 0;
 		uint8_t *buf = AMediaCodec_getOutputBuffer(d->codec, oBufidx, &bufsize);
 
 		if (buf == NULL) {
@@ -395,31 +396,38 @@ static void dec_process(MSFilter *f) {
 			continue;
 		}
 
-		format = AMediaCodec_getOutputFormat(d->codec);
-
-		if (format != NULL) {
-			AMediaFormat_getInt32(format, "width", &width);
-			AMediaFormat_getInt32(format, "height", &height);
-			AMediaFormat_getInt32(format, "color-format", &color);
-
-			d->vsize.width = width;
-			d->vsize.height = height;
-			AMediaFormat_delete(format);
-		}
-
-		if (width != 0 && height != 0) {
-			if (d->useMediaImage) {
-				AMediaImage image;
+		if (d->useMediaImage) {
+			AMediaImage image;
+			
+			if (AMediaCodec_getOutputImage(d->codec, oBufidx, &image)) {
 				int dst_pix_strides[4] = {1, 1, 1, 1};
-				MSRect dst_roi = {0, 0, pic.w, pic.h};
+				MSRect dst_roi = {0, 0, image.crop_rect.w, image.crop_rect.h};
+				
+				d->vsize.width = image.crop_rect.w;
+				d->vsize.height = image.crop_rect.h;
+				
+				om = ms_yuv_buf_allocator_get(d->buf_allocator, &pic, d->vsize.width, d->vsize.height);
+				ms_yuv_buf_copy_with_pix_strides(image.buffers, image.row_strides, image.pixel_strides, image.crop_rect,
+													pic.planes, pic.strides, dst_pix_strides, dst_roi);
+				AMediaImage_close(&image);
+			}else{
+				ms_error("AMediaCodec_getOutputImage() failed");
+			}
+		} else {
+			int width = 0, height = 0;
+			
+			format = AMediaCodec_getOutputFormat(d->codec);
 
-				if (AMediaCodec_getOutputImage(d->codec, oBufidx, &image)) {
-					om = ms_yuv_buf_allocator_get(d->buf_allocator, &pic, width, height);
-					ms_yuv_buf_copy_with_pix_strides(image.buffers, image.row_strides, image.pixel_strides, image.crop_rect,
-														pic.planes, pic.strides, dst_pix_strides, dst_roi);
-					AMediaImage_close(&image);
-				}
-			} else {
+			if (format != NULL) {
+				AMediaFormat_getInt32(format, "width", &width);
+				AMediaFormat_getInt32(format, "height", &height);
+				AMediaFormat_getInt32(format, "color-format", &color);
+
+				AMediaFormat_delete(format);
+			}
+			if (width != 0 && height != 0) {
+				d->vsize.width = width;
+				d->vsize.height = height;
 				if (color == 19) {
 					//YUV
 					int ysize = width * height;
@@ -432,19 +440,22 @@ static void dec_process(MSFilter *f) {
 					uint8_t *cbcr_src = (uint8_t *)(buf + width * height);
 					om = copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(d->buf_allocator, buf, cbcr_src, 0, width, height, width, width, TRUE, FALSE);
 				}
+			} else {
+				ms_error("MSMediaCodecH264Dec: width and height are not known !");
 			}
+		}
+		if (om){
 
 			if (!d->first_image_decoded) {
-				ms_message("First frame decoded %ix%i", width, height);
+				ms_message("First frame decoded %ix%i", d->vsize.width, d->vsize.height);
 				d->first_image_decoded = TRUE;
 				ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_FIRST_IMAGE_DECODED);
 			}
 
 			ms_average_fps_update(&d->fps, f->ticker->time);
 			ms_queue_put(f->outputs[0], om);
-		} else {
-			ms_error("MSMediaCodecH264Dec: width and height are not known !");
 		}
+		
 		AMediaCodec_releaseOutputBuffer(d->codec, oBufidx, FALSE);
 	}
 

src/android/mediacodech264enc.c

@@ -273,11 +273,10 @@ static void enc_process(MSFilter *f) {
 								int src_pix_strides[4] = {1, 1, 1, 1};
 								ms_yuv_buf_copy_with_pix_strides(pic.planes, pic.strides, src_pix_strides, src_roi, image.buffers, image.row_strides, image.pixel_strides, image.crop_rect);
 								bufsize = image.row_strides[0] * image.height * 3 / 2;
-								AMediaImage_close(&image);
 							} else {
 								ms_error("%s: encoder requires non YUV420 format", f->desc->name);
-								AMediaImage_close(&image);
 							}
+							AMediaImage_close(&image);
 						}
 					} else {
 						if (d->isPlanar) {
@@ -405,7 +404,7 @@ static int enc_set_configuration(MSFilter *f, void *arg) {
 	
 	if (d->codec_started){
 		AMediaFormat *afmt = AMediaFormat_new();
-		/*Force a key-frame*/
+		/*Update the output bitrate*/
 		ms_filter_lock(f);
 		AMediaFormat_setInt32(afmt, "video-bitrate", d->vconf.required_bitrate);
 		AMediaCodec_setParams(d->codec, afmt);
@@ -422,10 +421,10 @@ static int enc_set_br(MSFilter *f, void *arg) {
 	if (d->codec_started) {
 		/* Encoding is already ongoing, do not change video size, only bitrate. */
 		d->vconf.required_bitrate = br;
-		/* TODO apply the new bitrate request to the running MediaCodec*/
+		/* apply the new bitrate request to the running MediaCodec*/
 		enc_set_configuration(f, &d->vconf);
 	} else {
-		MSVideoConfiguration best_vconf = ms_video_find_best_configuration_for_bitrate(d->vconf_list, br, ms_factory_get_cpu_count(f->factory));
+		MSVideoConfiguration best_vconf = ms_video_find_best_configuration_for_size_and_bitrate(d->vconf_list, d->vconf.vsize, ms_factory_get_cpu_count(f->factory),  br);
 		enc_set_configuration(f, &best_vconf);
 	}