fix crash in MediaCodec, and add better logs to diagnose problems

src/android/android_mediacodec.cpp

@@ -179,14 +179,17 @@ media_status_t AMediaCodec_start(AMediaCodec *codec){
 	jmethodID methodID = env->GetMethodID(mediaCodecClass, "start", "()V");
 	if (methodID != NULL){
 		env->CallVoidMethod(codec->jcodec, methodID);
-		handle_java_exception();
+		if (handle_java_exception()==-1) {
+			env->DeleteLocalRef(mediaCodecClass);
+			return AMEDIA_ERROR_BASE;
+		}
 	} else {
 		ms_error("start() not found in class MediaCodec !");
 		env->ExceptionClear();
 		env->DeleteLocalRef(mediaCodecClass);
 		return AMEDIA_ERROR_BASE;
 	}
-    env->DeleteLocalRef(mediaCodecClass);
+	env->DeleteLocalRef(mediaCodecClass);
 	return AMEDIA_OK;
 }
 
@@ -373,12 +376,12 @@ ssize_t AMediaCodec_dequeueInputBuffer(AMediaCodec *codec, int64_t timeoutUs){
 	jmethodID methodID = env->GetMethodID(mediaCodecClass,"dequeueInputBuffer","(J)I");
 	if (methodID != NULL){
 		jindex = env->CallIntMethod(codec->jcodec, methodID, timeoutUs);
-		handle_java_exception();
+		if (handle_java_exception() == -1) jindex = AMEDIA_ERROR_UNKNOWN; /*return value to notify the exception*/
+		/*otherwise, if -1 is returned as index, it just means that no buffer are available at this time (not an error)*/
 	} else {
-		ms_error("stop() not found in class mediacodec !");
+		ms_error("dequeueInputBuffer() not found in class mediacodec !");
 		env->ExceptionClear();
-		env->DeleteLocalRef(mediaCodecClass);
-		return -1;
+		jindex = -1;
 	}
 	env->DeleteLocalRef(mediaCodecClass);
 	return (ssize_t) jindex;
@@ -386,6 +389,8 @@ ssize_t AMediaCodec_dequeueInputBuffer(AMediaCodec *codec, int64_t timeoutUs){
 
 media_status_t AMediaCodec_queueInputBuffer(AMediaCodec *codec, size_t idx, off_t offset, size_t size, uint64_t time, uint32_t flags){
    	JNIEnv *env = ms_get_jni_env();
+	media_status_t ret = AMEDIA_OK;
+	
 	jclass mediaCodecClass = env->FindClass("android/media/MediaCodec");
 	if (mediaCodecClass == NULL){
 		ms_error("Couldn't find android/media/MediaCodec class !");
@@ -396,15 +401,16 @@ media_status_t AMediaCodec_queueInputBuffer(AMediaCodec *codec, size_t idx, off_
 	jmethodID methodID = env->GetMethodID(mediaCodecClass,"queueInputBuffer","(IIIJI)V");
 	if (methodID != NULL){
 		env->CallVoidMethod(codec->jcodec, methodID, idx, offset, size, time, flags);
-		handle_java_exception();
+		if (handle_java_exception() == -1){
+			ret = AMEDIA_ERROR_BASE;
+		}
 	} else {
 		ms_error("queueInputBuffer() not found in class mediacodec !");
 		env->ExceptionClear();
-		env->DeleteLocalRef(mediaCodecClass);
-		return AMEDIA_ERROR_BASE;
+		ret = AMEDIA_ERROR_BASE;
 	}
 	env->DeleteLocalRef(mediaCodecClass);
-	return AMEDIA_OK;
+	return ret;
 }
 
 ssize_t AMediaCodec_dequeueOutputBuffer(AMediaCodec *codec, AMediaCodecBufferInfo *info, int64_t timeoutUs) {
@@ -426,7 +432,7 @@ ssize_t AMediaCodec_dequeueOutputBuffer(AMediaCodec *codec, AMediaCodecBufferInf
 	} else {
 		ms_error("init not found in class MediaCodec$BufferInfo !");
     	env->ExceptionClear();
-    	return -1;
+    	return AMEDIA_ERROR_UNKNOWN;
 	}
 	env->DeleteLocalRef(mediaBufferInfoClass);
 
@@ -438,16 +444,20 @@ ssize_t AMediaCodec_dequeueOutputBuffer(AMediaCodec *codec, AMediaCodecBufferInf
 			env->ExceptionDescribe();
 			env->ExceptionClear();
 			ms_error("Exception");
+			jindex = AMEDIA_ERROR_UNKNOWN; /*return value to notify the exception*/
+			/*otherwise, if -1 is returned as index, it just means that no buffer are available at this time (not an error)*/
 		}
 	} else {
 		ms_error("dequeueOutputBuffer() not found in class format !");
 		env->ExceptionClear(); //very important.
-		return -1;
+		jindex = AMEDIA_ERROR_UNKNOWN;
 	}
 
-	info->size = env->GetIntField(jinfo,size);
-	info->offset = env->GetIntField(jinfo,offset);
-	info->flags = env->GetIntField(jinfo,flags);
+	if (jindex >= 0){
+		info->size = env->GetIntField(jinfo,size);
+		info->offset = env->GetIntField(jinfo,offset);
+		info->flags = env->GetIntField(jinfo,flags);
+	}
 	env->DeleteLocalRef(mediaCodecClass);
 	env->DeleteLocalRef(jinfo);
 	return (ssize_t) jindex;
@@ -475,6 +485,7 @@ AMediaFormat* AMediaCodec_getOutputFormat(AMediaCodec *codec){
 	}else{
 		ms_error("getOutputFormat() not found in class format !");
 		env->ExceptionClear(); //very important.
+		return NULL;
 	}
 	env->DeleteLocalRef(mediaCodecClass);
 	format->jformat = jformat;
@@ -494,11 +505,12 @@ media_status_t AMediaCodec_releaseOutputBuffer(AMediaCodec *codec, size_t idx, b
 	if (stopID != NULL){
 		env->CallVoidMethod(codec->jcodec,stopID,(int)idx,FALSE);
 		handle_java_exception();
-	if (env->ExceptionCheck()) {
-				env->ExceptionDescribe();
-				env->ExceptionClear();
-				ms_error("Exception");
-			}
+		if (env->ExceptionCheck()) {
+			env->ExceptionDescribe();
+			env->ExceptionClear();
+			ms_error("Exception");
+			return AMEDIA_ERROR_BASE;
+		}
 	} else {
 		ms_error("releaseOutputBuffer() not found in class format !");
 		env->ExceptionClear(); //very important.

src/android/mediacodech264dec.c

@@ -231,7 +231,7 @@ static void dec_process(MSFilter *f){
 	DecData *d=(DecData*)f->data;
 	MSPicture pic = {0};
 	mblk_t *im,*om = NULL;
-	ssize_t oBufidx;
+	ssize_t oBufidx = -1;
 	size_t bufsize;
 	bool_t need_reinit=FALSE;
 	bool_t request_pli=FALSE;
@@ -281,9 +281,11 @@ static void dec_process(MSFilter *f){
 					struct timespec ts;
 					clock_gettime(CLOCK_MONOTONIC, &ts);
 					memcpy(buf,d->bitstream,(size_t)size);
-					AMediaCodec_queueInputBuffer(d->codec, iBufidx, 0, (size_t)size, ts.tv_nsec/1000, 0);
+					AMediaCodec_queueInputBuffer(d->codec, iBufidx, 0, (size_t)size, (ts.tv_nsec/1000) + 10000LL, 0);
 					d->first_buffer_queued = TRUE;
 				}
+			}else if (iBufidx == AMEDIA_ERROR_UNKNOWN){
+				ms_error("MSMediaCodecH264Dec: AMediaCodec_dequeueInputBuffer() had an exception");
 			}
 		}
 		d->packet_num++;
@@ -338,6 +340,9 @@ static void dec_process(MSFilter *f){
 		}
 		AMediaCodec_releaseOutputBuffer(d->codec, oBufidx, FALSE);
 	}
+	if (oBufidx == AMEDIA_ERROR_UNKNOWN){
+		ms_error("MSMediaCodecH264Dec: AMediaCodec_dequeueOutputBuffer() had an exception");
+	}
 
 	if (d->avpf_enabled && request_pli) {
     	ms_filter_notify_no_arg(f, MS_VIDEO_DECODER_SEND_PLI);

src/android/mediacodech264enc.c

@@ -61,6 +61,16 @@ typedef struct _EncData{
 	bool isYUV;
 }EncData;
 
+static void set_mblk(mblk_t **packet, mblk_t *newone){
+	if (newone){
+		newone = copyb(newone);
+	}
+	if (*packet){
+		freemsg(*packet);
+	}
+	*packet = newone;
+}
+
 static void enc_init(MSFilter *f){
 	MSVideoSize vsize;
 	EncData *d=ms_new0(EncData,1);
@@ -119,25 +129,15 @@ static void enc_postprocess(MSFilter *f) {
 	rfc3984_destroy(d->packer);
 	AMediaCodec_flush(d->codec);
 	AMediaCodec_stop(d->codec);
+	AMediaCodec_delete(d->codec);
+	set_mblk(&d->sps, NULL);
+	set_mblk(&d->pps, NULL);
 	d->packer=NULL;
 }
 
-static void set_mblk(mblk_t **packet, mblk_t *newone){
-	if (newone){
-		newone = copyb(newone);
-	}
-	if (*packet){
-		freemsg(*packet);
-	}
-	*packet = newone;
-}
-
 static void enc_uninit(MSFilter *f){
 	EncData *d=(EncData*)f->data;
-	set_mblk(&d->sps, NULL);
-	set_mblk(&d->pps, NULL);
-	AMediaCodec_stop(d->codec);
-	AMediaCodec_delete(d->codec);
+	
 	ms_free(d);
 }
 
@@ -190,6 +190,8 @@ static void enc_process(MSFilter *f){
 					}
 					AMediaCodec_queueInputBuffer(d->codec, ibufidx, 0, (size_t)(pic.w * pic.h)*3/2, f->ticker->time*1000,0);
 				}
+			}else if (ibufidx == AMEDIA_ERROR_UNKNOWN){
+				ms_error("MSMediaCodecH264Enc: AMediaCodec_dequeueInputBuffer() had an exception");
 			}
 
 			bool have_seen_sps_pps = FALSE; /*this checks whether at a single timestamp point we dequeued SPS PPS before IDR*/
@@ -198,8 +200,9 @@ static void enc_process(MSFilter *f){
 				if (buf){
 					mblk_t *m;
 					ms_h264_bitstream_to_nalus(buf, info.size, &nalus);
-					m = ms_queue_peek_first(&nalus);
-					if (m){
+					
+					if (!ms_queue_empty(&nalus)){
+						m = ms_queue_peek_first(&nalus);
 						switch(ms_h264_nalu_get_type(m)){
 							case MSH264NaluTypeIDR:
 								ms_iframe_requests_limiter_notify_iframe_sent(&d->iframe_limiter, f->ticker->time);
@@ -239,6 +242,9 @@ static void enc_process(MSFilter *f){
 				}
 				AMediaCodec_releaseOutputBuffer(d->codec, obufidx, FALSE);
 			}
+			if (obufidx == AMEDIA_ERROR_UNKNOWN){
+				ms_error("MSMediaCodecH264Enc: AMediaCodec_dequeueOutputBuffer() had an exception");
+			}
 		}
 		freemsg(im);
 	}