videostream.c 28.8 KB
Newer Older
aymeric's avatar
aymeric committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2006  Simon MORLAT (simon.morlat@linphone.org)

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/

Simon Morlat's avatar
Simon Morlat committed
20 21 22 23
#ifdef HAVE_CONFIG_H
#include "mediastreamer-config.h"
#endif

aymeric's avatar
aymeric committed
24 25
#include "mediastreamer2/mediastream.h"
#include "mediastreamer2/msfilter.h"
26
#include "mediastreamer2/msinterfaces.h"
aymeric's avatar
aymeric committed
27 28 29
#include "mediastreamer2/msvideo.h"
#include "mediastreamer2/msrtp.h"
#include "mediastreamer2/msvideoout.h"
30
#include "mediastreamer2/msextdisplay.h"
Ghislain MARY's avatar
Ghislain MARY committed
31
#include "private.h"
Simon Morlat's avatar
Simon Morlat committed
32

Guillaume Beraudo's avatar
Guillaume Beraudo committed
33
#include <ortp/zrtp.h>
aymeric's avatar
aymeric committed
34

Simon Morlat's avatar
Simon Morlat committed
35

Ghislain MARY's avatar
Ghislain MARY committed
36
void video_stream_free(VideoStream *stream) {
37 38 39 40 41 42 43 44
	/* Prevent filters from being destroyed two times */
	if (stream->source_performs_encoding == TRUE) {
		stream->ms.encoder = NULL;
	}
	if (stream->output_performs_decoding == TRUE) {
		stream->ms.decoder = NULL;
	}

Ghislain MARY's avatar
Ghislain MARY committed
45
	media_stream_free(&stream->ms);
46

aymeric's avatar
aymeric committed
47 48 49 50
	if (stream->source != NULL)
		ms_filter_destroy (stream->source);
	if (stream->output != NULL)
		ms_filter_destroy (stream->output);
51 52
	if (stream->sizeconv != NULL)
		ms_filter_destroy (stream->sizeconv);
aymeric's avatar
aymeric committed
53 54 55 56
	if (stream->pixconv!=NULL)
		ms_filter_destroy(stream->pixconv);
	if (stream->tee!=NULL)
		ms_filter_destroy(stream->tee);
57 58 59 60
	if (stream->tee2!=NULL)
		ms_filter_destroy(stream->tee2);
	if (stream->jpegwriter!=NULL)
		ms_filter_destroy(stream->jpegwriter);
61 62
	if (stream->output2!=NULL)
		ms_filter_destroy(stream->output2);
63 64
	if (stream->display_name!=NULL)
		ms_free(stream->display_name);
65

aymeric's avatar
aymeric committed
66 67 68
	ms_free (stream);
}

69 70
static void event_cb(void *ud, MSFilter* f, unsigned int event, void *eventdata){
	VideoStream *st=(VideoStream*)ud;
Simon Morlat's avatar
Simon Morlat committed
71
	ms_message("event_cb called %u", event);
72 73 74 75 76
	if (st->eventcb!=NULL){
		st->eventcb(st->event_pointer,f,event,eventdata);
	}
}

aymeric's avatar
aymeric committed
77 78 79 80 81 82 83 84
static void video_steam_process_rtcp(VideoStream *stream, mblk_t *m){
	do{
		if (rtcp_is_SR(m)){
			const report_block_t *rb;
			ms_message("video_steam_process_rtcp: receiving RTCP SR");
			rb=rtcp_SR_get_report_block(m,0);
			if (rb){
				unsigned int ij;
Ghislain MARY's avatar
Ghislain MARY committed
85
				float rt=rtp_session_get_round_trip_propagation(stream->ms.session);
aymeric's avatar
aymeric committed
86 87
				float flost;
				ij=report_block_get_interarrival_jitter(rb);
aymeric's avatar
aymeric committed
88
				flost=(float)(100.0*report_block_get_fraction_lost(rb)/256.0);
89
				ms_message("video_steam_process_rtcp: interarrival jitter=%u , lost packets percentage since last report=%f, round trip time=%f seconds",ij,flost,rt);
Ghislain MARY's avatar
Ghislain MARY committed
90 91
				if (stream->ms.rc)
					ms_bitrate_controller_process_rtcp(stream->ms.rc,m);
aymeric's avatar
aymeric committed
92 93 94 95 96
			}
		}
	}while(rtcp_next_packet(m));
}

97
static void stop_preload_graph(VideoStream *stream){
Ghislain MARY's avatar
Ghislain MARY committed
98 99 100 101 102
	ms_ticker_detach(stream->ms.ticker,stream->ms.rtprecv);
	ms_filter_unlink(stream->ms.rtprecv,0,stream->ms.voidsink,0);
	ms_filter_destroy(stream->ms.voidsink);
	ms_filter_destroy(stream->ms.rtprecv);
	stream->ms.voidsink=stream->ms.rtprecv=NULL;
103
	stream->prepare_ongoing = FALSE;
104 105
}

aymeric's avatar
aymeric committed
106
void video_stream_iterate(VideoStream *stream){
107
	/*
aymeric's avatar
aymeric committed
108 109 110
	if (stream->output!=NULL)
		ms_filter_call_method_noarg(stream->output,
			MS_VIDEO_OUT_HANDLE_RESIZING);
111
	*/
Ghislain MARY's avatar
Ghislain MARY committed
112
	if (stream->ms.evq){
113
		OrtpEvent *ev;
Ghislain MARY's avatar
Ghislain MARY committed
114
		while (NULL != (ev=ortp_ev_queue_get(stream->ms.evq))) {
115 116
			OrtpEventType evt=ortp_event_get_type(ev);
			if (evt == ORTP_EVENT_RTCP_PACKET_RECEIVED){
aymeric's avatar
aymeric committed
117 118
				OrtpEventData *evd=ortp_event_get_data(ev);
				video_steam_process_rtcp(stream,evd->packet);
Ghislain MARY's avatar
Ghislain MARY committed
119 120
			}else if ((evt == ORTP_EVENT_STUN_PACKET_RECEIVED) && (stream->ms.ice_check_list)) {
				ice_handle_stun_packet(stream->ms.ice_check_list,stream->ms.session,ortp_event_get_data(ev));
aymeric's avatar
aymeric committed
121
			}
122
			ortp_event_destroy(ev);
aymeric's avatar
aymeric committed
123 124
		}
	}
125
	media_stream_iterate(&stream->ms);
aymeric's avatar
aymeric committed
126 127
}

128 129 130
static void choose_display_name(VideoStream *stream){
#ifdef WIN32
	stream->display_name=ms_strdup("MSDrawDibDisplay");
131 132
#elif defined(ANDROID)
	stream->display_name=ms_strdup("MSAndroidDisplay");
133 134
#elif __APPLE__ && !defined(__ios)
	stream->display_name=ms_strdup("MSOSXGLDisplay");
135 136 137
#elif defined(HAVE_GL)
	stream->display_name=ms_strdup("MSGLXVideo");
#elif defined (HAVE_XV)
Simon Morlat's avatar
Simon Morlat committed
138
	stream->display_name=ms_strdup("MSX11Video");
Simon Morlat's avatar
Simon Morlat committed
139
#elif defined(__ios)
140
	stream->display_name=ms_strdup("IOSDisplay");	
Simon Morlat's avatar
Simon Morlat committed
141 142
#else
	stream->display_name=ms_strdup("MSVideoOut");
143 144 145
#endif
}

146
VideoStream *video_stream_new(int loc_rtp_port, int loc_rtcp_port, bool_t use_ipv6){
aymeric's avatar
aymeric committed
147
	VideoStream *stream = (VideoStream *)ms_new0 (VideoStream, 1);
Ghislain MARY's avatar
Ghislain MARY committed
148 149
	stream->ms.type = VideoStreamType;
	stream->ms.session=create_duplex_rtpsession(loc_rtp_port,loc_rtcp_port,use_ipv6);
150
	stream->ms.qi=ms_quality_indicator_new(stream->ms.session);
Ghislain MARY's avatar
Ghislain MARY committed
151 152 153 154
	stream->ms.evq=ortp_ev_queue_new();
	stream->ms.rtpsend=ms_filter_new(MS_RTP_SEND_ID);
	stream->ms.ice_check_list=NULL;
	rtp_session_register_event_queue(stream->ms.session,stream->ms.evq);
smorlat's avatar
smorlat committed
155 156
	stream->sent_vsize.width=MS_VIDEO_SIZE_CIF_W;
	stream->sent_vsize.height=MS_VIDEO_SIZE_CIF_H;
157
	stream->dir=VideoStreamSendRecv;
158
	stream->display_filter_auto_rotate_enabled=0;
159 160
	stream->source_performs_encoding = FALSE;
	stream->output_performs_decoding = FALSE;
161 162
	choose_display_name(stream);

aymeric's avatar
aymeric committed
163 164 165
	return stream;
}

smorlat's avatar
smorlat committed
166
void video_stream_set_sent_video_size(VideoStream *stream, MSVideoSize vsize){
167
	ms_message("Setting video size %dx%d", vsize.width, vsize.height);
='s avatar
= committed
168
	stream->sent_vsize=vsize;
smorlat's avatar
smorlat committed
169 170
}

aymeric's avatar
aymeric committed
171
void video_stream_set_relay_session_id(VideoStream *stream, const char *id){
Ghislain MARY's avatar
Ghislain MARY committed
172
	ms_filter_call_method(stream->ms.rtpsend, MS_RTP_SEND_SET_RELAY_SESSION_ID,(void*)id);
aymeric's avatar
aymeric committed
173 174
}

smorlat's avatar
smorlat committed
175
void video_stream_enable_self_view(VideoStream *stream, bool_t val){
='s avatar
= committed
176
	MSFilter *out=stream->output;
177 178 179 180
	stream->corner=val ? 0 : -1;
	if (out){
		ms_filter_call_method(out,MS_VIDEO_DISPLAY_SET_LOCAL_VIEW_MODE,&stream->corner);
	}
smorlat's avatar
smorlat committed
181
}
aymeric's avatar
aymeric committed
182

183 184 185 186 187
void video_stream_set_render_callback (VideoStream *s, VideoStreamRenderCallback cb, void *user_pointer){
	s->rendercb=cb;
	s->render_pointer=user_pointer;
}

188 189 190 191 192
void video_stream_set_event_callback (VideoStream *s, VideoStreamEventCallback cb, void *user_pointer){
	s->eventcb=cb;
	s->event_pointer=user_pointer;
}

193 194 195 196 197 198 199 200 201
void video_stream_set_display_filter_name(VideoStream *s, const char *fname){
	if (s->display_name!=NULL){
		ms_free(s->display_name);
		s->display_name=NULL;
	}
	if (fname!=NULL)
		s->display_name=ms_strdup(fname);
}

202 203

static void ext_display_cb(void *ud, MSFilter* f, unsigned int event, void *eventdata){
204 205 206 207 208 209 210 211 212
	MSExtDisplayOutput *output=(MSExtDisplayOutput*)eventdata;
	VideoStream *st=(VideoStream*)ud;
	if (st->rendercb!=NULL){
		st->rendercb(st->render_pointer,
		            output->local_view.w!=0 ? &output->local_view : NULL,
		            output->remote_view.w!=0 ? &output->remote_view : NULL);
	}
}

213
void video_stream_set_direction(VideoStream *vs, VideoStreamDir dir){
='s avatar
= committed
214
	vs->dir=dir;
215 216
}

Simon Morlat's avatar
Simon Morlat committed
217 218 219 220
static MSVideoSize get_compatible_size(MSVideoSize maxsize, MSVideoSize wished_size){
	int max_area=maxsize.width*maxsize.height;
	int whished_area=wished_size.width*wished_size.height;
	if (whished_area>max_area){
221
		return maxsize;
Simon Morlat's avatar
Simon Morlat committed
222 223 224 225
	}
	return wished_size;
}

226 227 228 229 230 231 232 233 234 235
static MSVideoSize get_with_same_orientation(MSVideoSize size, MSVideoSize refsize){
	if (ms_video_size_get_orientation(refsize)!=ms_video_size_get_orientation(size)){
		int tmp;
		tmp=size.width;
		size.width=size.height;
		size.height=tmp;
	}
	return size;
}

236 237 238 239
static void configure_video_source(VideoStream *stream){
	MSVideoSize vsize,cam_vsize;
	float fps=15;
	MSPixFmt format;
240
	bool_t encoder_has_builtin_converter = FALSE;
241

242 243
	/* transmit orientation to source filter */
	ms_filter_call_method(stream->source,MS_VIDEO_CAPTURE_SET_DEVICE_ORIENTATION,&stream->device_orientation);
244 245
	
    /* transmit its preview window id if any to source filter*/
246 247 248
	if (stream->preview_window_id!=0){
		video_stream_set_native_preview_window_id(stream, stream->preview_window_id);
	}
249

Ghislain MARY's avatar
Ghislain MARY committed
250 251
	ms_filter_call_method(stream->ms.encoder, MS_VIDEO_ENCODER_HAS_BUILTIN_CONVERTER, &encoder_has_builtin_converter);
	ms_filter_call_method(stream->ms.encoder,MS_FILTER_GET_VIDEO_SIZE,&vsize);
Simon Morlat's avatar
Simon Morlat committed
252
	vsize=get_compatible_size(vsize,stream->sent_vsize);
253 254 255
	ms_filter_call_method(stream->source,MS_FILTER_SET_VIDEO_SIZE,&vsize);
	/*the camera may not support the target size and suggest a one close to the target */
	ms_filter_call_method(stream->source,MS_FILTER_GET_VIDEO_SIZE,&cam_vsize);
256 257
	if (cam_vsize.width*cam_vsize.height<=vsize.width*vsize.height &&
			cam_vsize.width != vsize.width){
258
		vsize=cam_vsize;
259
		ms_message("Output video size adjusted to match camera resolution (%ix%i)\n",vsize.width,vsize.height);
260
	} else if (cam_vsize.width*cam_vsize.height>vsize.width*vsize.height){
261
#if TARGET_IPHONE_SIMULATOR || defined(__arm__)
262 263 264 265 266 267
		ms_error("Camera is proposing a size bigger than encoder's suggested size (%ix%i > %ix%i) "
		           "Using the camera size as fallback because cropping or resizing is not implemented for arm.",
		           cam_vsize.width,cam_vsize.height,vsize.width,vsize.height);
		vsize=cam_vsize;
#else
		vsize=get_with_same_orientation(vsize,cam_vsize);
268
		ms_warning("Camera video size greater than encoder one. A scaling filter will be used!\n");
269
#endif
270
	}
Ghislain MARY's avatar
Ghislain MARY committed
271 272
	ms_filter_call_method(stream->ms.encoder,MS_FILTER_SET_VIDEO_SIZE,&vsize);
	ms_filter_call_method(stream->ms.encoder,MS_FILTER_GET_FPS,&fps);
273 274 275 276 277 278 279
	ms_message("Setting sent vsize=%ix%i, fps=%f",vsize.width,vsize.height,fps);
	/* configure the filters */
	if (ms_filter_get_id(stream->source)!=MS_STATIC_IMAGE_ID) {
		ms_filter_call_method(stream->source,MS_FILTER_SET_FPS,&fps);
	}
	/* get the output format for webcam reader */
	ms_filter_call_method(stream->source,MS_FILTER_GET_PIX_FMT,&format);
280

281
	if ((encoder_has_builtin_converter == TRUE) || (stream->source_performs_encoding == TRUE)) {
Ghislain MARY's avatar
Ghislain MARY committed
282
		ms_filter_call_method(stream->ms.encoder, MS_FILTER_SET_PIX_FMT, &format);
283 284 285 286 287 288 289 290 291 292 293
	} else {
		if (format==MS_MJPEG){
			stream->pixconv=ms_filter_new(MS_MJPEG_DEC_ID);
		}else{
			stream->pixconv = ms_filter_new(MS_PIX_CONV_ID);
			/*set it to the pixconv */
			ms_filter_call_method(stream->pixconv,MS_FILTER_SET_PIX_FMT,&format);
			ms_filter_call_method(stream->pixconv,MS_FILTER_SET_VIDEO_SIZE,&cam_vsize);
		}
		stream->sizeconv=ms_filter_new(MS_SIZE_CONV_ID);
		ms_filter_call_method(stream->sizeconv,MS_FILTER_SET_VIDEO_SIZE,&vsize);
294
	}
Ghislain MARY's avatar
Ghislain MARY committed
295 296 297
	if (stream->ms.rc){
		ms_bitrate_controller_destroy(stream->ms.rc);
		stream->ms.rc=NULL;
298
	}
Ghislain MARY's avatar
Ghislain MARY committed
299 300
	if (stream->ms.use_rc){
		stream->ms.rc=ms_av_bitrate_controller_new(NULL,NULL,stream->ms.session,stream->ms.encoder);
301
	}
302 303
}

304 305
int video_stream_start (VideoStream *stream, RtpProfile *profile, const char *rem_rtp_ip, int rem_rtp_port,
	const char *rem_rtcp_ip, int rem_rtcp_port, int payload, int jitt_comp, MSWebCam *cam){
aymeric's avatar
aymeric committed
306
	PayloadType *pt;
Ghislain MARY's avatar
Ghislain MARY committed
307
	RtpSession *rtps=stream->ms.session;
aymeric's avatar
aymeric committed
308
	MSPixFmt format;
309
	MSVideoSize disp_size;
smorlat's avatar
smorlat committed
310 311 312
	int tmp;
	JBParameters jbp;
	const int socket_buf_size=2000000;
aymeric's avatar
aymeric committed
313

314 315
	if (cam==NULL){
		cam=ms_web_cam_manager_get_default_cam (
316
		      ms_web_cam_manager_get());
317 318
	}

aymeric's avatar
aymeric committed
319 320 321 322 323
	pt=rtp_profile_get_payload(profile,payload);
	if (pt==NULL){
		ms_error("videostream.c: undefined payload type.");
		return -1;
	}
324

325 326 327 328
	if ((cam != NULL) && (cam->desc->encode_to_mime_type != NULL) && (cam->desc->encode_to_mime_type(cam, pt->mime_type) == TRUE)) {
		stream->source_performs_encoding = TRUE;
	}

aymeric's avatar
aymeric committed
329
	rtp_session_set_profile(rtps,profile);
330
	if (rem_rtp_port>0) rtp_session_set_remote_addr_full(rtps,rem_rtp_ip,rem_rtp_port,rem_rtcp_ip,rem_rtcp_port);
aymeric's avatar
aymeric committed
331 332 333
	rtp_session_set_payload_type(rtps,payload);
	rtp_session_set_jitter_compensation(rtps,jitt_comp);

Ghislain MARY's avatar
Ghislain MARY committed
334
	rtp_session_signal_connect(stream->ms.session,"payload_type_changed",
335
			(RtpCallback)mediastream_payload_type_changed,(unsigned long)&stream->ms);
aymeric's avatar
aymeric committed
336

Ghislain MARY's avatar
Ghislain MARY committed
337
	rtp_session_get_jitter_buffer_params(stream->ms.session,&jbp);
smorlat's avatar
smorlat committed
338
	jbp.max_packets=1000;//needed for high resolution video
Ghislain MARY's avatar
Ghislain MARY committed
339 340 341
	rtp_session_set_jitter_buffer_params(stream->ms.session,&jbp);
	rtp_session_set_rtp_socket_recv_buffer_size(stream->ms.session,socket_buf_size);
	rtp_session_set_rtp_socket_send_buffer_size(stream->ms.session,socket_buf_size);
342

343
	if (stream->dir==VideoStreamSendRecv || stream->dir==VideoStreamSendOnly){
344
		MSConnectionHelper ch;
345 346
		/*plumb the outgoing stream */

Ghislain MARY's avatar
Ghislain MARY committed
347
		if (rem_rtp_port>0) ms_filter_call_method(stream->ms.rtpsend,MS_RTP_SEND_SET_SESSION,stream->ms.session);
348 349 350 351 352 353 354
		if (stream->source_performs_encoding == FALSE) {
			stream->ms.encoder=ms_filter_create_encoder(pt->mime_type);
			if ((stream->ms.encoder==NULL) ){
				/* big problem: we don't have a registered codec for this payload...*/
				ms_error("videostream.c: No encoder available for payload %i:%s.",payload,pt->mime_type);
				return -1;
			}
355 356
		}
		/* creates the filters */
357
		stream->cam=cam;
358 359
		stream->source = ms_web_cam_create_reader(cam);
		stream->tee = ms_filter_new(MS_TEE_ID);
360 361 362
		if (stream->source_performs_encoding == TRUE) {
			stream->ms.encoder = stream->source;	/* Consider the encoder is the source */
		}
363

364 365
		if (pt->normal_bitrate>0){
			ms_message("Limiting bitrate of video encoder to %i bits/s",pt->normal_bitrate);
Ghislain MARY's avatar
Ghislain MARY committed
366
			ms_filter_call_method(stream->ms.encoder,MS_FILTER_SET_BITRATE,&pt->normal_bitrate);
367 368
		}
		if (pt->send_fmtp){
Ghislain MARY's avatar
Ghislain MARY committed
369
			ms_filter_call_method(stream->ms.encoder,MS_FILTER_ADD_FMTP,pt->send_fmtp);
370
		}
371 372 373 374 375
		if (stream->use_preview_window){
			if (stream->rendercb==NULL){
				stream->output2=ms_filter_new_from_name (stream->display_name);
			}
		}
376

377
		configure_video_source (stream);
378 379 380 381 382 383 384 385 386 387
		/* and then connect all */
		ms_connection_helper_start(&ch);
		ms_connection_helper_link(&ch, stream->source, -1, 0);
		if (stream->pixconv) {
			ms_connection_helper_link(&ch, stream->pixconv, 0, 0);
		}
		if (stream->sizeconv) {
			ms_connection_helper_link(&ch, stream->sizeconv, 0, 0);
		}
		ms_connection_helper_link(&ch, stream->tee, 0, 0);
388 389 390
		if (stream->source_performs_encoding == FALSE) {
			ms_connection_helper_link(&ch, stream->ms.encoder, 0, 0);
		}
Ghislain MARY's avatar
Ghislain MARY committed
391
		ms_connection_helper_link(&ch, stream->ms.rtpsend, 0, -1);
392
		if (stream->output2){
393
			if (stream->preview_window_id!=0){
394 395 396 397
				ms_filter_call_method(stream->output2, MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&stream->preview_window_id);
			}
			ms_filter_link(stream->tee,1,stream->output2,0);
		}
aymeric's avatar
aymeric committed
398
	}
399
	if (stream->dir==VideoStreamSendRecv || stream->dir==VideoStreamRecvOnly){
400
		MSConnectionHelper ch;
401
		MSVideoDisplayDecodingSupport decoding_support;
402

403 404
		if (stream->rendercb!=NULL){
			stream->output=ms_filter_new(MS_EXT_DISPLAY_ID);
405
			ms_filter_set_notify_callback(stream->output,ext_display_cb,stream);
406 407 408
		}else{
			stream->output=ms_filter_new_from_name (stream->display_name);
		}
409 410 411 412 413 414

		/* Don't allow null output */
		if(stream->output == NULL) {
			ms_fatal("No video display filter could be instantiated. Please check build-time configuration");
		}

415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443
		/* Check if the output filter can perform the decoding process */
		decoding_support.mime_type = pt->mime_type;
		decoding_support.supported = FALSE;
		ms_filter_call_method(stream->output, MS_VIDEO_DISPLAY_SUPPORT_DECODING, &decoding_support);
		stream->output_performs_decoding = decoding_support.supported;

		/*plumb the incoming stream */
		if (stream->output_performs_decoding == TRUE) {
			stream->ms.decoder = stream->output;	/* Consider the decoder is the output */
		} else {
			stream->ms.decoder=ms_filter_create_decoder(pt->mime_type);
			if ((stream->ms.decoder==NULL) ){
				/* big problem: we don't have a registered decoderfor this payload...*/
				ms_error("videostream.c: No decoder available for payload %i:%s.",payload,pt->mime_type);
				ms_filter_destroy(stream->output);
				return -1;
			}
		}
		ms_filter_set_notify_callback(stream->ms.decoder, event_cb, stream);

		stream->ms.rtprecv = ms_filter_new (MS_RTP_RECV_ID);
		ms_filter_call_method(stream->ms.rtprecv,MS_RTP_RECV_SET_SESSION,stream->ms.session);

		if (stream->output_performs_decoding == FALSE) {
			stream->jpegwriter=ms_filter_new(MS_JPEG_WRITER_ID);
			if (stream->jpegwriter)
				stream->tee2=ms_filter_new(MS_TEE_ID);
		}

444 445
		/* set parameters to the decoder*/
		if (pt->send_fmtp){
Ghislain MARY's avatar
Ghislain MARY committed
446
			ms_filter_call_method(stream->ms.decoder,MS_FILTER_ADD_FMTP,pt->send_fmtp);
447 448
		}
		if (pt->recv_fmtp!=NULL)
Ghislain MARY's avatar
Ghislain MARY committed
449
			ms_filter_call_method(stream->ms.decoder,MS_FILTER_ADD_FMTP,(void*)pt->recv_fmtp);
450

451 452
		/*force the decoder to output YUV420P */
		format=MS_YUV420P;
Ghislain MARY's avatar
Ghislain MARY committed
453
		ms_filter_call_method(stream->ms.decoder,MS_FILTER_SET_PIX_FMT,&format);
454 455

		/*configure the display window */
456 457 458 459 460 461 462 463 464 465 466 467 468 469
		if(stream->output != NULL) {
			disp_size.width=MS_VIDEO_SIZE_CIF_W;
			disp_size.height=MS_VIDEO_SIZE_CIF_H;
			tmp=1;
			ms_filter_call_method(stream->output,MS_FILTER_SET_VIDEO_SIZE,&disp_size);
			ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_ENABLE_AUTOFIT,&tmp);
			ms_filter_call_method(stream->output,MS_FILTER_SET_PIX_FMT,&format);
			ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SET_LOCAL_VIEW_MODE,&stream->corner);
			if (stream->window_id!=0){
				ms_filter_call_method(stream->output, MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&stream->window_id);
			}
			if (stream->display_filter_auto_rotate_enabled) {
				ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SET_DEVICE_ORIENTATION,&stream->device_orientation);
			}
470
		}
471
		/* and connect the filters */
472
		ms_connection_helper_start (&ch);
Ghislain MARY's avatar
Ghislain MARY committed
473
		ms_connection_helper_link (&ch,stream->ms.rtprecv,-1,0);
474 475 476
		if (stream->output_performs_decoding == FALSE) {
			ms_connection_helper_link (&ch,stream->ms.decoder,0,0);
		}
477 478 479 480
		if (stream->tee2){
			ms_connection_helper_link (&ch,stream->tee2,0,0);
			ms_filter_link(stream->tee2,1,stream->jpegwriter,0);
		}
481 482
		if (stream->output!=NULL)
			ms_connection_helper_link (&ch,stream->output,0,-1);
483
		/* the video source must be send for preview , if it exists*/
484
		if (stream->tee!=NULL && stream->output!=NULL && stream->output2==NULL)
485
			ms_filter_link(stream->tee,1,stream->output,1);
aymeric's avatar
aymeric committed
486
	}
487
	if (stream->dir == VideoStreamSendOnly) {
Ghislain MARY's avatar
Ghislain MARY committed
488 489 490 491
		stream->ms.rtprecv = ms_filter_new (MS_RTP_RECV_ID);
		ms_filter_call_method(stream->ms.rtprecv, MS_RTP_RECV_SET_SESSION, stream->ms.session);
		stream->ms.voidsink = ms_filter_new(MS_VOID_SINK_ID);
		ms_filter_link(stream->ms.rtprecv, 0, stream->ms.voidsink, 0);
492
	}
493

aymeric's avatar
aymeric committed
494
	/* create the ticker */
Ghislain MARY's avatar
Ghislain MARY committed
495
	if (stream->ms.ticker==NULL) start_ticker(&stream->ms);
496 497 498
	
	stream->ms.start_time=ms_time(NULL);
	stream->ms.is_beginning=TRUE;
499

500 501
	/* attach the graphs */
	if (stream->source)
Ghislain MARY's avatar
Ghislain MARY committed
502 503 504
		ms_ticker_attach (stream->ms.ticker, stream->source);
	if (stream->ms.rtprecv)
		ms_ticker_attach (stream->ms.ticker, stream->ms.rtprecv);
aymeric's avatar
aymeric committed
505 506 507
	return 0;
}

508
void video_stream_prepare_video(VideoStream *stream){
509
	stream->prepare_ongoing = TRUE;
510
	video_stream_unprepare_video(stream);
Ghislain MARY's avatar
Ghislain MARY committed
511 512 513 514 515 516 517
	stream->ms.rtprecv=ms_filter_new(MS_RTP_RECV_ID);
	rtp_session_set_payload_type(stream->ms.session,0);
	ms_filter_call_method(stream->ms.rtprecv,MS_RTP_RECV_SET_SESSION,stream->ms.session);
	stream->ms.voidsink=ms_filter_new(MS_VOID_SINK_ID);
	ms_filter_link(stream->ms.rtprecv,0,stream->ms.voidsink,0);
	start_ticker(&stream->ms);
	ms_ticker_attach(stream->ms.ticker,stream->ms.rtprecv);
518 519
}

520
void video_stream_unprepare_video(VideoStream *stream){
Ghislain MARY's avatar
Ghislain MARY committed
521
	if (stream->ms.voidsink) {
522 523 524 525
		stop_preload_graph(stream);
	}
}

526 527 528 529 530
void video_stream_update_video_params(VideoStream *stream){
	/*calling video_stream_change_camera() does the job of unplumbing/replumbing and configuring the new graph*/
	video_stream_change_camera(stream,stream->cam);
}

531
void video_stream_change_camera(VideoStream *stream, MSWebCam *cam){
Simon Morlat's avatar
Simon Morlat committed
532
	bool_t keep_source=(cam==stream->cam);
533 534
	bool_t encoder_has_builtin_converter = (!stream->pixconv && !stream->sizeconv);

Ghislain MARY's avatar
Ghislain MARY committed
535 536
	if (stream->ms.ticker && stream->source){
		ms_ticker_detach(stream->ms.ticker,stream->source);
537
		/*unlink source filters and subsequent post processin filters */
538
		if (encoder_has_builtin_converter || (stream->source_performs_encoding == TRUE)) {
539 540 541 542 543 544
			ms_filter_unlink(stream->source, 0, stream->tee, 0);
		} else {
			ms_filter_unlink (stream->source, 0, stream->pixconv, 0);
			ms_filter_unlink (stream->pixconv, 0, stream->sizeconv, 0);
			ms_filter_unlink (stream->sizeconv, 0, stream->tee, 0);
		}
545
		/*destroy the filters */
Simon Morlat's avatar
Simon Morlat committed
546
		if (!keep_source) ms_filter_destroy(stream->source);
547
		if (!encoder_has_builtin_converter && (stream->source_performs_encoding == FALSE)) {
548 549 550
			ms_filter_destroy(stream->pixconv);
			ms_filter_destroy(stream->sizeconv);
		}
551 552

		/*re create new ones and configure them*/
Simon Morlat's avatar
Simon Morlat committed
553
		if (!keep_source) stream->source = ms_web_cam_create_reader(cam);
554
		stream->cam=cam;
555 556 557 558 559 560 561 562 563 564 565

		/* update orientation */
		if (stream->source){
			ms_filter_call_method(stream->source,MS_VIDEO_CAPTURE_SET_DEVICE_ORIENTATION,&stream->device_orientation);
			if (!stream->display_filter_auto_rotate_enabled)
				ms_filter_call_method(stream->source,MS_VIDEO_DISPLAY_SET_DEVICE_ORIENTATION,&stream->device_orientation);
		}
		if (stream->output && stream->display_filter_auto_rotate_enabled) {
			ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SET_DEVICE_ORIENTATION,&stream->device_orientation);
		}

566
		configure_video_source(stream);
567

568
		if (encoder_has_builtin_converter || (stream->source_performs_encoding == TRUE)) {
569 570 571 572 573 574 575
			ms_filter_link (stream->source, 0, stream->tee, 0);
		}
		else {
			ms_filter_link (stream->source, 0, stream->pixconv, 0);
			ms_filter_link (stream->pixconv, 0, stream->sizeconv, 0);
			ms_filter_link (stream->sizeconv, 0, stream->tee, 0);
		}
576

Ghislain MARY's avatar
Ghislain MARY committed
577
		ms_ticker_attach(stream->ms.ticker,stream->source);
578 579 580
	}
}

aymeric's avatar
aymeric committed
581
void video_stream_send_vfu(VideoStream *stream){
Ghislain MARY's avatar
Ghislain MARY committed
582 583
	if (stream->ms.encoder)
		ms_filter_call_method_noarg(stream->ms.encoder, MS_VIDEO_ENCODER_REQ_VFU);
aymeric's avatar
aymeric committed
584 585 586 587 588
}

void
video_stream_stop (VideoStream * stream)
{
589 590
	stream->eventcb = NULL;
	stream->event_pointer = NULL;
Ghislain MARY's avatar
Ghislain MARY committed
591
	if (stream->ms.ticker){
592
		if (stream->prepare_ongoing == TRUE) {
593
			stop_preload_graph(stream);
594 595
		} else {
			if (stream->source)
Ghislain MARY's avatar
Ghislain MARY committed
596 597 598
				ms_ticker_detach(stream->ms.ticker,stream->source);
			if (stream->ms.rtprecv)
				ms_ticker_detach(stream->ms.ticker,stream->ms.rtprecv);
599

Ghislain MARY's avatar
Ghislain MARY committed
600 601 602
			if (stream->ms.ice_check_list != NULL) {
				ice_check_list_print_route(stream->ms.ice_check_list, "Video session's route");
				stream->ms.ice_check_list = NULL;
603
			}
Yann Diorcet's avatar
Yann Diorcet committed
604 605
			rtp_stats_display(rtp_session_get_stats(stream->ms.session),
				"             VIDEO SESSION'S RTP STATISTICS                ");
606 607

			if (stream->source){
608 609 610 611 612 613 614 615 616 617
				MSConnectionHelper ch;
				ms_connection_helper_start(&ch);
				ms_connection_helper_unlink(&ch, stream->source, -1, 0);
				if (stream->pixconv) {
					ms_connection_helper_unlink(&ch, stream->pixconv, 0, 0);
				}
				if (stream->sizeconv) {
					ms_connection_helper_unlink(&ch, stream->sizeconv, 0, 0);
				}
				ms_connection_helper_unlink(&ch, stream->tee, 0, 0);
618 619 620
				if (stream->source_performs_encoding == FALSE) {
					ms_connection_helper_unlink(&ch, stream->ms.encoder, 0, 0);
				}
Ghislain MARY's avatar
Ghislain MARY committed
621
				ms_connection_helper_unlink(&ch, stream->ms.rtpsend, 0, -1);
622 623 624
				if (stream->output2){
					ms_filter_unlink(stream->tee,1,stream->output2,0);
				}
625
			}
Ghislain MARY's avatar
Ghislain MARY committed
626 627 628
			if (stream->ms.voidsink) {
				ms_filter_unlink(stream->ms.rtprecv, 0, stream->ms.voidsink, 0);
			} else if (stream->ms.rtprecv){
629 630
				MSConnectionHelper h;
				ms_connection_helper_start (&h);
Ghislain MARY's avatar
Ghislain MARY committed
631
				ms_connection_helper_unlink (&h,stream->ms.rtprecv,-1,0);
632 633 634
				if (stream->output_performs_decoding == FALSE) {
					ms_connection_helper_unlink (&h,stream->ms.decoder,0,0);
				}
635 636 637 638 639 640 641 642
				if (stream->tee2){
					ms_connection_helper_unlink (&h,stream->tee2,0,0);
					ms_filter_unlink(stream->tee2,1,stream->jpegwriter,0);
				}
				if(stream->output)
					ms_connection_helper_unlink (&h,stream->output,0,-1);
				if (stream->tee && stream->output && stream->output2==NULL)
					ms_filter_unlink(stream->tee,1,stream->output,1);
643
			}
644
		}
aymeric's avatar
aymeric committed
645 646 647 648 649
	}
	video_stream_free (stream);
}


650 651 652 653 654 655 656
void video_stream_show_video(VideoStream *stream, bool_t show){
	if (stream->output){
		ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SHOW_VIDEO,&show);
	}
}


657 658 659
unsigned long video_stream_get_native_window_id(VideoStream *stream){
	unsigned long id;
	if (stream->output){
unknown's avatar
unknown committed
660
		if (ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_GET_NATIVE_WINDOW_ID,&id)==0)
661 662
			return id;
	}
663 664 665 666 667
	return stream->window_id;
}

void video_stream_set_native_window_id(VideoStream *stream, unsigned long id){
	stream->window_id=id;
668 669 670
	if (stream->output){
		ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&id);
	}
671 672 673 674
}

void video_stream_set_native_preview_window_id(VideoStream *stream, unsigned long id){
	stream->preview_window_id=id;
675
#ifndef __ios	
676 677 678
	if (stream->output2){
		ms_filter_call_method(stream->output2,MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&id);
	}
679
#endif
680 681
	if (stream->source){
		ms_filter_call_method(stream->source,MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&id);
682
	}
683 684
}

Simon Morlat's avatar
wip  
Simon Morlat committed
685
unsigned long video_stream_get_native_preview_window_id(VideoStream *stream){
Simon Morlat's avatar
Simon Morlat committed
686
	unsigned long id=0;
Simon Morlat's avatar
Simon Morlat committed
687 688 689 690 691
	if (stream->output2){
		if (ms_filter_call_method(stream->output2,MS_VIDEO_DISPLAY_GET_NATIVE_WINDOW_ID,&id)==0)
			return id;
	}
	if (stream->source){
692
		if (ms_filter_has_method(stream->source,MS_VIDEO_DISPLAY_GET_NATIVE_WINDOW_ID) 
Simon Morlat's avatar
Simon Morlat committed
693
		    && ms_filter_call_method(stream->source,MS_VIDEO_DISPLAY_GET_NATIVE_WINDOW_ID,&id)==0)
694 695 696 697 698 699
			return id;
	}
	return stream->preview_window_id;
}

void video_stream_use_preview_video_window(VideoStream *stream, bool_t yesno){
='s avatar
= committed
700
	stream->use_preview_window=yesno;
701
}
aymeric's avatar
aymeric committed
702

703
void video_stream_set_device_rotation(VideoStream *stream, int orientation){
704
	stream->device_orientation = orientation;
705 706
}

707 708 709 710 711 712 713 714 715 716
int video_stream_get_camera_sensor_rotation(VideoStream *stream) {
	int rotation = -1;
	if (stream->source) {
		if (ms_filter_has_method(stream->source, MS_VIDEO_CAPTURE_GET_CAMERA_SENSOR_ROTATION)
			&& ms_filter_call_method(stream->source, MS_VIDEO_CAPTURE_GET_CAMERA_SENSOR_ROTATION, &rotation) == 0)
			return rotation;
	}
	return -1;
}

Simon Morlat's avatar
Simon Morlat committed
717 718 719 720 721 722 723 724
VideoPreview * video_preview_new(void){
	VideoPreview *stream = (VideoPreview *)ms_new0 (VideoPreview, 1);
	stream->sent_vsize.width=MS_VIDEO_SIZE_CIF_W;
	stream->sent_vsize.height=MS_VIDEO_SIZE_CIF_H;
	choose_display_name(stream);
	return stream;
}

aymeric's avatar
aymeric committed
725

Simon Morlat's avatar
Simon Morlat committed
726
void video_preview_start(VideoPreview *stream, MSWebCam *device){
aymeric's avatar
aymeric committed
727
	MSPixFmt format;
aymeric's avatar
aymeric committed
728
	float fps=(float)29.97;
smorlat's avatar
smorlat committed
729
	int mirroring=1;
Simon Morlat's avatar
Simon Morlat committed
730
	int corner=-1;
Simon Morlat's avatar
Simon Morlat committed
731 732 733
	MSVideoSize disp_size=stream->sent_vsize;
	MSVideoSize vsize=disp_size;
	const char *displaytype=stream->display_name;
aymeric's avatar
aymeric committed
734 735

	stream->source = ms_web_cam_create_reader(device);
736

737

aymeric's avatar
aymeric committed
738
	/* configure the filters */
smorlat's avatar
smorlat committed
739
	ms_filter_call_method(stream->source,MS_FILTER_SET_VIDEO_SIZE,&vsize);
740 741
	if (ms_filter_get_id(stream->source)!=MS_STATIC_IMAGE_ID)
		ms_filter_call_method(stream->source,MS_FILTER_SET_FPS,&fps);
smorlat's avatar
smorlat committed
742
	ms_filter_call_method(stream->source,MS_FILTER_GET_PIX_FMT,&format);
743
	ms_filter_call_method(stream->source,MS_FILTER_GET_VIDEO_SIZE,&vsize);
aymeric's avatar
aymeric committed
744 745 746 747 748 749 750
	if (format==MS_MJPEG){
		stream->pixconv=ms_filter_new(MS_MJPEG_DEC_ID);
	}else{
		stream->pixconv=ms_filter_new(MS_PIX_CONV_ID);
		ms_filter_call_method(stream->pixconv,MS_FILTER_SET_PIX_FMT,&format);
		ms_filter_call_method(stream->pixconv,MS_FILTER_SET_VIDEO_SIZE,&vsize);
	}
751

aymeric's avatar
aymeric committed
752
	format=MS_YUV420P;
Simon Morlat's avatar
Simon Morlat committed
753

754
	stream->output2=ms_filter_new_from_name (displaytype);
Simon Morlat's avatar
Simon Morlat committed
755 756 757 758
	ms_filter_call_method(stream->output2,MS_FILTER_SET_PIX_FMT,&format);
	ms_filter_call_method(stream->output2,MS_FILTER_SET_VIDEO_SIZE,&disp_size);
	ms_filter_call_method(stream->output2,MS_VIDEO_DISPLAY_ENABLE_MIRRORING,&mirroring);
	ms_filter_call_method(stream->output2,MS_VIDEO_DISPLAY_SET_LOCAL_VIEW_MODE,&corner);
aymeric's avatar
aymeric committed
759
	/* and then connect all */
760

aymeric's avatar
aymeric committed
761
	ms_filter_link(stream->source,0, stream->pixconv,0);
Simon Morlat's avatar
Simon Morlat committed
762
	ms_filter_link(stream->pixconv, 0, stream->output2, 0);
763

764
	if (stream->preview_window_id!=0){
765
		video_stream_set_native_preview_window_id(stream, stream->preview_window_id);
766
	}
aymeric's avatar
aymeric committed
767
	/* create the ticker */
Ghislain MARY's avatar
Ghislain MARY committed
768 769 770
	stream->ms.ticker = ms_ticker_new();
	ms_ticker_set_name(stream->ms.ticker,"Video MSTicker");
	ms_ticker_attach (stream->ms.ticker, stream->source);
aymeric's avatar
aymeric committed
771 772 773
}

void video_preview_stop(VideoStream *stream){
Ghislain MARY's avatar
Ghislain MARY committed
774
	ms_ticker_detach(stream->ms.ticker, stream->source);
aymeric's avatar
aymeric committed
775
	ms_filter_unlink(stream->source,0,stream->pixconv,0);
Simon Morlat's avatar
Simon Morlat committed
776
	ms_filter_unlink(stream->pixconv,0,stream->output2,0);
aymeric's avatar
aymeric committed
777 778 779
	video_stream_free(stream);
}

780 781
int video_stream_recv_only_start(VideoStream *videostream, RtpProfile *profile, const char *addr, int port, int used_pt, int jitt_comp){
	video_stream_set_direction(videostream,VideoStreamRecvOnly);
782
	return video_stream_start(videostream,profile,addr,port,addr,port+1,used_pt,jitt_comp,NULL);
783 784 785
}

int video_stream_send_only_start(VideoStream *videostream,
786
				RtpProfile *profile, const char *addr, int port, int rtcp_port,
787 788
				int used_pt, int  jitt_comp, MSWebCam *device){
	video_stream_set_direction (videostream,VideoStreamSendOnly);
789
	return video_stream_start(videostream,profile,addr,port,addr,rtcp_port,used_pt,jitt_comp,device);
790 791 792 793 794 795 796 797 798 799
}


void video_stream_recv_only_stop(VideoStream *vs){
	video_stream_stop(vs);
}

void video_stream_send_only_stop(VideoStream *vs){
	video_stream_stop(vs);
}
Guillaume Beraudo's avatar
Guillaume Beraudo committed
800 801 802

/* enable ZRTP on the video stream using information from the audio stream */
void video_stream_enable_zrtp(VideoStream *vstream, AudioStream *astream, OrtpZrtpParams *param){
Ghislain MARY's avatar
Ghislain MARY committed
803 804
	if (astream->ms.zrtp_context != NULL) {
		vstream->ms.zrtp_context=ortp_zrtp_multistream_new(astream->ms.zrtp_context, vstream->ms.session, param);
Guillaume Beraudo's avatar
Guillaume Beraudo committed
805 806
	}
}
807

808 809 810
void video_stream_enable_display_filter_auto_rotate(VideoStream* stream, bool_t enable) {
    stream->display_filter_auto_rotate_enabled = enable;
}