videostream.c 30.8 KB
Newer Older
aymeric's avatar
aymeric committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2006  Simon MORLAT (simon.morlat@linphone.org)

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/

Simon Morlat's avatar
Simon Morlat committed
20 21 22 23
#ifdef HAVE_CONFIG_H
#include "mediastreamer-config.h"
#endif

aymeric's avatar
aymeric committed
24 25
#include "mediastreamer2/mediastream.h"
#include "mediastreamer2/msfilter.h"
26
#include "mediastreamer2/msinterfaces.h"
aymeric's avatar
aymeric committed
27 28 29
#include "mediastreamer2/msvideo.h"
#include "mediastreamer2/msrtp.h"
#include "mediastreamer2/msvideoout.h"
30
#include "mediastreamer2/msextdisplay.h"
Simon Morlat's avatar
Simon Morlat committed
31

Guillaume Beraudo's avatar
Guillaume Beraudo committed
32
#include <ortp/zrtp.h>
aymeric's avatar
aymeric committed
33

Simon Morlat's avatar
Simon Morlat committed
34

Simon Morlat's avatar
Simon Morlat committed
35

36
extern RtpSession * create_duplex_rtpsession( int loc_rtp_port, int loc_rtcp_port, bool_t ipv6);
aymeric's avatar
aymeric committed
37 38 39 40 41 42

#define MAX_RTP_SIZE	UDP_MAX_SIZE

/* this code is not part of the library itself, it is part of the mediastream program */
void video_stream_free (VideoStream * stream)
{
43 44
	if (stream->ortpZrtpContext)
		ortp_zrtp_context_destroy(stream->ortpZrtpContext);
aymeric's avatar
aymeric committed
45 46 47 48 49 50
	if (stream->session!=NULL){
		rtp_session_unregister_event_queue(stream->session,stream->evq);
		rtp_session_destroy(stream->session);
	}
	if (stream->rtprecv != NULL)
		ms_filter_destroy (stream->rtprecv);
51
	if (stream->rtpsend!=NULL)
aymeric's avatar
aymeric committed
52 53 54 55 56
		ms_filter_destroy (stream->rtpsend);
	if (stream->source != NULL)
		ms_filter_destroy (stream->source);
	if (stream->output != NULL)
		ms_filter_destroy (stream->output);
57 58
	if (stream->encoder != NULL)
		ms_filter_destroy (stream->encoder);
aymeric's avatar
aymeric committed
59 60
	if (stream->decoder != NULL)
		ms_filter_destroy (stream->decoder);
61 62
	if (stream->sizeconv != NULL)
		ms_filter_destroy (stream->sizeconv);
aymeric's avatar
aymeric committed
63 64 65 66
	if (stream->pixconv!=NULL)
		ms_filter_destroy(stream->pixconv);
	if (stream->tee!=NULL)
		ms_filter_destroy(stream->tee);
67 68 69 70
	if (stream->tee2!=NULL)
		ms_filter_destroy(stream->tee2);
	if (stream->jpegwriter!=NULL)
		ms_filter_destroy(stream->jpegwriter);
71 72
	if (stream->output2!=NULL)
		ms_filter_destroy(stream->output2);
73 74
	if (stream->voidsink!=NULL)
		ms_filter_destroy(stream->voidsink);
aymeric's avatar
aymeric committed
75 76 77 78
	if (stream->ticker != NULL)
		ms_ticker_destroy (stream->ticker);
	if (stream->evq!=NULL)
		ortp_ev_queue_destroy(stream->evq);
79 80
	if (stream->display_name!=NULL)
		ms_free(stream->display_name);
81 82 83
	if (stream->rc!=NULL){
		ms_bitrate_controller_destroy(stream->rc);
	}
84

aymeric's avatar
aymeric committed
85 86 87
	ms_free (stream);
}

88 89
static void event_cb(void *ud, MSFilter* f, unsigned int event, void *eventdata){
	VideoStream *st=(VideoStream*)ud;
Simon Morlat's avatar
Simon Morlat committed
90
	ms_message("event_cb called %u", event);
91 92 93 94 95
	if (st->eventcb!=NULL){
		st->eventcb(st->event_pointer,f,event,eventdata);
	}
}

aymeric's avatar
aymeric committed
96 97 98 99 100 101 102 103 104 105 106
/*this function must be called from the MSTicker thread:
it replaces one filter by another one.
This is a dirty hack that works anyway.
It would be interesting to have something that does the job
simplier within the MSTicker api
*/
void video_stream_change_decoder(VideoStream *stream, int payload){
	RtpSession *session=stream->session;
	RtpProfile *prof=rtp_session_get_profile(session);
	PayloadType *pt=rtp_profile_get_payload(prof,payload);
	if (pt!=NULL){
Simon Morlat's avatar
Simon Morlat committed
107
		MSFilter *dec;
108

109 110 111 112 113
		if (stream->decoder!=NULL && stream->decoder->desc->enc_fmt!=NULL &&
		    strcasecmp(pt->mime_type,stream->decoder->desc->enc_fmt)==0){
			/* same formats behind different numbers, nothing to do */
				return;
		}
Simon Morlat's avatar
Simon Morlat committed
114
		dec=ms_filter_create_decoder(pt->mime_type);
aymeric's avatar
aymeric committed
115 116
		if (dec!=NULL){
			ms_filter_unlink(stream->rtprecv, 0, stream->decoder, 0);
117 118
			if (stream->tee2)
				ms_filter_unlink(stream->decoder,0,stream->tee2,0);
119
			else if(stream->output)
120
				ms_filter_unlink(stream->decoder,0,stream->output,0);
aymeric's avatar
aymeric committed
121 122 123 124 125 126
			ms_filter_postprocess(stream->decoder);
			ms_filter_destroy(stream->decoder);
			stream->decoder=dec;
			if (pt->recv_fmtp!=NULL)
				ms_filter_call_method(stream->decoder,MS_FILTER_ADD_FMTP,(void*)pt->recv_fmtp);
			ms_filter_link (stream->rtprecv, 0, stream->decoder, 0);
127 128
			if (stream->tee2)
				ms_filter_link(stream->decoder,0,stream->tee2,0);
129
			else if(stream->output)
130
				ms_filter_link (stream->decoder,0 , stream->output, 0);
aymeric's avatar
aymeric committed
131
			ms_filter_preprocess(stream->decoder,stream->ticker);
132
			ms_filter_set_notify_callback(dec, event_cb, stream);
aymeric's avatar
aymeric committed
133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148
		}else{
			ms_warning("No decoder found for %s",pt->mime_type);
		}
	}else{
		ms_warning("No payload defined with number %i",payload);
	}
}

static void video_steam_process_rtcp(VideoStream *stream, mblk_t *m){
	do{
		if (rtcp_is_SR(m)){
			const report_block_t *rb;
			ms_message("video_steam_process_rtcp: receiving RTCP SR");
			rb=rtcp_SR_get_report_block(m,0);
			if (rb){
				unsigned int ij;
149
				float rt=rtp_session_get_round_trip_propagation(stream->session);
aymeric's avatar
aymeric committed
150 151
				float flost;
				ij=report_block_get_interarrival_jitter(rb);
aymeric's avatar
aymeric committed
152
				flost=(float)(100.0*report_block_get_fraction_lost(rb)/256.0);
153
				ms_message("video_steam_process_rtcp: interarrival jitter=%u , lost packets percentage since last report=%f, round trip time=%f seconds",ij,flost,rt);
154 155
				if (stream->rc)
					ms_bitrate_controller_process_rtcp(stream->rc,m);
aymeric's avatar
aymeric committed
156 157 158 159 160
			}
		}
	}while(rtcp_next_packet(m));
}

161 162
static void stop_preload_graph(VideoStream *stream){
	ms_ticker_detach(stream->ticker,stream->rtprecv);
163 164 165 166
	ms_filter_unlink(stream->rtprecv,0,stream->voidsink,0);
	ms_filter_destroy(stream->voidsink);
	ms_filter_destroy(stream->rtprecv);
	stream->voidsink=stream->rtprecv=NULL;
167
	stream->prepare_ongoing = FALSE;
168 169
}

aymeric's avatar
aymeric committed
170
void video_stream_iterate(VideoStream *stream){
171
	/*
aymeric's avatar
aymeric committed
172 173 174
	if (stream->output!=NULL)
		ms_filter_call_method_noarg(stream->output,
			MS_VIDEO_OUT_HANDLE_RESIZING);
175
	*/
aymeric's avatar
aymeric committed
176
	if (stream->evq){
177 178
		OrtpEvent *ev;
		while (NULL != (ev=ortp_ev_queue_get(stream->evq))) {
179 180
			OrtpEventType evt=ortp_event_get_type(ev);
			if (evt == ORTP_EVENT_RTCP_PACKET_RECEIVED){
aymeric's avatar
aymeric committed
181 182
				OrtpEventData *evd=ortp_event_get_data(ev);
				video_steam_process_rtcp(stream,evd->packet);
183
			}else if ((evt == ORTP_EVENT_STUN_PACKET_RECEIVED) && (stream->ice_check_list)) {
Ghislain MARY's avatar
Ghislain MARY committed
184
				ice_handle_stun_packet(stream->ice_check_list,stream->session,ortp_event_get_data(ev));
aymeric's avatar
aymeric committed
185
			}
186
			ortp_event_destroy(ev);
aymeric's avatar
aymeric committed
187 188
		}
	}
189
	if (stream->ice_check_list) ice_check_list_process(stream->ice_check_list,stream->session);
aymeric's avatar
aymeric committed
190 191 192 193 194 195 196 197
}

static void payload_type_changed(RtpSession *session, unsigned long data){
	VideoStream *stream=(VideoStream*)data;
	int pt=rtp_session_get_recv_payload_type(stream->session);
	video_stream_change_decoder(stream,pt);
}

198 199 200
static void choose_display_name(VideoStream *stream){
#ifdef WIN32
	stream->display_name=ms_strdup("MSDrawDibDisplay");
201 202
#elif defined(ANDROID)
	stream->display_name=ms_strdup("MSAndroidDisplay");
203 204
#elif __APPLE__ && !defined(__ios)
	stream->display_name=ms_strdup("MSOSXGLDisplay");
205 206 207
#elif defined(HAVE_GL)
	stream->display_name=ms_strdup("MSGLXVideo");
#elif defined (HAVE_XV)
Simon Morlat's avatar
Simon Morlat committed
208
	stream->display_name=ms_strdup("MSX11Video");
Simon Morlat's avatar
Simon Morlat committed
209
#elif defined(__ios)
210
	stream->display_name=ms_strdup("IOSDisplay");	
Simon Morlat's avatar
Simon Morlat committed
211 212
#else
	stream->display_name=ms_strdup("MSVideoOut");
213 214 215
#endif
}

216
VideoStream *video_stream_new(int loc_rtp_port, int loc_rtcp_port, bool_t use_ipv6){
aymeric's avatar
aymeric committed
217
	VideoStream *stream = (VideoStream *)ms_new0 (VideoStream, 1);
218
	stream->session=create_duplex_rtpsession(loc_rtp_port,loc_rtcp_port,use_ipv6);
aymeric's avatar
aymeric committed
219 220
	stream->evq=ortp_ev_queue_new();
	stream->rtpsend=ms_filter_new(MS_RTP_SEND_ID);
221
	stream->ice_check_list=NULL;
aymeric's avatar
aymeric committed
222
	rtp_session_register_event_queue(stream->session,stream->evq);
smorlat's avatar
smorlat committed
223 224
	stream->sent_vsize.width=MS_VIDEO_SIZE_CIF_W;
	stream->sent_vsize.height=MS_VIDEO_SIZE_CIF_H;
225
	stream->dir=VideoStreamSendRecv;
226
	stream->display_filter_auto_rotate_enabled=0;
227 228
	choose_display_name(stream);

aymeric's avatar
aymeric committed
229 230 231
	return stream;
}

Simon Morlat's avatar
Simon Morlat committed
232 233 234 235 236
int video_stream_set_dscp(VideoStream *stream, int dscp){
	ms_message("Setting DSCP to %i for video stream.",dscp);
	return rtp_session_set_dscp(stream->session,dscp);
}

smorlat's avatar
smorlat committed
237
void video_stream_set_sent_video_size(VideoStream *stream, MSVideoSize vsize){
238
	ms_message("Setting video size %dx%d", vsize.width, vsize.height);
='s avatar
= committed
239
	stream->sent_vsize=vsize;
smorlat's avatar
smorlat committed
240 241
}

aymeric's avatar
aymeric committed
242 243 244 245
void video_stream_set_relay_session_id(VideoStream *stream, const char *id){
	ms_filter_call_method(stream->rtpsend, MS_RTP_SEND_SET_RELAY_SESSION_ID,(void*)id);
}

smorlat's avatar
smorlat committed
246
void video_stream_enable_self_view(VideoStream *stream, bool_t val){
='s avatar
= committed
247
	MSFilter *out=stream->output;
248 249 250 251
	stream->corner=val ? 0 : -1;
	if (out){
		ms_filter_call_method(out,MS_VIDEO_DISPLAY_SET_LOCAL_VIEW_MODE,&stream->corner);
	}
smorlat's avatar
smorlat committed
252
}
aymeric's avatar
aymeric committed
253 254

void video_stream_enable_adaptive_bitrate_control(VideoStream *s, bool_t yesno){
255
	s->use_rc=yesno;
aymeric's avatar
aymeric committed
256 257
}

258 259 260 261
void video_stream_enable_adaptive_jittcomp(VideoStream *st, bool_t enabled) {
	rtp_session_enable_adaptive_jitter_compensation(st->session, enabled);
}

262 263 264 265 266
void video_stream_set_render_callback (VideoStream *s, VideoStreamRenderCallback cb, void *user_pointer){
	s->rendercb=cb;
	s->render_pointer=user_pointer;
}

267 268 269 270 271
void video_stream_set_event_callback (VideoStream *s, VideoStreamEventCallback cb, void *user_pointer){
	s->eventcb=cb;
	s->event_pointer=user_pointer;
}

272 273 274 275 276 277 278 279 280
void video_stream_set_display_filter_name(VideoStream *s, const char *fname){
	if (s->display_name!=NULL){
		ms_free(s->display_name);
		s->display_name=NULL;
	}
	if (fname!=NULL)
		s->display_name=ms_strdup(fname);
}

281 282

static void ext_display_cb(void *ud, MSFilter* f, unsigned int event, void *eventdata){
283 284 285 286 287 288 289 290 291
	MSExtDisplayOutput *output=(MSExtDisplayOutput*)eventdata;
	VideoStream *st=(VideoStream*)ud;
	if (st->rendercb!=NULL){
		st->rendercb(st->render_pointer,
		            output->local_view.w!=0 ? &output->local_view : NULL,
		            output->remote_view.w!=0 ? &output->remote_view : NULL);
	}
}

292
void video_stream_set_direction(VideoStream *vs, VideoStreamDir dir){
='s avatar
= committed
293
	vs->dir=dir;
294 295
}

Simon Morlat's avatar
Simon Morlat committed
296 297 298 299
static MSVideoSize get_compatible_size(MSVideoSize maxsize, MSVideoSize wished_size){
	int max_area=maxsize.width*maxsize.height;
	int whished_area=wished_size.width*wished_size.height;
	if (whished_area>max_area){
300
		return maxsize;
Simon Morlat's avatar
Simon Morlat committed
301 302 303 304
	}
	return wished_size;
}

305 306 307 308 309 310 311 312 313 314
static MSVideoSize get_with_same_orientation(MSVideoSize size, MSVideoSize refsize){
	if (ms_video_size_get_orientation(refsize)!=ms_video_size_get_orientation(size)){
		int tmp;
		tmp=size.width;
		size.width=size.height;
		size.height=tmp;
	}
	return size;
}

315 316 317 318
static void configure_video_source(VideoStream *stream){
	MSVideoSize vsize,cam_vsize;
	float fps=15;
	MSPixFmt format;
319
	bool_t encoder_has_builtin_converter = FALSE;
320

321 322
	/* transmit orientation to source filter */
	ms_filter_call_method(stream->source,MS_VIDEO_CAPTURE_SET_DEVICE_ORIENTATION,&stream->device_orientation);
323 324
	
    /* transmit its preview window id if any to source filter*/
325 326 327
	if (stream->preview_window_id!=0){
		video_stream_set_native_preview_window_id(stream, stream->preview_window_id);
	}
328

329
	ms_filter_call_method(stream->encoder, MS_VIDEO_ENCODER_HAS_BUILTIN_CONVERTER, &encoder_has_builtin_converter);
330
	ms_filter_call_method(stream->encoder,MS_FILTER_GET_VIDEO_SIZE,&vsize);
Simon Morlat's avatar
Simon Morlat committed
331
	vsize=get_compatible_size(vsize,stream->sent_vsize);
332 333 334
	ms_filter_call_method(stream->source,MS_FILTER_SET_VIDEO_SIZE,&vsize);
	/*the camera may not support the target size and suggest a one close to the target */
	ms_filter_call_method(stream->source,MS_FILTER_GET_VIDEO_SIZE,&cam_vsize);
335 336
	if (cam_vsize.width*cam_vsize.height<=vsize.width*vsize.height &&
			cam_vsize.width != vsize.width){
337
		vsize=cam_vsize;
338
		ms_message("Output video size adjusted to match camera resolution (%ix%i)\n",vsize.width,vsize.height);
339
	} else if (cam_vsize.width*cam_vsize.height>vsize.width*vsize.height){
340
#if TARGET_IPHONE_SIMULATOR || defined(__arm__)
341 342 343 344 345 346
		ms_error("Camera is proposing a size bigger than encoder's suggested size (%ix%i > %ix%i) "
		           "Using the camera size as fallback because cropping or resizing is not implemented for arm.",
		           cam_vsize.width,cam_vsize.height,vsize.width,vsize.height);
		vsize=cam_vsize;
#else
		vsize=get_with_same_orientation(vsize,cam_vsize);
347
		ms_warning("Camera video size greater than encoder one. A scaling filter will be used!\n");
348
#endif
349
	}
350 351 352 353 354 355 356 357 358
	ms_filter_call_method(stream->encoder,MS_FILTER_SET_VIDEO_SIZE,&vsize);
	ms_filter_call_method(stream->encoder,MS_FILTER_GET_FPS,&fps);
	ms_message("Setting sent vsize=%ix%i, fps=%f",vsize.width,vsize.height,fps);
	/* configure the filters */
	if (ms_filter_get_id(stream->source)!=MS_STATIC_IMAGE_ID) {
		ms_filter_call_method(stream->source,MS_FILTER_SET_FPS,&fps);
	}
	/* get the output format for webcam reader */
	ms_filter_call_method(stream->source,MS_FILTER_GET_PIX_FMT,&format);
359

360 361 362 363 364 365 366 367 368 369 370 371 372
	if (encoder_has_builtin_converter == TRUE) {
		ms_filter_call_method(stream->encoder, MS_FILTER_SET_PIX_FMT, &format);
	} else {
		if (format==MS_MJPEG){
			stream->pixconv=ms_filter_new(MS_MJPEG_DEC_ID);
		}else{
			stream->pixconv = ms_filter_new(MS_PIX_CONV_ID);
			/*set it to the pixconv */
			ms_filter_call_method(stream->pixconv,MS_FILTER_SET_PIX_FMT,&format);
			ms_filter_call_method(stream->pixconv,MS_FILTER_SET_VIDEO_SIZE,&cam_vsize);
		}
		stream->sizeconv=ms_filter_new(MS_SIZE_CONV_ID);
		ms_filter_call_method(stream->sizeconv,MS_FILTER_SET_VIDEO_SIZE,&vsize);
373
	}
374 375 376 377 378 379 380
	if (stream->rc){
		ms_bitrate_controller_destroy(stream->rc);
		stream->rc=NULL;
	}
	if (stream->use_rc){
		stream->rc=ms_av_bitrate_controller_new(NULL,NULL,stream->session,stream->encoder);
	}
381 382
}

383 384 385 386 387 388 389 390 391 392 393
static void start_ticker(VideoStream *stream){
	MSTickerParams params={0};
	params.name="Video MSTicker";
#ifdef __ios
    params.prio=MS_TICKER_PRIO_HIGH;
#else
	params.prio=MS_TICKER_PRIO_NORMAL;
#endif
	stream->ticker = ms_ticker_new_with_params(&params);
}

394 395
int video_stream_start (VideoStream *stream, RtpProfile *profile, const char *rem_rtp_ip, int rem_rtp_port,
	const char *rem_rtcp_ip, int rem_rtcp_port, int payload, int jitt_comp, MSWebCam *cam){
aymeric's avatar
aymeric committed
396 397 398
	PayloadType *pt;
	RtpSession *rtps=stream->session;
	MSPixFmt format;
399
	MSVideoSize disp_size;
smorlat's avatar
smorlat committed
400 401 402
	int tmp;
	JBParameters jbp;
	const int socket_buf_size=2000000;
aymeric's avatar
aymeric committed
403

404 405
	if (cam==NULL){
		cam=ms_web_cam_manager_get_default_cam (
406
		      ms_web_cam_manager_get());
407 408
	}

aymeric's avatar
aymeric committed
409 410 411 412 413
	pt=rtp_profile_get_payload(profile,payload);
	if (pt==NULL){
		ms_error("videostream.c: undefined payload type.");
		return -1;
	}
414

aymeric's avatar
aymeric committed
415
	rtp_session_set_profile(rtps,profile);
416
	if (rem_rtp_port>0) rtp_session_set_remote_addr_full(rtps,rem_rtp_ip,rem_rtp_port,rem_rtcp_ip,rem_rtcp_port);
aymeric's avatar
aymeric committed
417 418 419 420 421 422
	rtp_session_set_payload_type(rtps,payload);
	rtp_session_set_jitter_compensation(rtps,jitt_comp);

	rtp_session_signal_connect(stream->session,"payload_type_changed",
			(RtpCallback)payload_type_changed,(unsigned long)stream);

smorlat's avatar
smorlat committed
423 424 425 426 427
	rtp_session_get_jitter_buffer_params(stream->session,&jbp);
	jbp.max_packets=1000;//needed for high resolution video
	rtp_session_set_jitter_buffer_params(stream->session,&jbp);
	rtp_session_set_rtp_socket_recv_buffer_size(stream->session,socket_buf_size);
	rtp_session_set_rtp_socket_send_buffer_size(stream->session,socket_buf_size);
428

429
	if (stream->dir==VideoStreamSendRecv || stream->dir==VideoStreamSendOnly){
430
		MSConnectionHelper ch;
431 432
		/*plumb the outgoing stream */

433
		if (rem_rtp_port>0) ms_filter_call_method(stream->rtpsend,MS_RTP_SEND_SET_SESSION,stream->session);
434 435 436 437 438 439 440
		stream->encoder=ms_filter_create_encoder(pt->mime_type);
		if ((stream->encoder==NULL) ){
			/* big problem: we don't have a registered codec for this payload...*/
			ms_error("videostream.c: No encoder available for payload %i:%s.",payload,pt->mime_type);
			return -1;
		}
		/* creates the filters */
441
		stream->cam=cam;
442 443
		stream->source = ms_web_cam_create_reader(cam);
		stream->tee = ms_filter_new(MS_TEE_ID);
444

445 446 447 448 449 450 451
		if (pt->normal_bitrate>0){
			ms_message("Limiting bitrate of video encoder to %i bits/s",pt->normal_bitrate);
			ms_filter_call_method(stream->encoder,MS_FILTER_SET_BITRATE,&pt->normal_bitrate);
		}
		if (pt->send_fmtp){
			ms_filter_call_method(stream->encoder,MS_FILTER_ADD_FMTP,pt->send_fmtp);
		}
452 453 454 455 456
		if (stream->use_preview_window){
			if (stream->rendercb==NULL){
				stream->output2=ms_filter_new_from_name (stream->display_name);
			}
		}
457

458
		configure_video_source (stream);
459 460 461 462 463 464 465 466 467 468 469 470
		/* and then connect all */
		ms_connection_helper_start(&ch);
		ms_connection_helper_link(&ch, stream->source, -1, 0);
		if (stream->pixconv) {
			ms_connection_helper_link(&ch, stream->pixconv, 0, 0);
		}
		if (stream->sizeconv) {
			ms_connection_helper_link(&ch, stream->sizeconv, 0, 0);
		}
		ms_connection_helper_link(&ch, stream->tee, 0, 0);
		ms_connection_helper_link(&ch, stream->encoder, 0, 0);
		ms_connection_helper_link(&ch, stream->rtpsend, 0, -1);
471
		if (stream->output2){
472
			if (stream->preview_window_id!=0){
473 474 475 476
				ms_filter_call_method(stream->output2, MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&stream->preview_window_id);
			}
			ms_filter_link(stream->tee,1,stream->output2,0);
		}
aymeric's avatar
aymeric committed
477
	}
478
	if (stream->dir==VideoStreamSendRecv || stream->dir==VideoStreamRecvOnly){
479
		MSConnectionHelper ch;
480 481 482 483 484 485 486
		/*plumb the incoming stream */
		stream->decoder=ms_filter_create_decoder(pt->mime_type);
		if ((stream->decoder==NULL) ){
			/* big problem: we don't have a registered decoderfor this payload...*/
			ms_error("videostream.c: No decoder available for payload %i:%s.",payload,pt->mime_type);
			return -1;
		}
487 488
		ms_filter_set_notify_callback(stream->decoder, event_cb, stream);

489 490
		stream->rtprecv = ms_filter_new (MS_RTP_RECV_ID);
		ms_filter_call_method(stream->rtprecv,MS_RTP_RECV_SET_SESSION,stream->session);
491

492

493
		stream->jpegwriter=ms_filter_new(MS_JPEG_WRITER_ID);
494 495
		if (stream->jpegwriter)
			stream->tee2=ms_filter_new(MS_TEE_ID);
496

497 498
		if (stream->rendercb!=NULL){
			stream->output=ms_filter_new(MS_EXT_DISPLAY_ID);
499
			ms_filter_set_notify_callback(stream->output,ext_display_cb,stream);
500 501 502
		}else{
			stream->output=ms_filter_new_from_name (stream->display_name);
		}
503 504 505 506 507 508

		/* Don't allow null output */
		if(stream->output == NULL) {
			ms_fatal("No video display filter could be instantiated. Please check build-time configuration");
		}

509 510 511 512 513 514
		/* set parameters to the decoder*/
		if (pt->send_fmtp){
			ms_filter_call_method(stream->decoder,MS_FILTER_ADD_FMTP,pt->send_fmtp);
		}
		if (pt->recv_fmtp!=NULL)
			ms_filter_call_method(stream->decoder,MS_FILTER_ADD_FMTP,(void*)pt->recv_fmtp);
515

516 517 518 519 520
		/*force the decoder to output YUV420P */
		format=MS_YUV420P;
		ms_filter_call_method(stream->decoder,MS_FILTER_SET_PIX_FMT,&format);

		/*configure the display window */
521 522 523 524 525 526 527 528 529 530 531 532 533 534
		if(stream->output != NULL) {
			disp_size.width=MS_VIDEO_SIZE_CIF_W;
			disp_size.height=MS_VIDEO_SIZE_CIF_H;
			tmp=1;
			ms_filter_call_method(stream->output,MS_FILTER_SET_VIDEO_SIZE,&disp_size);
			ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_ENABLE_AUTOFIT,&tmp);
			ms_filter_call_method(stream->output,MS_FILTER_SET_PIX_FMT,&format);
			ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SET_LOCAL_VIEW_MODE,&stream->corner);
			if (stream->window_id!=0){
				ms_filter_call_method(stream->output, MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&stream->window_id);
			}
			if (stream->display_filter_auto_rotate_enabled) {
				ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SET_DEVICE_ORIENTATION,&stream->device_orientation);
			}
535
		}
536
		/* and connect the filters */
537 538 539 540 541 542 543
		ms_connection_helper_start (&ch);
		ms_connection_helper_link (&ch,stream->rtprecv,-1,0);
		ms_connection_helper_link (&ch,stream->decoder,0,0);
		if (stream->tee2){
			ms_connection_helper_link (&ch,stream->tee2,0,0);
			ms_filter_link(stream->tee2,1,stream->jpegwriter,0);
		}
544 545
		if (stream->output!=NULL)
			ms_connection_helper_link (&ch,stream->output,0,-1);
546
		/* the video source must be send for preview , if it exists*/
547
		if (stream->tee!=NULL && stream->output!=NULL && stream->output2==NULL)
548
			ms_filter_link(stream->tee,1,stream->output,1);
aymeric's avatar
aymeric committed
549
	}
550 551 552 553 554 555
	if (stream->dir == VideoStreamSendOnly) {
		stream->rtprecv = ms_filter_new (MS_RTP_RECV_ID);
		ms_filter_call_method(stream->rtprecv, MS_RTP_RECV_SET_SESSION, stream->session);
		stream->voidsink = ms_filter_new(MS_VOID_SINK_ID);
		ms_filter_link(stream->rtprecv, 0, stream->voidsink, 0);
	}
556

aymeric's avatar
aymeric committed
557
	/* create the ticker */
558
	if (stream->ticker==NULL) start_ticker(stream);
559

560 561 562 563 564
	/* attach the graphs */
	if (stream->source)
		ms_ticker_attach (stream->ticker, stream->source);
	if (stream->rtprecv)
		ms_ticker_attach (stream->ticker, stream->rtprecv);
aymeric's avatar
aymeric committed
565 566 567
	return 0;
}

568
void video_stream_prepare_video(VideoStream *stream){
569
	stream->prepare_ongoing = TRUE;
570
	video_stream_unprepare_video(stream);
571 572 573
	stream->rtprecv=ms_filter_new(MS_RTP_RECV_ID);
	rtp_session_set_payload_type(stream->session,0);
	ms_filter_call_method(stream->rtprecv,MS_RTP_RECV_SET_SESSION,stream->session);
574
	stream->voidsink=ms_filter_new(MS_VOID_SINK_ID);
575
	ms_filter_link(stream->rtprecv,0,stream->voidsink,0);
576
	start_ticker(stream);
577 578 579
	ms_ticker_attach(stream->ticker,stream->rtprecv);
}

580 581 582 583 584 585
void video_stream_unprepare_video(VideoStream *stream){
	if (stream->voidsink) {
		stop_preload_graph(stream);
	}
}

586 587 588 589 590
void video_stream_update_video_params(VideoStream *stream){
	/*calling video_stream_change_camera() does the job of unplumbing/replumbing and configuring the new graph*/
	video_stream_change_camera(stream,stream->cam);
}

591
void video_stream_change_camera(VideoStream *stream, MSWebCam *cam){
Simon Morlat's avatar
Simon Morlat committed
592
	bool_t keep_source=(cam==stream->cam);
593
	
594 595 596 597 598 599 600
	if (stream->ticker && stream->source){
		ms_ticker_detach(stream->ticker,stream->source);
		/*unlink source filters and subsequent post processin filters */
		ms_filter_unlink (stream->source, 0, stream->pixconv, 0);
		ms_filter_unlink (stream->pixconv, 0, stream->sizeconv, 0);
		ms_filter_unlink (stream->sizeconv, 0, stream->tee, 0);
		/*destroy the filters */
Simon Morlat's avatar
Simon Morlat committed
601
		if (!keep_source) ms_filter_destroy(stream->source);
602 603 604 605
		ms_filter_destroy(stream->pixconv);
		ms_filter_destroy(stream->sizeconv);

		/*re create new ones and configure them*/
Simon Morlat's avatar
Simon Morlat committed
606
		if (!keep_source) stream->source = ms_web_cam_create_reader(cam);
607
		stream->cam=cam;
608 609 610 611 612 613 614 615 616 617 618 619 620
        
        /* update orientation */
        if (stream->source){
            ms_filter_call_method(stream->source,MS_VIDEO_CAPTURE_SET_DEVICE_ORIENTATION,&stream->device_orientation);
            
            /* */
            if (!stream->display_filter_auto_rotate_enabled)
                ms_filter_call_method(stream->source,MS_VIDEO_DISPLAY_SET_DEVICE_ORIENTATION,&stream->device_orientation);
        }
        if (stream->output && stream->display_filter_auto_rotate_enabled) {
            ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SET_DEVICE_ORIENTATION,&stream->device_orientation);
        }
        
621
		configure_video_source(stream);
622
		
623 624 625
		ms_filter_link (stream->source, 0, stream->pixconv, 0);
		ms_filter_link (stream->pixconv, 0, stream->sizeconv, 0);
		ms_filter_link (stream->sizeconv, 0, stream->tee, 0);
626

627 628 629 630
		ms_ticker_attach(stream->ticker,stream->source);
	}
}

aymeric's avatar
aymeric committed
631 632
void video_stream_send_vfu(VideoStream *stream){
	if (stream->encoder)
633
		ms_filter_call_method_noarg(stream->encoder, MS_VIDEO_ENCODER_REQ_VFU);
aymeric's avatar
aymeric committed
634 635 636 637 638
}

void
video_stream_stop (VideoStream * stream)
{
639 640
	stream->eventcb = NULL;
	stream->event_pointer = NULL;
aymeric's avatar
aymeric committed
641
	if (stream->ticker){
642
		if (stream->prepare_ongoing == TRUE) {
643
			stop_preload_graph(stream);
644 645 646 647 648 649
		} else {
			if (stream->source)
				ms_ticker_detach(stream->ticker,stream->source);
			if (stream->rtprecv)
				ms_ticker_detach(stream->ticker,stream->rtprecv);

650 651 652 653
			if (stream->ice_check_list != NULL) {
				ice_check_list_print_route(stream->ice_check_list, "Video session's route");
				stream->ice_check_list = NULL;
			}
654 655 656
			rtp_stats_display(rtp_session_get_stats(stream->session),"Video session's RTP statistics");

			if (stream->source){
657 658 659 660 661 662 663 664 665 666 667 668
				MSConnectionHelper ch;
				ms_connection_helper_start(&ch);
				ms_connection_helper_unlink(&ch, stream->source, -1, 0);
				if (stream->pixconv) {
					ms_connection_helper_unlink(&ch, stream->pixconv, 0, 0);
				}
				if (stream->sizeconv) {
					ms_connection_helper_unlink(&ch, stream->sizeconv, 0, 0);
				}
				ms_connection_helper_unlink(&ch, stream->tee, 0, 0);
				ms_connection_helper_unlink(&ch, stream->encoder, 0, 0);
				ms_connection_helper_unlink(&ch, stream->rtpsend, 0, -1);
669 670 671
				if (stream->output2){
					ms_filter_unlink(stream->tee,1,stream->output2,0);
				}
672
			}
673 674 675
			if (stream->voidsink) {
				ms_filter_unlink(stream->rtprecv, 0, stream->voidsink, 0);
			} else if (stream->rtprecv){
676 677 678 679 680 681 682 683 684 685 686 687
				MSConnectionHelper h;
				ms_connection_helper_start (&h);
				ms_connection_helper_unlink (&h,stream->rtprecv,-1,0);
				ms_connection_helper_unlink (&h,stream->decoder,0,0);
				if (stream->tee2){
					ms_connection_helper_unlink (&h,stream->tee2,0,0);
					ms_filter_unlink(stream->tee2,1,stream->jpegwriter,0);
				}
				if(stream->output)
					ms_connection_helper_unlink (&h,stream->output,0,-1);
				if (stream->tee && stream->output && stream->output2==NULL)
					ms_filter_unlink(stream->tee,1,stream->output,1);
688
			}
689
		}
aymeric's avatar
aymeric committed
690 691 692 693 694 695
	}
	video_stream_free (stream);
}


void video_stream_set_rtcp_information(VideoStream *st, const char *cname, const char *tool){
='s avatar
= committed
696
	if (st->session!=NULL){
697
		rtp_session_set_source_description(st->session,cname,NULL,NULL,NULL,NULL,tool,NULL);
aymeric's avatar
aymeric committed
698 699 700
	}
}

701 702 703 704 705 706 707
void video_stream_show_video(VideoStream *stream, bool_t show){
	if (stream->output){
		ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SHOW_VIDEO,&show);
	}
}


708 709 710
unsigned long video_stream_get_native_window_id(VideoStream *stream){
	unsigned long id;
	if (stream->output){
unknown's avatar
unknown committed
711
		if (ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_GET_NATIVE_WINDOW_ID,&id)==0)
712 713
			return id;
	}
714 715 716 717 718
	return stream->window_id;
}

void video_stream_set_native_window_id(VideoStream *stream, unsigned long id){
	stream->window_id=id;
719 720 721
	if (stream->output){
		ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&id);
	}
722 723 724 725
}

void video_stream_set_native_preview_window_id(VideoStream *stream, unsigned long id){
	stream->preview_window_id=id;
726
#ifndef __ios	
727 728 729
	if (stream->output2){
		ms_filter_call_method(stream->output2,MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&id);
	}
730
#endif
731 732
	if (stream->source){
		ms_filter_call_method(stream->source,MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&id);
733
	}
734 735
}

Simon Morlat's avatar
wip  
Simon Morlat committed
736
unsigned long video_stream_get_native_preview_window_id(VideoStream *stream){
Simon Morlat's avatar
Simon Morlat committed
737
	unsigned long id=0;
Simon Morlat's avatar
Simon Morlat committed
738 739 740 741 742
	if (stream->output2){
		if (ms_filter_call_method(stream->output2,MS_VIDEO_DISPLAY_GET_NATIVE_WINDOW_ID,&id)==0)
			return id;
	}
	if (stream->source){
743
		if (ms_filter_has_method(stream->source,MS_VIDEO_DISPLAY_GET_NATIVE_WINDOW_ID) 
Simon Morlat's avatar
Simon Morlat committed
744
		    && ms_filter_call_method(stream->source,MS_VIDEO_DISPLAY_GET_NATIVE_WINDOW_ID,&id)==0)
745 746 747 748 749 750
			return id;
	}
	return stream->preview_window_id;
}

void video_stream_use_preview_video_window(VideoStream *stream, bool_t yesno){
='s avatar
= committed
751
	stream->use_preview_window=yesno;
752
}
aymeric's avatar
aymeric committed
753

754
void video_stream_set_device_rotation(VideoStream *stream, int orientation){
755
	stream->device_orientation = orientation;
756 757
}

Simon Morlat's avatar
Simon Morlat committed
758 759 760 761 762 763 764 765
VideoPreview * video_preview_new(void){
	VideoPreview *stream = (VideoPreview *)ms_new0 (VideoPreview, 1);
	stream->sent_vsize.width=MS_VIDEO_SIZE_CIF_W;
	stream->sent_vsize.height=MS_VIDEO_SIZE_CIF_H;
	choose_display_name(stream);
	return stream;
}

aymeric's avatar
aymeric committed
766

Simon Morlat's avatar
Simon Morlat committed
767
void video_preview_start(VideoPreview *stream, MSWebCam *device){
aymeric's avatar
aymeric committed
768
	MSPixFmt format;
aymeric's avatar
aymeric committed
769
	float fps=(float)29.97;
smorlat's avatar
smorlat committed
770
	int mirroring=1;
Simon Morlat's avatar
Simon Morlat committed
771
	int corner=-1;
Simon Morlat's avatar
Simon Morlat committed
772 773 774
	MSVideoSize disp_size=stream->sent_vsize;
	MSVideoSize vsize=disp_size;
	const char *displaytype=stream->display_name;
aymeric's avatar
aymeric committed
775 776

	stream->source = ms_web_cam_create_reader(device);
777

778

aymeric's avatar
aymeric committed
779
	/* configure the filters */
smorlat's avatar
smorlat committed
780
	ms_filter_call_method(stream->source,MS_FILTER_SET_VIDEO_SIZE,&vsize);
781 782
	if (ms_filter_get_id(stream->source)!=MS_STATIC_IMAGE_ID)
		ms_filter_call_method(stream->source,MS_FILTER_SET_FPS,&fps);
smorlat's avatar
smorlat committed
783
	ms_filter_call_method(stream->source,MS_FILTER_GET_PIX_FMT,&format);
784
	ms_filter_call_method(stream->source,MS_FILTER_GET_VIDEO_SIZE,&vsize);
aymeric's avatar
aymeric committed
785 786 787 788 789 790 791
	if (format==MS_MJPEG){
		stream->pixconv=ms_filter_new(MS_MJPEG_DEC_ID);
	}else{
		stream->pixconv=ms_filter_new(MS_PIX_CONV_ID);
		ms_filter_call_method(stream->pixconv,MS_FILTER_SET_PIX_FMT,&format);
		ms_filter_call_method(stream->pixconv,MS_FILTER_SET_VIDEO_SIZE,&vsize);
	}
792

aymeric's avatar
aymeric committed
793
	format=MS_YUV420P;
Simon Morlat's avatar
Simon Morlat committed
794

795
	stream->output2=ms_filter_new_from_name (displaytype);
Simon Morlat's avatar
Simon Morlat committed
796 797 798 799
	ms_filter_call_method(stream->output2,MS_FILTER_SET_PIX_FMT,&format);
	ms_filter_call_method(stream->output2,MS_FILTER_SET_VIDEO_SIZE,&disp_size);
	ms_filter_call_method(stream->output2,MS_VIDEO_DISPLAY_ENABLE_MIRRORING,&mirroring);
	ms_filter_call_method(stream->output2,MS_VIDEO_DISPLAY_SET_LOCAL_VIEW_MODE,&corner);
aymeric's avatar
aymeric committed
800
	/* and then connect all */
801

aymeric's avatar
aymeric committed
802
	ms_filter_link(stream->source,0, stream->pixconv,0);
Simon Morlat's avatar
Simon Morlat committed
803
	ms_filter_link(stream->pixconv, 0, stream->output2, 0);
804

805
	if (stream->preview_window_id!=0){
806
		video_stream_set_native_preview_window_id(stream, stream->preview_window_id);
807
	}
aymeric's avatar
aymeric committed
808
	/* create the ticker */
smorlat's avatar
smorlat committed
809 810
	stream->ticker = ms_ticker_new();
	ms_ticker_set_name(stream->ticker,"Video MSTicker");
aymeric's avatar
aymeric committed
811 812 813 814 815 816
	ms_ticker_attach (stream->ticker, stream->source);
}

void video_preview_stop(VideoStream *stream){
	ms_ticker_detach(stream->ticker, stream->source);
	ms_filter_unlink(stream->source,0,stream->pixconv,0);
Simon Morlat's avatar
Simon Morlat committed
817
	ms_filter_unlink(stream->pixconv,0,stream->output2,0);
aymeric's avatar
aymeric committed
818 819 820
	video_stream_free(stream);
}

821 822
int video_stream_recv_only_start(VideoStream *videostream, RtpProfile *profile, const char *addr, int port, int used_pt, int jitt_comp){
	video_stream_set_direction(videostream,VideoStreamRecvOnly);
823
	return video_stream_start(videostream,profile,addr,port,addr,port+1,used_pt,jitt_comp,NULL);
824 825 826
}

int video_stream_send_only_start(VideoStream *videostream,
827
				RtpProfile *profile, const char *addr, int port, int rtcp_port,
828 829
				int used_pt, int  jitt_comp, MSWebCam *device){
	video_stream_set_direction (videostream,VideoStreamSendOnly);
830
	return video_stream_start(videostream,profile,addr,port,addr,rtcp_port,used_pt,jitt_comp,device);
831 832 833 834 835 836 837 838 839 840
}


void video_stream_recv_only_stop(VideoStream *vs){
	video_stream_stop(vs);
}

void video_stream_send_only_stop(VideoStream *vs){
	video_stream_stop(vs);
}
Guillaume Beraudo's avatar
Guillaume Beraudo committed
841 842 843 844

/* enable ZRTP on the video stream using information from the audio stream */
void video_stream_enable_zrtp(VideoStream *vstream, AudioStream *astream, OrtpZrtpParams *param){
	if (astream->ortpZrtpContext != NULL) {
845
		vstream->ortpZrtpContext=ortp_zrtp_multistream_new(astream->ortpZrtpContext, vstream->session, param);
Guillaume Beraudo's avatar
Guillaume Beraudo committed
846 847
	}
}
848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877

bool_t video_stream_enable_strp(VideoStream* stream, enum ortp_srtp_crypto_suite_t suite, const char* snd_key, const char* rcv_key) {
	// assign new srtp transport to stream->session
	// with 2 Master Keys
	RtpTransport *rtp_tpt, *rtcp_tpt;	
	
	if (!ortp_srtp_supported()) {
		ms_error("ortp srtp support not enabled");
		return FALSE;
	}
	
	ms_message("%s: stream=%p key='%s' key='%s'", __FUNCTION__,
		stream, snd_key, rcv_key);
	 
	stream->srtp_session = ortp_srtp_create_configure_session(suite, 
		rtp_session_get_send_ssrc(stream->session), 
		snd_key,
		rcv_key); 
	
	if (!stream->srtp_session) {
		return FALSE;
	}
	
	// TODO: check who will free rtp_tpt ?
	srtp_transport_new(stream->srtp_session, &rtp_tpt, &rtcp_tpt);
	
	rtp_session_set_transports(stream->session, rtp_tpt, rtcp_tpt);
	
	return TRUE;
}
878 879 880 881

void video_stream_enable_display_filter_auto_rotate(VideoStream* stream, bool_t enable) {
    stream->display_filter_auto_rotate_enabled = enable;
}
882 883 884 885 886 887 888

void video_stream_get_local_rtp_stats(VideoStream *stream, rtp_stats_t *lstats){
	if (stream->session){
		const rtp_stats_t *stats=rtp_session_get_stats(stream->session);
		memcpy(lstats,stats,sizeof(*stats));
	}else memset(lstats,0,sizeof(rtp_stats_t));
}