videostream.c 34.8 KB
Newer Older
aymeric's avatar
aymeric committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2006  Simon MORLAT (simon.morlat@linphone.org)

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/

#include "mediastreamer2/mediastream.h"
#include "mediastreamer2/msfilter.h"
22
#include "mediastreamer2/msinterfaces.h"
aymeric's avatar
aymeric committed
23 24 25
#include "mediastreamer2/msvideo.h"
#include "mediastreamer2/msrtp.h"
#include "mediastreamer2/msvideoout.h"
26
#include "mediastreamer2/msextdisplay.h"
Ghislain MARY's avatar
Ghislain MARY committed
27
#include "private.h"
Simon Morlat's avatar
Simon Morlat committed
28

Guillaume Beraudo's avatar
Guillaume Beraudo committed
29
#include <ortp/zrtp.h>
aymeric's avatar
aymeric committed
30

Simon Morlat's avatar
Simon Morlat committed
31

Ghislain MARY's avatar
Ghislain MARY committed
32
void video_stream_free(VideoStream *stream) {
33 34 35 36 37 38 39 40
	/* Prevent filters from being destroyed two times */
	if (stream->source_performs_encoding == TRUE) {
		stream->ms.encoder = NULL;
	}
	if (stream->output_performs_decoding == TRUE) {
		stream->ms.decoder = NULL;
	}

Ghislain MARY's avatar
Ghislain MARY committed
41
	media_stream_free(&stream->ms);
42

aymeric's avatar
aymeric committed
43 44 45 46
	if (stream->source != NULL)
		ms_filter_destroy (stream->source);
	if (stream->output != NULL)
		ms_filter_destroy (stream->output);
47 48
	if (stream->sizeconv != NULL)
		ms_filter_destroy (stream->sizeconv);
aymeric's avatar
aymeric committed
49 50 51 52
	if (stream->pixconv!=NULL)
		ms_filter_destroy(stream->pixconv);
	if (stream->tee!=NULL)
		ms_filter_destroy(stream->tee);
53 54 55 56
	if (stream->tee2!=NULL)
		ms_filter_destroy(stream->tee2);
	if (stream->jpegwriter!=NULL)
		ms_filter_destroy(stream->jpegwriter);
57 58
	if (stream->output2!=NULL)
		ms_filter_destroy(stream->output2);
59 60
	if (stream->display_name!=NULL)
		ms_free(stream->display_name);
61

aymeric's avatar
aymeric committed
62 63 64
	ms_free (stream);
}

65 66
static void event_cb(void *ud, MSFilter* f, unsigned int event, void *eventdata){
	VideoStream *st=(VideoStream*)ud;
Simon Morlat's avatar
Simon Morlat committed
67
	ms_message("event_cb called %u", event);
68 69 70 71 72
	if (st->eventcb!=NULL){
		st->eventcb(st->event_pointer,f,event,eventdata);
	}
}

73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95
static void internal_event_cb(void *ud, MSFilter *f, unsigned int event, void *eventdata) {
	VideoStream *stream = (VideoStream *)ud;
	const MSVideoCodecSLI *sli;
	const MSVideoCodecRPSI *rpsi;

	switch (event) {
		case MS_VIDEO_DECODER_SEND_PLI:
			ms_message("Send PLI on videostream [%p]", stream);
			video_stream_send_pli(stream);
			break;
		case MS_VIDEO_DECODER_SEND_SLI:
			sli = (const MSVideoCodecSLI *)eventdata;
			ms_message("Send SLI on videostream [%p]", stream);
			video_stream_send_sli(stream, sli->first, sli->number, sli->picture_id);
			break;
		case MS_VIDEO_DECODER_SEND_RPSI:
			rpsi = (const MSVideoCodecRPSI *)eventdata;
			ms_message("Send RPSI on videostream [%p]", stream);
			video_stream_send_rpsi(stream, rpsi->bit_string, rpsi->bit_string_len);
			break;
	}
}

Simon Morlat's avatar
Simon Morlat committed
96
static void video_stream_process_rtcp(MediaStream *media_stream, mblk_t *m){
jehan's avatar
jehan committed
97
	VideoStream *stream = (VideoStream *)media_stream;
Ghislain MARY's avatar
Ghislain MARY committed
98 99
	unsigned int i;

aymeric's avatar
aymeric committed
100
	do{
101
		const report_block_t *rb = NULL;
aymeric's avatar
aymeric committed
102 103
		if (rtcp_is_SR(m)){
			rb=rtcp_SR_get_report_block(m,0);
104 105 106 107 108 109 110 111 112 113 114
		}else if (rtcp_is_RR(m)){
			rb=rtcp_RR_get_report_block(m,0);
		}
		if (rb){
			float rt=rtp_session_get_round_trip_propagation(stream->ms.sessions.rtp_session);
			float flost;
			i=report_block_get_interarrival_jitter(rb);
			flost=(float)(100.0*report_block_get_fraction_lost(rb)/256.0);
			ms_message("video_stream_process_rtcp[%p]: interarrival jitter=%u , lost packets percentage since last report=%f, round trip time=%f seconds",stream,i,flost,rt);
			if (stream->ms.rc)
				ms_bitrate_controller_process_rtcp(stream->ms.rc,m);
115 116 117
		} else if (rtcp_is_PSFB(m)) {
			if (rtcp_PSFB_get_type(m) == RTCP_PSFB_FIR) {
				/* Special case for FIR where the packet sender ssrc must be equal to 0. */
118
				if ((rtcp_PSFB_get_media_source_ssrc(m) == rtp_session_get_send_ssrc(stream->ms.sessions.rtp_session))
119 120 121 122
					&& (rtcp_PSFB_get_packet_sender_ssrc(m) == 0)) {
					for (i = 0; ; i++) {
						rtcp_fb_fir_fci_t *fci = rtcp_PSFB_fir_get_fci(m, i);
						if (fci == NULL) break;
123
						if (rtcp_fb_fir_fci_get_ssrc(fci) == rtp_session_get_recv_ssrc(stream->ms.sessions.rtp_session)) {
Ghislain MARY's avatar
Ghislain MARY committed
124
							uint8_t seq_nr = rtcp_fb_fir_fci_get_seq_nr(fci);
125 126 127 128 129 130
							ms_filter_call_method(stream->ms.encoder, MS_VIDEO_ENCODER_NOTIFY_FIR, &seq_nr);
							break;
						}
					}
				}
			} else {
131 132
				if ((rtcp_PSFB_get_media_source_ssrc(m) == rtp_session_get_send_ssrc(stream->ms.sessions.rtp_session))
					&& (rtcp_PSFB_get_packet_sender_ssrc(m) == rtp_session_get_recv_ssrc(stream->ms.sessions.rtp_session))) {
133 134 135 136
					switch (rtcp_PSFB_get_type(m)) {
						case RTCP_PSFB_PLI:
							ms_filter_call_method_noarg(stream->ms.encoder, MS_VIDEO_ENCODER_NOTIFY_PLI);
							break;
Ghislain MARY's avatar
Ghislain MARY committed
137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
						case RTCP_PSFB_SLI:
							for (i = 0; ; i++) {
								rtcp_fb_sli_fci_t *fci = rtcp_PSFB_sli_get_fci(m, i);
								MSVideoCodecSLI sli;
								if (fci == NULL) break;
								sli.first = rtcp_fb_sli_fci_get_first(fci);
								sli.number = rtcp_fb_sli_fci_get_number(fci);
								sli.picture_id = rtcp_fb_sli_fci_get_picture_id(fci);
								ms_filter_call_method(stream->ms.encoder, MS_VIDEO_ENCODER_NOTIFY_SLI, &sli);
							}
							break;
						case RTCP_PSFB_RPSI:
						{
							rtcp_fb_rpsi_fci_t *fci = rtcp_PSFB_rpsi_get_fci(m);
							MSVideoCodecRPSI rpsi;
							rpsi.bit_string = rtcp_fb_rpsi_fci_get_bit_string(fci);
							rpsi.bit_string_len = rtcp_PSFB_rpsi_get_fci_bit_string_len(m);
							ms_filter_call_method(stream->ms.encoder, MS_VIDEO_ENCODER_NOTIFY_RPSI, &rpsi);
						}
							break;
157 158 159 160 161
						default:
							break;
					}
				}
			}
aymeric's avatar
aymeric committed
162 163 164 165
		}
	}while(rtcp_next_packet(m));
}

166
static void stop_preload_graph(VideoStream *stream){
167
	ms_ticker_detach(stream->ms.sessions.ticker,stream->ms.rtprecv);
Ghislain MARY's avatar
Ghislain MARY committed
168 169 170 171
	ms_filter_unlink(stream->ms.rtprecv,0,stream->ms.voidsink,0);
	ms_filter_destroy(stream->ms.voidsink);
	ms_filter_destroy(stream->ms.rtprecv);
	stream->ms.voidsink=stream->ms.rtprecv=NULL;
172 173
}

aymeric's avatar
aymeric committed
174
void video_stream_iterate(VideoStream *stream){
175
	media_stream_iterate(&stream->ms);
aymeric's avatar
aymeric committed
176 177
}

Simon Morlat's avatar
Simon Morlat committed
178
const char *video_stream_get_default_video_renderer(void){
179
#ifdef WIN32
Simon Morlat's avatar
Simon Morlat committed
180
	return "MSDrawDibDisplay";
181
#elif defined(ANDROID)
Simon Morlat's avatar
Simon Morlat committed
182
	return "MSAndroidDisplay";
183
#elif __APPLE__ && !defined(__ios)
Simon Morlat's avatar
Simon Morlat committed
184
	return "MSOSXGLDisplay";
185
#elif defined (HAVE_XV)
Simon Morlat's avatar
Simon Morlat committed
186
	return "MSX11Video";
187
#elif defined(HAVE_GL)
Simon Morlat's avatar
Simon Morlat committed
188
	return "MSGLXVideo";
Simon Morlat's avatar
Simon Morlat committed
189
#elif defined(__ios)
Simon Morlat's avatar
Simon Morlat committed
190
	return "IOSDisplay";
Simon Morlat's avatar
Simon Morlat committed
191
#else
Simon Morlat's avatar
Simon Morlat committed
192
	return "MSVideoOut";
193 194 195
#endif
}

Simon Morlat's avatar
Simon Morlat committed
196 197 198 199
static void choose_display_name(VideoStream *stream){
	stream->display_name=ms_strdup(video_stream_get_default_video_renderer());
}

200
VideoStream *video_stream_new(int loc_rtp_port, int loc_rtcp_port, bool_t use_ipv6){
201 202 203 204 205 206 207 208 209
	MSMediaStreamSessions sessions={0};
	VideoStream *obj;
	sessions.rtp_session=create_duplex_rtpsession(loc_rtp_port,loc_rtcp_port,use_ipv6);
	obj=video_stream_new_with_sessions(&sessions);
	obj->ms.owns_sessions=TRUE;
	return obj;
}

VideoStream *video_stream_new_with_sessions(const MSMediaStreamSessions *sessions){
aymeric's avatar
aymeric committed
210
	VideoStream *stream = (VideoStream *)ms_new0 (VideoStream, 1);
Ghislain MARY's avatar
Ghislain MARY committed
211
	stream->ms.type = VideoStreamType;
212 213
	stream->ms.sessions=*sessions;
	stream->ms.qi=ms_quality_indicator_new(stream->ms.sessions.rtp_session);
Ghislain MARY's avatar
Ghislain MARY committed
214 215 216
	stream->ms.evq=ortp_ev_queue_new();
	stream->ms.rtpsend=ms_filter_new(MS_RTP_SEND_ID);
	stream->ms.ice_check_list=NULL;
217
	rtp_session_register_event_queue(stream->ms.sessions.rtp_session,stream->ms.evq);
Ghislain MARY's avatar
Ghislain MARY committed
218
	MS_VIDEO_SIZE_ASSIGN(stream->sent_vsize, CIF);
219
	stream->dir=VideoStreamSendRecv;
220
	stream->display_filter_auto_rotate_enabled=0;
221 222
	stream->source_performs_encoding = FALSE;
	stream->output_performs_decoding = FALSE;
223
	choose_display_name(stream);
Simon Morlat's avatar
Simon Morlat committed
224
	stream->ms.process_rtcp=video_stream_process_rtcp;
aymeric's avatar
aymeric committed
225 226 227
	return stream;
}

smorlat's avatar
smorlat committed
228
void video_stream_set_sent_video_size(VideoStream *stream, MSVideoSize vsize){
229
	ms_message("Setting video size %dx%d", vsize.width, vsize.height);
='s avatar
= committed
230
	stream->sent_vsize=vsize;
smorlat's avatar
smorlat committed
231 232
}

Ghislain MARY's avatar
Ghislain MARY committed
233
MSVideoSize video_stream_get_sent_video_size(const VideoStream *stream) {
Ghislain MARY's avatar
Ghislain MARY committed
234 235
	MSVideoSize vsize;
	MS_VIDEO_SIZE_ASSIGN(vsize, UNKNOWN);
236 237 238 239 240 241
	if (stream->ms.encoder != NULL) {
		ms_filter_call_method(stream->ms.encoder, MS_FILTER_GET_VIDEO_SIZE, &vsize);
	}
	return vsize;
}

Ghislain MARY's avatar
Ghislain MARY committed
242
MSVideoSize video_stream_get_received_video_size(const VideoStream *stream) {
Ghislain MARY's avatar
Ghislain MARY committed
243 244
	MSVideoSize vsize;
	MS_VIDEO_SIZE_ASSIGN(vsize, UNKNOWN);
245 246 247 248 249 250
	if (stream->ms.decoder != NULL) {
		ms_filter_call_method(stream->ms.decoder, MS_FILTER_GET_VIDEO_SIZE, &vsize);
	}
	return vsize;
}

aymeric's avatar
aymeric committed
251
void video_stream_set_relay_session_id(VideoStream *stream, const char *id){
Ghislain MARY's avatar
Ghislain MARY committed
252
	ms_filter_call_method(stream->ms.rtpsend, MS_RTP_SEND_SET_RELAY_SESSION_ID,(void*)id);
aymeric's avatar
aymeric committed
253 254
}

smorlat's avatar
smorlat committed
255
void video_stream_enable_self_view(VideoStream *stream, bool_t val){
='s avatar
= committed
256
	MSFilter *out=stream->output;
257 258 259 260
	stream->corner=val ? 0 : -1;
	if (out){
		ms_filter_call_method(out,MS_VIDEO_DISPLAY_SET_LOCAL_VIEW_MODE,&stream->corner);
	}
smorlat's avatar
smorlat committed
261
}
aymeric's avatar
aymeric committed
262

263 264 265 266 267
void video_stream_set_render_callback (VideoStream *s, VideoStreamRenderCallback cb, void *user_pointer){
	s->rendercb=cb;
	s->render_pointer=user_pointer;
}

268 269 270 271 272
void video_stream_set_event_callback (VideoStream *s, VideoStreamEventCallback cb, void *user_pointer){
	s->eventcb=cb;
	s->event_pointer=user_pointer;
}

273 274 275 276 277 278 279 280 281
void video_stream_set_display_filter_name(VideoStream *s, const char *fname){
	if (s->display_name!=NULL){
		ms_free(s->display_name);
		s->display_name=NULL;
	}
	if (fname!=NULL)
		s->display_name=ms_strdup(fname);
}

282 283

static void ext_display_cb(void *ud, MSFilter* f, unsigned int event, void *eventdata){
284 285 286 287 288 289 290 291 292
	MSExtDisplayOutput *output=(MSExtDisplayOutput*)eventdata;
	VideoStream *st=(VideoStream*)ud;
	if (st->rendercb!=NULL){
		st->rendercb(st->render_pointer,
		            output->local_view.w!=0 ? &output->local_view : NULL,
		            output->remote_view.w!=0 ? &output->remote_view : NULL);
	}
}

293
void video_stream_set_direction(VideoStream *vs, VideoStreamDir dir){
='s avatar
= committed
294
	vs->dir=dir;
295 296
}

Simon Morlat's avatar
Simon Morlat committed
297 298 299 300
static MSVideoSize get_compatible_size(MSVideoSize maxsize, MSVideoSize wished_size){
	int max_area=maxsize.width*maxsize.height;
	int whished_area=wished_size.width*wished_size.height;
	if (whished_area>max_area){
301
		return maxsize;
Simon Morlat's avatar
Simon Morlat committed
302 303 304 305
	}
	return wished_size;
}

306 307 308 309 310 311 312 313 314 315
static MSVideoSize get_with_same_orientation(MSVideoSize size, MSVideoSize refsize){
	if (ms_video_size_get_orientation(refsize)!=ms_video_size_get_orientation(size)){
		int tmp;
		tmp=size.width;
		size.width=size.height;
		size.height=tmp;
	}
	return size;
}

316 317 318 319
static void configure_video_source(VideoStream *stream){
	MSVideoSize vsize,cam_vsize;
	float fps=15;
	MSPixFmt format;
320
	bool_t encoder_has_builtin_converter = FALSE;
321

322 323
	/* transmit orientation to source filter */
	ms_filter_call_method(stream->source,MS_VIDEO_CAPTURE_SET_DEVICE_ORIENTATION,&stream->device_orientation);
Simon Morlat's avatar
Simon Morlat committed
324 325 326
	/* initialize the capture device orientation for preview */
	if( ms_filter_has_method(stream->source, MS_VIDEO_DISPLAY_SET_DEVICE_ORIENTATION) )
		ms_filter_call_method(stream->source,MS_VIDEO_DISPLAY_SET_DEVICE_ORIENTATION,&stream->device_orientation);
327

Simon Morlat's avatar
Simon Morlat committed
328
	/* transmit its preview window id if any to source filter*/
329 330 331
	if (stream->preview_window_id!=0){
		video_stream_set_native_preview_window_id(stream, stream->preview_window_id);
	}
332

Ghislain MARY's avatar
Ghislain MARY committed
333 334
	ms_filter_call_method(stream->ms.encoder, MS_VIDEO_ENCODER_HAS_BUILTIN_CONVERTER, &encoder_has_builtin_converter);
	ms_filter_call_method(stream->ms.encoder,MS_FILTER_GET_VIDEO_SIZE,&vsize);
Simon Morlat's avatar
Simon Morlat committed
335
	vsize=get_compatible_size(vsize,stream->sent_vsize);
336 337 338
	ms_filter_call_method(stream->source,MS_FILTER_SET_VIDEO_SIZE,&vsize);
	/*the camera may not support the target size and suggest a one close to the target */
	ms_filter_call_method(stream->source,MS_FILTER_GET_VIDEO_SIZE,&cam_vsize);
339 340
	if (cam_vsize.width*cam_vsize.height<=vsize.width*vsize.height &&
			cam_vsize.width != vsize.width){
341
		vsize=cam_vsize;
342
		ms_message("Output video size adjusted to match camera resolution (%ix%i)\n",vsize.width,vsize.height);
343
	} else if (cam_vsize.width*cam_vsize.height>vsize.width*vsize.height){
344
#if TARGET_IPHONE_SIMULATOR || defined(__arm__)
345 346 347 348 349 350
		ms_error("Camera is proposing a size bigger than encoder's suggested size (%ix%i > %ix%i) "
		           "Using the camera size as fallback because cropping or resizing is not implemented for arm.",
		           cam_vsize.width,cam_vsize.height,vsize.width,vsize.height);
		vsize=cam_vsize;
#else
		vsize=get_with_same_orientation(vsize,cam_vsize);
351
		ms_warning("Camera video size greater than encoder one. A scaling filter will be used!\n");
352
#endif
353
	}
Ghislain MARY's avatar
Ghislain MARY committed
354 355
	ms_filter_call_method(stream->ms.encoder,MS_FILTER_SET_VIDEO_SIZE,&vsize);
	ms_filter_call_method(stream->ms.encoder,MS_FILTER_GET_FPS,&fps);
356 357 358 359 360 361 362
	ms_message("Setting sent vsize=%ix%i, fps=%f",vsize.width,vsize.height,fps);
	/* configure the filters */
	if (ms_filter_get_id(stream->source)!=MS_STATIC_IMAGE_ID) {
		ms_filter_call_method(stream->source,MS_FILTER_SET_FPS,&fps);
	}
	/* get the output format for webcam reader */
	ms_filter_call_method(stream->source,MS_FILTER_GET_PIX_FMT,&format);
363

364
	if ((encoder_has_builtin_converter == TRUE) || (stream->source_performs_encoding == TRUE)) {
Ghislain MARY's avatar
Ghislain MARY committed
365
		ms_filter_call_method(stream->ms.encoder, MS_FILTER_SET_PIX_FMT, &format);
366 367 368 369 370 371 372 373 374 375 376
	} else {
		if (format==MS_MJPEG){
			stream->pixconv=ms_filter_new(MS_MJPEG_DEC_ID);
		}else{
			stream->pixconv = ms_filter_new(MS_PIX_CONV_ID);
			/*set it to the pixconv */
			ms_filter_call_method(stream->pixconv,MS_FILTER_SET_PIX_FMT,&format);
			ms_filter_call_method(stream->pixconv,MS_FILTER_SET_VIDEO_SIZE,&cam_vsize);
		}
		stream->sizeconv=ms_filter_new(MS_SIZE_CONV_ID);
		ms_filter_call_method(stream->sizeconv,MS_FILTER_SET_VIDEO_SIZE,&vsize);
377
	}
Ghislain MARY's avatar
Ghislain MARY committed
378 379 380
	if (stream->ms.rc){
		ms_bitrate_controller_destroy(stream->ms.rc);
		stream->ms.rc=NULL;
381
	}
Ghislain MARY's avatar
Ghislain MARY committed
382
	if (stream->ms.use_rc){
383
		stream->ms.rc=ms_av_bitrate_controller_new(NULL,NULL,stream->ms.sessions.rtp_session,stream->ms.encoder);
384
	}
385 386
}

387 388
int video_stream_start (VideoStream *stream, RtpProfile *profile, const char *rem_rtp_ip, int rem_rtp_port,
	const char *rem_rtcp_ip, int rem_rtcp_port, int payload, int jitt_comp, MSWebCam *cam){
aymeric's avatar
aymeric committed
389
	PayloadType *pt;
390
	RtpSession *rtps=stream->ms.sessions.rtp_session;
aymeric's avatar
aymeric committed
391
	MSPixFmt format;
392
	MSVideoSize disp_size;
smorlat's avatar
smorlat committed
393 394
	JBParameters jbp;
	const int socket_buf_size=2000000;
395
	bool_t avpf_enabled = FALSE;
aymeric's avatar
aymeric committed
396

397 398
	if (cam==NULL){
		cam=ms_web_cam_manager_get_default_cam (
399
		      ms_web_cam_manager_get());
400 401
	}

aymeric's avatar
aymeric committed
402 403
	pt=rtp_profile_get_payload(profile,payload);
	if (pt==NULL){
404
		ms_error("videostream.c: undefined payload type %d.", payload);
aymeric's avatar
aymeric committed
405 406
		return -1;
	}
407
	if (pt->flags & PAYLOAD_TYPE_RTCP_FEEDBACK_ENABLED) avpf_enabled = TRUE;
408

409 410 411 412
	if ((cam != NULL) && (cam->desc->encode_to_mime_type != NULL) && (cam->desc->encode_to_mime_type(cam, pt->mime_type) == TRUE)) {
		stream->source_performs_encoding = TRUE;
	}

aymeric's avatar
aymeric committed
413
	rtp_session_set_profile(rtps,profile);
Simon Morlat's avatar
Simon Morlat committed
414 415
	if (rem_rtp_port>0)
		rtp_session_set_remote_addr_full(rtps,rem_rtp_ip,rem_rtp_port,rem_rtcp_ip,rem_rtcp_port);
aymeric's avatar
aymeric committed
416 417 418
	rtp_session_set_payload_type(rtps,payload);
	rtp_session_set_jitter_compensation(rtps,jitt_comp);

419
	rtp_session_signal_connect(stream->ms.sessions.rtp_session,"payload_type_changed",
420
			(RtpCallback)mediastream_payload_type_changed,(unsigned long)&stream->ms);
aymeric's avatar
aymeric committed
421

422
	rtp_session_get_jitter_buffer_params(stream->ms.sessions.rtp_session,&jbp);
smorlat's avatar
smorlat committed
423
	jbp.max_packets=1000;//needed for high resolution video
424 425 426
	rtp_session_set_jitter_buffer_params(stream->ms.sessions.rtp_session,&jbp);
	rtp_session_set_rtp_socket_recv_buffer_size(stream->ms.sessions.rtp_session,socket_buf_size);
	rtp_session_set_rtp_socket_send_buffer_size(stream->ms.sessions.rtp_session,socket_buf_size);
427

428
	if (stream->dir==VideoStreamSendRecv || stream->dir==VideoStreamSendOnly){
429
		MSConnectionHelper ch;
430 431
		/*plumb the outgoing stream */

432
		if (rem_rtp_port>0) ms_filter_call_method(stream->ms.rtpsend,MS_RTP_SEND_SET_SESSION,stream->ms.sessions.rtp_session);
433 434
		if (stream->source_performs_encoding == FALSE) {
			stream->ms.encoder=ms_filter_create_encoder(pt->mime_type);
Simon Morlat's avatar
Simon Morlat committed
435
			if (stream->ms.encoder==NULL){
436 437 438 439
				/* big problem: we don't have a registered codec for this payload...*/
				ms_error("videostream.c: No encoder available for payload %i:%s.",payload,pt->mime_type);
				return -1;
			}
440 441
		}
		/* creates the filters */
442
		stream->cam=cam;
443 444
		stream->source = ms_web_cam_create_reader(cam);
		stream->tee = ms_filter_new(MS_TEE_ID);
445 446 447
		if (stream->source_performs_encoding == TRUE) {
			stream->ms.encoder = stream->source;	/* Consider the encoder is the source */
		}
448

449
		if (pt->normal_bitrate>0){
450
			MSVideoConfiguration *vconf_list = NULL;
451
			ms_message("Limiting bitrate of video encoder to %i bits/s",pt->normal_bitrate);
452 453
			ms_filter_call_method(stream->ms.encoder, MS_VIDEO_ENCODER_GET_CONFIGURATION_LIST, &vconf_list);
			if (vconf_list != NULL) {
454
				MSVideoConfiguration vconf = ms_video_find_best_configuration_for_bitrate(vconf_list, pt->normal_bitrate);
455 456 457 458
				ms_filter_call_method(stream->ms.encoder, MS_VIDEO_ENCODER_SET_CONFIGURATION, &vconf);
			} else {
				ms_filter_call_method(stream->ms.encoder, MS_FILTER_SET_BITRATE, &pt->normal_bitrate);
			}
459
			rtp_session_set_target_upload_bandwidth(stream->ms.sessions.rtp_session, pt->normal_bitrate);
460 461
		}
		if (pt->send_fmtp){
Ghislain MARY's avatar
Ghislain MARY committed
462
			ms_filter_call_method(stream->ms.encoder,MS_FILTER_ADD_FMTP,pt->send_fmtp);
463
		}
464
		ms_filter_call_method(stream->ms.encoder, MS_VIDEO_ENCODER_ENABLE_AVPF, &avpf_enabled);
465 466 467 468 469
		if (stream->use_preview_window){
			if (stream->rendercb==NULL){
				stream->output2=ms_filter_new_from_name (stream->display_name);
			}
		}
470

471
		configure_video_source (stream);
472 473 474 475 476 477 478 479 480 481
		/* and then connect all */
		ms_connection_helper_start(&ch);
		ms_connection_helper_link(&ch, stream->source, -1, 0);
		if (stream->pixconv) {
			ms_connection_helper_link(&ch, stream->pixconv, 0, 0);
		}
		if (stream->sizeconv) {
			ms_connection_helper_link(&ch, stream->sizeconv, 0, 0);
		}
		ms_connection_helper_link(&ch, stream->tee, 0, 0);
482 483 484
		if (stream->source_performs_encoding == FALSE) {
			ms_connection_helper_link(&ch, stream->ms.encoder, 0, 0);
		}
Ghislain MARY's avatar
Ghislain MARY committed
485
		ms_connection_helper_link(&ch, stream->ms.rtpsend, 0, -1);
486
		if (stream->output2){
487
			if (stream->preview_window_id!=0){
488 489 490 491
				ms_filter_call_method(stream->output2, MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&stream->preview_window_id);
			}
			ms_filter_link(stream->tee,1,stream->output2,0);
		}
aymeric's avatar
aymeric committed
492
	}
493
	if (stream->dir==VideoStreamSendRecv || stream->dir==VideoStreamRecvOnly){
494 495
		MSConnectionHelper ch;

496 497 498 499 500 501 502 503 504
		/* create decoder first */
		stream->ms.decoder=ms_filter_create_decoder(pt->mime_type);
		if (stream->ms.decoder==NULL){
			/* big problem: we don't have a registered decoderfor this payload...*/
			ms_error("videostream.c: No decoder available for payload %i:%s.",payload,pt->mime_type);
			return -1;
		}

		/* display logic */
505
		if (stream->rendercb!=NULL){
506
			/* rendering logic delegated to user supplied callback */
507
			stream->output=ms_filter_new(MS_EXT_DISPLAY_ID);
508
			ms_filter_add_notify_callback(stream->output,ext_display_cb,stream,TRUE);
509
		}else{
510 511 512 513 514 515 516 517 518 519 520 521 522 523 524
			/* no user supplied callback -> create filter */
			MSVideoDisplayDecodingSupport decoding_support;

			/* Check if the decoding filter can perform the rendering */
			decoding_support.mime_type = pt->mime_type;
			decoding_support.supported = FALSE;
			ms_filter_call_method(stream->ms.decoder, MS_VIDEO_DECODER_SUPPORT_RENDERING, &decoding_support);
			stream->output_performs_decoding = decoding_support.supported;

			if (stream->output_performs_decoding) {
				stream->output = stream->ms.decoder;
			} else {
				/* Create default display filter */
				stream->output = ms_filter_new_from_name (stream->display_name);
			}
525
		}
526 527 528 529 530 531

		/* Don't allow null output */
		if(stream->output == NULL) {
			ms_fatal("No video display filter could be instantiated. Please check build-time configuration");
		}

532 533
		ms_filter_add_notify_callback(stream->ms.decoder, event_cb, stream, FALSE);
		ms_filter_add_notify_callback(stream->ms.decoder, internal_event_cb, stream, FALSE);
534 535

		stream->ms.rtprecv = ms_filter_new (MS_RTP_RECV_ID);
536
		ms_filter_call_method(stream->ms.rtprecv,MS_RTP_RECV_SET_SESSION,stream->ms.sessions.rtp_session);
537 538 539 540 541 542 543

		if (stream->output_performs_decoding == FALSE) {
			stream->jpegwriter=ms_filter_new(MS_JPEG_WRITER_ID);
			if (stream->jpegwriter)
				stream->tee2=ms_filter_new(MS_TEE_ID);
		}

544 545
		/* set parameters to the decoder*/
		if (pt->send_fmtp){
Ghislain MARY's avatar
Ghislain MARY committed
546
			ms_filter_call_method(stream->ms.decoder,MS_FILTER_ADD_FMTP,pt->send_fmtp);
547 548
		}
		if (pt->recv_fmtp!=NULL)
Ghislain MARY's avatar
Ghislain MARY committed
549
			ms_filter_call_method(stream->ms.decoder,MS_FILTER_ADD_FMTP,(void*)pt->recv_fmtp);
550
		ms_filter_call_method(stream->ms.decoder, MS_VIDEO_DECODER_ENABLE_AVPF, &avpf_enabled);
551

552 553
		/*force the decoder to output YUV420P */
		format=MS_YUV420P;
Ghislain MARY's avatar
Ghislain MARY committed
554
		ms_filter_call_method(stream->ms.decoder,MS_FILTER_SET_PIX_FMT,&format);
555 556

		/*configure the display window */
557
		if(stream->output != NULL) {
558
			int autofit = 1;
559 560 561 562 563 564
			disp_size.width=MS_VIDEO_SIZE_CIF_W;
			disp_size.height=MS_VIDEO_SIZE_CIF_H;
			ms_filter_call_method(stream->output,MS_FILTER_SET_VIDEO_SIZE,&disp_size);
			ms_filter_call_method(stream->output,MS_FILTER_SET_PIX_FMT,&format);
			ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SET_LOCAL_VIEW_MODE,&stream->corner);
			if (stream->window_id!=0){
565
				autofit = 0;
566 567
				ms_filter_call_method(stream->output, MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&stream->window_id);
			}
568
			ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_ENABLE_AUTOFIT,&autofit);
569 570 571
			if (stream->display_filter_auto_rotate_enabled) {
				ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SET_DEVICE_ORIENTATION,&stream->device_orientation);
			}
572
		}
573
		/* and connect the filters */
574
		ms_connection_helper_start (&ch);
Ghislain MARY's avatar
Ghislain MARY committed
575
		ms_connection_helper_link (&ch,stream->ms.rtprecv,-1,0);
576 577 578
		if (stream->output_performs_decoding == FALSE) {
			ms_connection_helper_link (&ch,stream->ms.decoder,0,0);
		}
579 580 581 582
		if (stream->tee2){
			ms_connection_helper_link (&ch,stream->tee2,0,0);
			ms_filter_link(stream->tee2,1,stream->jpegwriter,0);
		}
583 584
		if (stream->output!=NULL)
			ms_connection_helper_link (&ch,stream->output,0,-1);
585
		/* the video source must be send for preview , if it exists*/
586
		if (stream->tee!=NULL && stream->output!=NULL && stream->output2==NULL)
587
			ms_filter_link(stream->tee,1,stream->output,1);
aymeric's avatar
aymeric committed
588
	}
589
	if (stream->dir == VideoStreamSendOnly) {
Ghislain MARY's avatar
Ghislain MARY committed
590
		stream->ms.rtprecv = ms_filter_new (MS_RTP_RECV_ID);
591
		ms_filter_call_method(stream->ms.rtprecv, MS_RTP_RECV_SET_SESSION, stream->ms.sessions.rtp_session);
Ghislain MARY's avatar
Ghislain MARY committed
592 593
		stream->ms.voidsink = ms_filter_new(MS_VOID_SINK_ID);
		ms_filter_link(stream->ms.rtprecv, 0, stream->ms.voidsink, 0);
594
	}
595

aymeric's avatar
aymeric committed
596
	/* create the ticker */
597
	if (stream->ms.sessions.ticker==NULL) media_stream_start_ticker(&stream->ms);
598

599 600
	stream->ms.start_time=ms_time(NULL);
	stream->ms.is_beginning=TRUE;
601

602 603
	/* attach the graphs */
	if (stream->source)
604
		ms_ticker_attach (stream->ms.sessions.ticker, stream->source);
Ghislain MARY's avatar
Ghislain MARY committed
605
	if (stream->ms.rtprecv)
606
		ms_ticker_attach (stream->ms.sessions.ticker, stream->ms.rtprecv);
607

608
	stream->ms.state=MSStreamStarted;
aymeric's avatar
aymeric committed
609 610 611
	return 0;
}

612 613
void video_stream_prepare_video(VideoStream *stream){
	video_stream_unprepare_video(stream);
Ghislain MARY's avatar
Ghislain MARY committed
614
	stream->ms.rtprecv=ms_filter_new(MS_RTP_RECV_ID);
615 616
	rtp_session_set_payload_type(stream->ms.sessions.rtp_session,0);
	ms_filter_call_method(stream->ms.rtprecv,MS_RTP_RECV_SET_SESSION,stream->ms.sessions.rtp_session);
Ghislain MARY's avatar
Ghislain MARY committed
617 618
	stream->ms.voidsink=ms_filter_new(MS_VOID_SINK_ID);
	ms_filter_link(stream->ms.rtprecv,0,stream->ms.voidsink,0);
619 620 621
	media_stream_start_ticker(&stream->ms);
	ms_ticker_attach(stream->ms.sessions.ticker,stream->ms.rtprecv);
	stream->ms.state=MSStreamPreparing;
622 623
}

624
void video_stream_unprepare_video(VideoStream *stream){
625
	if (stream->ms.state==MSStreamPreparing) {
626
		stop_preload_graph(stream);
627
		stream->ms.state=MSStreamInitialized;
628 629 630
	}
}

631 632 633 634 635
void video_stream_update_video_params(VideoStream *stream){
	/*calling video_stream_change_camera() does the job of unplumbing/replumbing and configuring the new graph*/
	video_stream_change_camera(stream,stream->cam);
}

636
void video_stream_change_camera(VideoStream *stream, MSWebCam *cam){
Simon Morlat's avatar
Simon Morlat committed
637
	bool_t keep_source=(cam==stream->cam);
638 639
	bool_t encoder_has_builtin_converter = (!stream->pixconv && !stream->sizeconv);

640 641
	if (stream->ms.sessions.ticker && stream->source){
		ms_ticker_detach(stream->ms.sessions.ticker,stream->source);
642
		/*unlink source filters and subsequent post processin filters */
643
		if (encoder_has_builtin_converter || (stream->source_performs_encoding == TRUE)) {
644 645 646 647 648 649
			ms_filter_unlink(stream->source, 0, stream->tee, 0);
		} else {
			ms_filter_unlink (stream->source, 0, stream->pixconv, 0);
			ms_filter_unlink (stream->pixconv, 0, stream->sizeconv, 0);
			ms_filter_unlink (stream->sizeconv, 0, stream->tee, 0);
		}
650
		/*destroy the filters */
Simon Morlat's avatar
Simon Morlat committed
651
		if (!keep_source) ms_filter_destroy(stream->source);
652
		if (!encoder_has_builtin_converter && (stream->source_performs_encoding == FALSE)) {
653 654 655
			ms_filter_destroy(stream->pixconv);
			ms_filter_destroy(stream->sizeconv);
		}
656 657

		/*re create new ones and configure them*/
Simon Morlat's avatar
Simon Morlat committed
658
		if (!keep_source) stream->source = ms_web_cam_create_reader(cam);
659
		stream->cam=cam;
660 661 662 663 664 665 666 667 668 669 670

		/* update orientation */
		if (stream->source){
			ms_filter_call_method(stream->source,MS_VIDEO_CAPTURE_SET_DEVICE_ORIENTATION,&stream->device_orientation);
			if (!stream->display_filter_auto_rotate_enabled)
				ms_filter_call_method(stream->source,MS_VIDEO_DISPLAY_SET_DEVICE_ORIENTATION,&stream->device_orientation);
		}
		if (stream->output && stream->display_filter_auto_rotate_enabled) {
			ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SET_DEVICE_ORIENTATION,&stream->device_orientation);
		}

671
		configure_video_source(stream);
672

673
		if (encoder_has_builtin_converter || (stream->source_performs_encoding == TRUE)) {
674 675 676 677 678 679 680
			ms_filter_link (stream->source, 0, stream->tee, 0);
		}
		else {
			ms_filter_link (stream->source, 0, stream->pixconv, 0);
			ms_filter_link (stream->pixconv, 0, stream->sizeconv, 0);
			ms_filter_link (stream->sizeconv, 0, stream->tee, 0);
		}
681

682
		ms_ticker_attach(stream->ms.sessions.ticker,stream->source);
683 684 685
	}
}

686 687 688 689 690 691 692 693 694 695 696 697
void video_stream_send_fir(VideoStream *stream) {
	if (stream->ms.sessions.rtp_session != NULL) {
		rtp_session_send_rtcp_fb_fir(stream->ms.sessions.rtp_session);
	}
}

void video_stream_send_pli(VideoStream *stream) {
	if (stream->ms.sessions.rtp_session != NULL) {
		rtp_session_send_rtcp_fb_pli(stream->ms.sessions.rtp_session);
	}
}

Ghislain MARY's avatar
Ghislain MARY committed
698 699 700 701 702 703 704 705 706 707 708 709
void video_stream_send_sli(VideoStream *stream, uint16_t first, uint16_t number, uint8_t picture_id) {
	if (stream->ms.sessions.rtp_session != NULL) {
		rtp_session_send_rtcp_fb_sli(stream->ms.sessions.rtp_session, first, number, picture_id);
	}
}

void video_stream_send_rpsi(VideoStream *stream, uint8_t *bit_string, uint16_t bit_string_len) {
	if (stream->ms.sessions.rtp_session != NULL) {
		rtp_session_send_rtcp_fb_rpsi(stream->ms.sessions.rtp_session, bit_string, bit_string_len);
	}
}

aymeric's avatar
aymeric committed
710
void video_stream_send_vfu(VideoStream *stream){
Ghislain MARY's avatar
Ghislain MARY committed
711 712
	if (stream->ms.encoder)
		ms_filter_call_method_noarg(stream->ms.encoder, MS_VIDEO_ENCODER_REQ_VFU);
aymeric's avatar
aymeric committed
713 714 715 716 717
}

void
video_stream_stop (VideoStream * stream)
{
718 719
	stream->eventcb = NULL;
	stream->event_pointer = NULL;
720 721
	if (stream->ms.sessions.ticker){
		if (stream->ms.state == MSStreamPreparing) {
722
			stop_preload_graph(stream);
723 724
		} else {
			if (stream->source)
725
				ms_ticker_detach(stream->ms.sessions.ticker,stream->source);
Ghislain MARY's avatar
Ghislain MARY committed
726
			if (stream->ms.rtprecv)
727
				ms_ticker_detach(stream->ms.sessions.ticker,stream->ms.rtprecv);
728

Ghislain MARY's avatar
Ghislain MARY committed
729 730 731
			if (stream->ms.ice_check_list != NULL) {
				ice_check_list_print_route(stream->ms.ice_check_list, "Video session's route");
				stream->ms.ice_check_list = NULL;
732
			}
733
			rtp_stats_display(rtp_session_get_stats(stream->ms.sessions.rtp_session),
Yann Diorcet's avatar
Yann Diorcet committed
734
				"             VIDEO SESSION'S RTP STATISTICS                ");
735 736

			if (stream->source){
737 738 739 740 741 742 743 744 745 746
				MSConnectionHelper ch;
				ms_connection_helper_start(&ch);
				ms_connection_helper_unlink(&ch, stream->source, -1, 0);
				if (stream->pixconv) {
					ms_connection_helper_unlink(&ch, stream->pixconv, 0, 0);
				}
				if (stream->sizeconv) {
					ms_connection_helper_unlink(&ch, stream->sizeconv, 0, 0);
				}
				ms_connection_helper_unlink(&ch, stream->tee, 0, 0);
747 748 749
				if (stream->source_performs_encoding == FALSE) {
					ms_connection_helper_unlink(&ch, stream->ms.encoder, 0, 0);
				}
Ghislain MARY's avatar
Ghislain MARY committed
750
				ms_connection_helper_unlink(&ch, stream->ms.rtpsend, 0, -1);
751 752 753
				if (stream->output2){
					ms_filter_unlink(stream->tee,1,stream->output2,0);
				}
754
			}
Ghislain MARY's avatar
Ghislain MARY committed
755 756 757
			if (stream->ms.voidsink) {
				ms_filter_unlink(stream->ms.rtprecv, 0, stream->ms.voidsink, 0);
			} else if (stream->ms.rtprecv){
758 759
				MSConnectionHelper h;
				ms_connection_helper_start (&h);
Ghislain MARY's avatar
Ghislain MARY committed
760
				ms_connection_helper_unlink (&h,stream->ms.rtprecv,-1,0);
761 762 763
				if (stream->output_performs_decoding == FALSE) {
					ms_connection_helper_unlink (&h,stream->ms.decoder,0,0);
				}
764 765 766 767 768 769 770 771
				if (stream->tee2){
					ms_connection_helper_unlink (&h,stream->tee2,0,0);
					ms_filter_unlink(stream->tee2,1,stream->jpegwriter,0);
				}
				if(stream->output)
					ms_connection_helper_unlink (&h,stream->output,0,-1);
				if (stream->tee && stream->output && stream->output2==NULL)
					ms_filter_unlink(stream->tee,1,stream->output,1);
772
			}
773
		}
aymeric's avatar
aymeric committed
774 775 776 777 778
	}
	video_stream_free (stream);
}


779 780 781 782 783 784 785
void video_stream_show_video(VideoStream *stream, bool_t show){
	if (stream->output){
		ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SHOW_VIDEO,&show);
	}
}


786 787 788
unsigned long video_stream_get_native_window_id(VideoStream *stream){
	unsigned long id;
	if (stream->output){
unknown's avatar
unknown committed
789
		if (ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_GET_NATIVE_WINDOW_ID,&id)==0)
790 791
			return id;
	}
792 793 794 795 796
	return stream->window_id;
}

void video_stream_set_native_window_id(VideoStream *stream, unsigned long id){
	stream->window_id=id;
797 798 799
	if (stream->output){
		ms_filter_call_method(stream->output,MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&id);
	}
800 801 802 803
}

void video_stream_set_native_preview_window_id(VideoStream *stream, unsigned long id){
	stream->preview_window_id=id;
804
#ifndef __ios
805 806 807
	if (stream->output2){
		ms_filter_call_method(stream->output2,MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&id);
	}
808
#endif
809 810
	if (stream->source){
		ms_filter_call_method(stream->source,MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID,&id);
811
	}
812 813
}

Simon Morlat's avatar
wip  
Simon Morlat committed
814
unsigned long video_stream_get_native_preview_window_id(VideoStream *stream){
Simon Morlat's avatar
Simon Morlat committed
815
	unsigned long id=0;
Simon Morlat's avatar
Simon Morlat committed
816 817 818 819 820
	if (stream->output2){
		if (ms_filter_call_method(stream->output2,MS_VIDEO_DISPLAY_GET_NATIVE_WINDOW_ID,&id)==0)
			return id;
	}
	if (stream->source){
821
		if (ms_filter_has_method(stream->source,MS_VIDEO_DISPLAY_GET_NATIVE_WINDOW_ID)
Simon Morlat's avatar
Simon Morlat committed
822
		    && ms_filter_call_method(stream->source,MS_VIDEO_DISPLAY_GET_NATIVE_WINDOW_ID,&id)==0)
823 824 825 826 827 828
			return id;
	}
	return stream->preview_window_id;
}

void video_stream_use_preview_video_window(VideoStream *stream, bool_t yesno){
='s avatar
= committed
829
	stream->use_preview_window=yesno;
830
}
aymeric's avatar
aymeric committed
831

832
void video_stream_set_device_rotation(VideoStream *stream, int orientation){
833
	stream->device_orientation = orientation;
834 835
}

836 837 838 839 840 841 842 843 844 845
int video_stream_get_camera_sensor_rotation(VideoStream *stream) {
	int rotation = -1;
	if (stream->source) {
		if (ms_filter_has_method(stream->source, MS_VIDEO_CAPTURE_GET_CAMERA_SENSOR_ROTATION)
			&& ms_filter_call_method(stream->source, MS_VIDEO_CAPTURE_GET_CAMERA_SENSOR_ROTATION, &rotation) == 0)
			return rotation;
	}
	return -1;
}

Simon Morlat's avatar
Simon Morlat committed
846 847
VideoPreview * video_preview_new(void){
	VideoPreview *stream = (VideoPreview *)ms_new0 (VideoPreview, 1);
Ghislain MARY's avatar
Ghislain MARY committed
848
	MS_VIDEO_SIZE_ASSIGN(stream->sent_vsize, CIF);
Simon Morlat's avatar
Simon Morlat committed
849 850 851 852
	choose_display_name(stream);
	return stream;
}

aymeric's avatar
aymeric committed
853

Simon Morlat's avatar
Simon Morlat committed
854
void video_preview_start(VideoPreview *stream, MSWebCam *device){
aymeric's avatar
aymeric committed
855
	MSPixFmt format;
aymeric's avatar
aymeric committed
856
	float fps=(float)29.97;