mediastream.h 32.6 KB
Newer Older
aymeric's avatar
aymeric committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2006  Simon MORLAT (simon.morlat@linphone.org)

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/


#ifndef MEDIASTREAM_H
#define MEDIASTREAM_H

24 25 26
#include <ortp/ortp.h>
#include <ortp/event.h>

Simon Morlat's avatar
Simon Morlat committed
27 28 29 30 31
#include <mediastreamer2/msfilter.h>
#include <mediastreamer2/msticker.h>
#include <mediastreamer2/mssndcard.h>
#include <mediastreamer2/mswebcam.h>
#include <mediastreamer2/msvideo.h>
32 33
#include <mediastreamer2/bitratecontrol.h>
#include <mediastreamer2/qualityindicator.h>
34
#include <mediastreamer2/ice.h>
35
#include <mediastreamer2/zrtp.h>
johan's avatar
johan committed
36
#include <mediastreamer2/dtls_srtp.h>
johan's avatar
johan committed
37
#include <mediastreamer2/ms_srtp.h>
aymeric's avatar
aymeric committed
38

Ghislain MARY's avatar
Ghislain MARY committed
39

Simon Morlat's avatar
Simon Morlat committed
40 41 42
#define PAYLOAD_TYPE_FLAG_CAN_RECV	PAYLOAD_TYPE_USER_FLAG_1
#define PAYLOAD_TYPE_FLAG_CAN_SEND	PAYLOAD_TYPE_USER_FLAG_2

Ghislain MARY's avatar
Ghislain MARY committed
43 44 45 46 47

#ifdef __cplusplus
extern "C" {
#endif

48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70
/**
 * @addtogroup ring_api
 * @{
**/

struct _RingStream
{
	MSTicker *ticker;
	MSFilter *source;
	MSFilter *gendtmf;
	MSFilter *write_resampler;
	MSFilter *sndwrite;
};

typedef struct _RingStream RingStream;

MS2_PUBLIC RingStream *ring_start (const char * file, int interval, MSSndCard *sndcard);
MS2_PUBLIC RingStream *ring_start_with_cb(const char * file, int interval, MSSndCard *sndcard, MSFilterNotifyFunc func, void * user_data);
MS2_PUBLIC void ring_stop (RingStream * stream);

/**
 * @}
**/
71 72 73
/**
 * The MediaStream is an object describing a stream (one of AudioStream or VideoStream).
**/
jehan's avatar
jehan committed
74 75 76 77 78
typedef struct _MediaStream MediaStream;

/*
 * internal cb to process rtcp stream
 * */
79
typedef void (*media_stream_process_rtcp_callback_t)(MediaStream *stream, mblk_t *m);
jehan's avatar
jehan committed
80

81 82
struct _MSMediaStreamSessions{
	RtpSession *rtp_session;
jehan's avatar
jehan committed
83
	MSSrtpCtx srtp_session;
84
	MSSrtpCtx srtp_rtcp_session;
85
	MSZrtpContext *zrtp_context;
johan's avatar
johan committed
86
	MSDtlsSrtpContext *dtls_context;
87
	MSTicker *ticker;
jehan's avatar
jehan committed
88
	bool_t is_secured;
89 90 91
};

typedef struct _MSMediaStreamSessions MSMediaStreamSessions;
jehan's avatar
jehan committed
92

93 94 95 96 97
MS2_PUBLIC void ms_media_stream_sessions_uninit(MSMediaStreamSessions *sessions);

typedef enum _MSStreamState{
	MSStreamInitialized,
	MSStreamPreparing,
98 99
	MSStreamStarted,
	MSStreamStopped
100 101
}MSStreamState;

Simon Morlat's avatar
Simon Morlat committed
102 103 104
#define AudioStreamType MSAudio
#define VideoStreamType MSVideo

105 106 107
/**
 * Base struct for both AudioStream and VideoStream structure.
**/
Ghislain MARY's avatar
Ghislain MARY committed
108
struct _MediaStream {
Simon Morlat's avatar
Simon Morlat committed
109
	MSFormatType type;
110 111
	MSStreamState state;
	MSMediaStreamSessions sessions;
Ghislain MARY's avatar
Ghislain MARY committed
112 113 114 115 116 117 118
	OrtpEvQueue *evq;
	MSFilter *rtprecv;
	MSFilter *rtpsend;
	MSFilter *encoder;
	MSFilter *decoder;
	MSFilter *voidsink;
	MSBitrateController *rc;
119
	MSQualityIndicator *qi;
Ghislain MARY's avatar
Ghislain MARY committed
120
	IceCheckList *ice_check_list;
121 122
	time_t start_time;
	time_t last_iterate_time;
123 124
	uint64_t last_packet_count;
	time_t last_packet_time;
125 126
	bool_t rc_enable;
	MSQosAnalyzerAlgorithm rc_algorithm;
127
	bool_t is_beginning;
128
	bool_t owns_sessions;
129
	bool_t pad;
jehan's avatar
jehan committed
130
	/**
131
	 * defines encoder target network bit rate, uses #media_stream_set_target_network_bitrate() setter.
jehan's avatar
jehan committed
132 133
	 * */
	int target_bitrate;
134
	media_stream_process_rtcp_callback_t process_rtcp;
Ghislain MARY's avatar
Ghislain MARY committed
135 136
};

jehan's avatar
jehan committed
137 138 139 140 141

/**
 * @addtogroup audio_stream_api
 * @{
**/
Ghislain MARY's avatar
Ghislain MARY committed
142

johan's avatar
johan committed
143 144
MS2_PUBLIC bool_t media_stream_dtls_supported(void);

145 146 147 148 149 150 151 152
MS2_PUBLIC void media_stream_set_rtcp_information(MediaStream *stream, const char *cname, const char *tool);

MS2_PUBLIC void media_stream_get_local_rtp_stats(MediaStream *stream, rtp_stats_t *stats);

MS2_PUBLIC int media_stream_set_dscp(MediaStream *stream, int dscp);

MS2_PUBLIC void media_stream_enable_adaptive_bitrate_control(MediaStream *stream, bool_t enabled);

153 154
MS2_PUBLIC void media_stream_set_adaptive_bitrate_algorithm(MediaStream *stream, MSQosAnalyzerAlgorithm algorithm);

155 156
MS2_PUBLIC void media_stream_enable_adaptive_jittcomp(MediaStream *stream, bool_t enabled);

157 158 159
/*
 * deprecated, use media_stream_set_srtp_recv_key and media_stream_set_srtp_send_key.
**/
jehan's avatar
jehan committed
160 161 162
MS2_PUBLIC bool_t media_stream_enable_srtp(MediaStream* stream, MSCryptoSuite suite, const char* snd_key, const char* rcv_key);

/**
163
 * @param[in] stream MediaStream object
jehan's avatar
jehan committed
164 165
 * @return true if stream is encrypted
 * */
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180
MS2_PUBLIC bool_t media_stream_secured(const MediaStream *stream);
#define media_stream_is_secured media_stream_secured

/**
 * Tells whether AVPF is enabled or not.
 * @param[in] stream #MediaStream object.
 * @return True if AVPF is enabled, false otherwise.
 */
MS2_PUBLIC bool_t media_stream_avpf_enabled(const MediaStream *stream);

/**
 * Gets the AVPF Regular RTCP report interval.
 * @param[in] stream #MediaStream object.
 * @return The AVPF Regular RTCP report interval in seconds.
 */
181
MS2_PUBLIC uint16_t media_stream_get_avpf_rr_interval(const MediaStream *stream);
182

183 184 185 186 187 188
/**
 * Gets the RTP session of the media stream.
 * @param[in] stream #MediaStream object.
 * @return The RTP session of the media stream.
 */
MS2_PUBLIC RtpSession * media_stream_get_rtp_session(const MediaStream *stream);
189

190
MS2_PUBLIC const MSQualityIndicator *media_stream_get_quality_indicator(MediaStream *stream);
jehan's avatar
jehan committed
191 192 193
/* *
 * returns a realtime indicator of the stream quality between 0 and 5
 * */
194
MS2_PUBLIC float media_stream_get_quality_rating(MediaStream *stream);
195

196 197
MS2_PUBLIC float media_stream_get_average_quality_rating(MediaStream *stream);

198 199 200 201
MS2_PUBLIC float media_stream_get_lq_quality_rating(MediaStream *stream);

MS2_PUBLIC float media_stream_get_average_lq_quality_rating(MediaStream *stream);

202
/**
jehan's avatar
jehan committed
203 204 205 206 207 208 209 210 211 212 213 214
 * <br>For multirate codecs like OPUS, encoder output target bitrate must be set.
 * <br>Encoder will compute output codec bitrate from this value.
 * <br> default value is the value corresponding the rtp PayloadType
 * @param stream stream to apply parameter on
 * @param target_bitrate in bit per seconds
 * @return 0 if succeed
 * */
MS2_PUBLIC int media_stream_set_target_network_bitrate(MediaStream *stream,int target_bitrate);

/**
 * get the stream target bitrate.
 * @param stream stream to apply parameter on
215
 * @return target_bitrate in bit per seconds
jehan's avatar
jehan committed
216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232
 * */
MS2_PUBLIC int media_stream_get_target_network_bitrate(const MediaStream *stream);

/**
 * get current stream  upload bitrate. Value is updated every seconds
 * @param stream
 * @return bitrate in bit per seconds
 * */
MS2_PUBLIC float media_stream_get_up_bw(const MediaStream *stream);

/**
 * get current stream download bitrate. Value is updated every seconds
 * @param stream
 * @return bitrate in bit per seconds
 * */
MS2_PUBLIC float media_stream_get_down_bw(const MediaStream *stream);

233 234 235 236 237 238 239 240 241 242 243 244 245 246
/**
 * get current stream rtcp upload bitrate. Value is updated every seconds
 * @param stream
 * @return bitrate in bit per seconds
 * */
MS2_PUBLIC float media_stream_get_rtcp_up_bw(const MediaStream *stream);

/**
 * get current stream rtcp download bitrate. Value is updated every seconds
 * @param stream
 * @return bitrate in bit per seconds
 * */
MS2_PUBLIC float media_stream_get_rtcp_down_bw(const MediaStream *stream);

247 248 249 250 251 252
/**
 * Returns the sessions that were used in the media stream (RTP, SRTP, ZRTP...) so that they can be re-used.
 * As a result of calling this function, the media stream no longer owns the sessions and thus will not free them.
**/
MS2_PUBLIC void media_stream_reclaim_sessions(MediaStream *stream, MSMediaStreamSessions *sessions);

jehan's avatar
jehan committed
253

254
MS2_PUBLIC void media_stream_iterate(MediaStream * stream);
jehan's avatar
jehan committed
255

256 257 258 259 260
/**
 * Returns TRUE if stream was still actively receiving packets (RTP or RTCP) in the last period specified in timeout_seconds.
**/
MS2_PUBLIC bool_t media_stream_alive(MediaStream *stream, int timeout_seconds);

jehan's avatar
jehan committed
261
/**
jehan's avatar
jehan committed
262
 * @return current streams state
jehan's avatar
jehan committed
263 264
 * */
MS2_PUBLIC MSStreamState media_stream_get_state(const MediaStream *stream);
265

266 267 268
typedef enum EchoLimiterType{
	ELInactive,
	ELControlMic,
269
	ELControlFull
270 271
} EchoLimiterType;

272 273 274 275 276 277

typedef enum EqualizerLocation {
	MSEqualizerHP = 0,
	MSEqualizerMic
} EqualizerLocation;

jehan's avatar
jehan committed
278 279


aymeric's avatar
aymeric committed
280 281
struct _AudioStream
{
Ghislain MARY's avatar
Ghislain MARY committed
282
	MediaStream ms;
aymeric's avatar
aymeric committed
283 284 285
	MSFilter *soundread;
	MSFilter *soundwrite;
	MSFilter *dtmfgen;
286
	MSFilter *dtmfgen_rtp;
Yann Diorcet's avatar
Yann Diorcet committed
287
	MSFilter *plc;
aymeric's avatar
aymeric committed
288
	MSFilter *ec;/*echo canceler*/
289
	MSFilter *volsend,*volrecv; /*MSVolumes*/
290 291 292
	MSFilter *local_mixer;
	MSFilter *local_player;
	MSFilter *local_player_resampler;
293 294
	MSFilter *read_resampler;
	MSFilter *write_resampler;
smorlat's avatar
smorlat committed
295
	MSFilter *equalizer;
Simon Morlat's avatar
Simon Morlat committed
296
	MSFilter *dummy;
297 298 299
	MSFilter *recv_tee;
	MSFilter *recorder_mixer;
	MSFilter *recorder;
300
	MSFilter *outbound_mixer;
Simon Morlat's avatar
Simon Morlat committed
301 302 303 304 305 306
	struct {
		MSFilter *resampler;
		MSFilter *encoder;
		MSFilter *recorder;
		MSFilter *video_input;
	}av_recorder;
307 308 309 310 311 312 313 314 315
	struct _AVPlayer{
		MSFilter *player;
		MSFilter *resampler;
		MSFilter *decoder;
		MSFilter *video_output;
		int audiopin;
		int videopin;
		bool_t plumbed;
	}av_player;
316
	char *recorder_file;
317
	EchoLimiterType el_type; /*use echo limiter: two MSVolume, measured input level controlling local output level*/
318
	EqualizerLocation eq_loc;
Ghislain MARY's avatar
Ghislain MARY committed
319
	uint32_t features;
320
	struct _VideoStream *videostream;/*the stream with which this audiostream is paired*/
smorlat's avatar
smorlat committed
321
	bool_t play_dtmfs;
smorlat's avatar
smorlat committed
322
	bool_t use_gc;
323
	bool_t use_agc;
smorlat's avatar
smorlat committed
324
	bool_t eq_active;
325
	bool_t use_ng;/*noise gate*/
326
	bool_t is_ec_delay_set;
aymeric's avatar
aymeric committed
327 328
};

329 330 331
/**
 * The AudioStream holds all resources to create and run typical VoIP audiostream.
**/
aymeric's avatar
aymeric committed
332 333 334 335
typedef struct _AudioStream AudioStream;


/* start a thread that does sampling->encoding->rtp_sending|rtp_receiving->decoding->playing */
Simon Morlat's avatar
Simon Morlat committed
336
MS2_PUBLIC AudioStream *audio_stream_start (RtpProfile * prof, int locport, const char *remip,
aymeric's avatar
aymeric committed
337 338
				 int remport, int payload_type, int jitt_comp, bool_t echo_cancel);

Simon Morlat's avatar
Simon Morlat committed
339
MS2_PUBLIC AudioStream *audio_stream_start_with_sndcards(RtpProfile * prof, int locport, const char *remip4, int remport, int payload_type, int jitt_comp, MSSndCard *playcard, MSSndCard *captcard, bool_t echocancel);
aymeric's avatar
aymeric committed
340

341

Simon Morlat's avatar
Simon Morlat committed
342
MS2_PUBLIC int audio_stream_start_with_files (AudioStream * stream, RtpProfile * prof,
343 344 345
						const char *remip, int remport, int rem_rtcp_port,
						int pt, int jitt_comp,
						const char * infile,  const char * outfile);
aymeric's avatar
aymeric committed
346

347 348
/**
 * Starts an audio stream from/to local wav files or soundcards.
349
 *
350 351
 * This method starts the processing of the audio stream, that is playing from wav file or soundcard, voice processing, encoding,
 * sending through RTP, receiving from RTP, decoding, voice processing and wav file recording or soundcard playback.
352 353
 *
 *
354
 * @param stream an AudioStream previously created with audio_stream_new().
355
 * @param profile a RtpProfile containing all PayloadType possible during the audio session.
356 357 358
 * @param rem_rtp_ip remote IP address where to send the encoded audio.
 * @param rem_rtp_port remote IP port where to send the encoded audio.
 * @param rem_rtcp_ip remote IP address for RTCP.
359
 * @param rem_rtcp_port remote port for RTCP.
360
 * @param payload payload type index to use for the sending stream. This index must point to a valid PayloadType in the RtpProfile.
361 362 363 364 365
 * @param jitt_comp Nominal jitter buffer size in milliseconds.
 * @param infile path to wav file to play out (can be NULL)
 * @param outfile path to wav file to record into (can be NULL)
 * @param playcard The soundcard to be used for playback (can be NULL)
 * @param captcard The soundcard to be used for catpure. (can be NULL)
366
 * @param use_ec whether echo cancellation is to be performed.
367
 * @return 0 if sucessful, -1 otherwise.
368
**/
369 370
MS2_PUBLIC int audio_stream_start_full(AudioStream *stream, RtpProfile *profile, const char *rem_rtp_ip,int rem_rtp_port,
	const char *rem_rtcp_ip, int rem_rtcp_port, int payload,int jitt_comp, const char *infile, const char *outfile,
371 372
	MSSndCard *playcard, MSSndCard *captcard, bool_t use_ec);

373

Simon Morlat's avatar
Simon Morlat committed
374 375
MS2_PUBLIC void audio_stream_play(AudioStream *st, const char *name);
MS2_PUBLIC void audio_stream_record(AudioStream *st, const char *name);
aymeric's avatar
aymeric committed
376

377
static MS2_INLINE void audio_stream_set_rtcp_information(AudioStream *st, const char *cname, const char *tool) {
Ghislain MARY's avatar
Ghislain MARY committed
378 379
	media_stream_set_rtcp_information(&st->ms, cname, tool);
}
aymeric's avatar
aymeric committed
380

Simon Morlat's avatar
Simon Morlat committed
381
MS2_PUBLIC void audio_stream_play_received_dtmfs(AudioStream *st, bool_t yesno);
smorlat's avatar
smorlat committed
382

383 384
/**
 * Creates an AudioStream object listening on a RTP port.
385 386
 * @param loc_rtp_port the local UDP port to listen for RTP packets.
 * @param loc_rtcp_port the local UDP port to listen for RTCP packets
387
 * @param ipv6 TRUE if ipv6 must be used.
388
 * @return a new AudioStream.
389
**/
390
MS2_PUBLIC AudioStream *audio_stream_new(int loc_rtp_port, int loc_rtcp_port, bool_t ipv6);
Simon Morlat's avatar
Simon Morlat committed
391

jehan's avatar
jehan committed
392 393 394 395 396 397 398 399 400 401
/**
 * Creates an AudioStream object listening on a RTP port for a dedicated address.
 * @param loc_ip the local ip to listen for RTP packets. Can be ::, O.O.O.O or any ip4/6 addresses
 * @param loc_rtp_port the local UDP port to listen for RTP packets.
 * @param loc_rtcp_port the local UDP port to listen for RTCP packets
 * @return a new AudioStream.
**/
MS2_PUBLIC AudioStream *audio_stream_new2(const char* ip, int loc_rtp_port, int loc_rtcp_port);


402 403
/**Creates an AudioStream object from initialized MSMediaStreamSessions.
 * @param sessions the MSMediaStreamSessions
404
 * @return a new AudioStream
405 406 407
**/
MS2_PUBLIC AudioStream *audio_stream_new_with_sessions(const MSMediaStreamSessions *sessions);

408 409 410 411 412 413 414 415
#define AUDIO_STREAM_FEATURE_PLC 		(1 << 0)
#define AUDIO_STREAM_FEATURE_EC 		(1 << 1)
#define AUDIO_STREAM_FEATURE_EQUALIZER		(1 << 2)
#define AUDIO_STREAM_FEATURE_VOL_SND 		(1 << 3)
#define AUDIO_STREAM_FEATURE_VOL_RCV 		(1 << 4)
#define AUDIO_STREAM_FEATURE_DTMF		(1 << 5)
#define AUDIO_STREAM_FEATURE_DTMF_ECHO		(1 << 6)
#define AUDIO_STREAM_FEATURE_MIXED_RECORDING	(1 << 7)
416
#define AUDIO_STREAM_FEATURE_LOCAL_PLAYING	(1 << 8)
417
#define AUDIO_STREAM_FEATURE_REMOTE_PLAYING	(1 << 9)
418

Yann Diorcet's avatar
Yann Diorcet committed
419 420 421 422 423 424 425
#define AUDIO_STREAM_FEATURE_ALL	(\
					AUDIO_STREAM_FEATURE_PLC | \
					AUDIO_STREAM_FEATURE_EC | \
					AUDIO_STREAM_FEATURE_EQUALIZER | \
					AUDIO_STREAM_FEATURE_VOL_SND | \
					AUDIO_STREAM_FEATURE_VOL_RCV | \
					AUDIO_STREAM_FEATURE_DTMF | \
426
					AUDIO_STREAM_FEATURE_DTMF_ECHO |\
427
					AUDIO_STREAM_FEATURE_MIXED_RECORDING |\
428 429
					AUDIO_STREAM_FEATURE_LOCAL_PLAYING | \
					AUDIO_STREAM_FEATURE_REMOTE_PLAYING \
Yann Diorcet's avatar
Yann Diorcet committed
430 431
					)

432

Yann Diorcet's avatar
Yann Diorcet committed
433 434
MS2_PUBLIC uint32_t audio_stream_get_features(AudioStream *st);
MS2_PUBLIC void audio_stream_set_features(AudioStream *st, uint32_t features);
435

Simon Morlat's avatar
Simon Morlat committed
436 437 438
MS2_PUBLIC void audio_stream_prepare_sound(AudioStream *st, MSSndCard *playcard, MSSndCard *captcard);
MS2_PUBLIC void audio_stream_unprepare_sound(AudioStream *st);
MS2_PUBLIC bool_t audio_stream_started(AudioStream *stream);
439 440
/**
 * Starts an audio stream from local soundcards.
441
 *
442 443
 * This method starts the processing of the audio stream, that is capture from soundcard, voice processing, encoding,
 * sending through RTP, receiving from RTP, decoding, voice processing and soundcard playback.
444
 *
445 446 447 448 449 450 451 452 453 454 455
 * @param stream an AudioStream previously created with audio_stream_new().
 * @param prof a RtpProfile containing all PayloadType possible during the audio session.
 * @param remip remote IP address where to send the encoded audio.
 * @param remport remote IP port where to send the encoded audio
 * @param rem_rtcp_port remote port for RTCP.
 * @param payload_type payload type index to use for the sending stream. This index must point to a valid PayloadType in the RtpProfile.
 * @param jitt_comp Nominal jitter buffer size in milliseconds.
 * @param playcard The soundcard to be used for playback
 * @param captcard The soundcard to be used for catpure.
 * @param echo_cancel whether echo cancellation is to be performed.
**/
Simon Morlat's avatar
Simon Morlat committed
456 457
MS2_PUBLIC int audio_stream_start_now(AudioStream * stream, RtpProfile * prof,  const char *remip, int remport, int rem_rtcp_port, int payload_type, int jitt_comp,MSSndCard *playcard, MSSndCard *captcard, bool_t echo_cancel);
MS2_PUBLIC void audio_stream_set_relay_session_id(AudioStream *stream, const char *relay_session_id);
aymeric's avatar
aymeric committed
458
/*returns true if we are still receiving some data from remote end in the last timeout seconds*/
Simon Morlat's avatar
Simon Morlat committed
459
MS2_PUBLIC bool_t audio_stream_alive(AudioStream * stream, int timeout);
aymeric's avatar
aymeric committed
460

461 462 463 464
/**
 * Executes background low priority tasks related to audio processing (RTP statistics analysis).
 * It should be called periodically, for example with an interval of 100 ms or so.
 */
465 466
MS2_PUBLIC void audio_stream_iterate(AudioStream *stream);

jehan's avatar
jehan committed
467 468 469
/**
 * enable echo-limiter dispositve: one MSVolume in input branch controls a MSVolume in the output branch
 * */
Simon Morlat's avatar
Simon Morlat committed
470
MS2_PUBLIC void audio_stream_enable_echo_limiter(AudioStream *stream, EchoLimiterType type);
471

jehan's avatar
jehan committed
472 473 474
/**
 * enable gain control, to be done before start()
 * */
Simon Morlat's avatar
Simon Morlat committed
475
MS2_PUBLIC void audio_stream_enable_gain_control(AudioStream *stream, bool_t val);
smorlat's avatar
smorlat committed
476

jehan's avatar
jehan committed
477 478 479
/**
 * enable automatic gain control, to be done before start()
 * */
Simon Morlat's avatar
Simon Morlat committed
480
MS2_PUBLIC void audio_stream_enable_automatic_gain_control(AudioStream *stream, bool_t val);
481

jehan's avatar
jehan committed
482 483 484
/**
 * to be done before start
 *  */
Simon Morlat's avatar
Simon Morlat committed
485
MS2_PUBLIC void audio_stream_set_echo_canceller_params(AudioStream *st, int tail_len_ms, int delay_ms, int framesize);
486

jehan's avatar
jehan committed
487 488 489
/**
 * enable adaptive rate control
 * */
490
static MS2_INLINE void audio_stream_enable_adaptive_bitrate_control(AudioStream *stream, bool_t enabled) {
Ghislain MARY's avatar
Ghislain MARY committed
491 492
	media_stream_enable_adaptive_bitrate_control(&stream->ms, enabled);
}
493

jehan's avatar
jehan committed
494 495 496
/**
 *  Enable adaptive jitter compensation
 *  */
497
static MS2_INLINE void audio_stream_enable_adaptive_jittcomp(AudioStream *stream, bool_t enabled) {
Ghislain MARY's avatar
Ghislain MARY committed
498 499
	media_stream_enable_adaptive_jittcomp(&stream->ms, enabled);
}
500

Simon Morlat's avatar
Simon Morlat committed
501
MS2_PUBLIC void audio_stream_set_mic_gain(AudioStream *stream, float gain);
smorlat's avatar
smorlat committed
502

jehan's avatar
jehan committed
503 504 505
/**
 *  enable/disable rtp stream
 *  */
Simon Morlat's avatar
Simon Morlat committed
506
MS2_PUBLIC void audio_stream_mute_rtp(AudioStream *stream, bool_t val);
Simon Morlat's avatar
Simon Morlat committed
507

jehan's avatar
jehan committed
508 509 510
/**
 * enable noise gate, must be done before start()
 * */
Simon Morlat's avatar
Simon Morlat committed
511
MS2_PUBLIC void audio_stream_enable_noise_gate(AudioStream *stream, bool_t val);
512

jehan's avatar
jehan committed
513 514 515
/**
 * enable parametric equalizer in the stream that goes to the speaker
 * */
Simon Morlat's avatar
Simon Morlat committed
516
MS2_PUBLIC void audio_stream_enable_equalizer(AudioStream *stream, bool_t enabled);
smorlat's avatar
smorlat committed
517

Simon Morlat's avatar
Simon Morlat committed
518
MS2_PUBLIC void audio_stream_equalizer_set_gain(AudioStream *stream, int frequency, float gain, int freq_width);
smorlat's avatar
smorlat committed
519

jehan's avatar
jehan committed
520 521 522
/**
 *  stop the audio streaming thread and free everything
 *  */
Simon Morlat's avatar
Simon Morlat committed
523
MS2_PUBLIC void audio_stream_stop (AudioStream * stream);
aymeric's avatar
aymeric committed
524

jehan's avatar
jehan committed
525 526 527
/**
 *  send a dtmf
 *  */
Simon Morlat's avatar
Simon Morlat committed
528
MS2_PUBLIC int audio_stream_send_dtmf (AudioStream * stream, char dtmf);
aymeric's avatar
aymeric committed
529

530 531
MS2_PUBLIC MSFilter *audio_stream_get_local_player(AudioStream *stream);

532 533 534 535 536 537
MS2_PUBLIC int audio_stream_mixed_record_open(AudioStream *st, const char*filename);

MS2_PUBLIC int audio_stream_mixed_record_start(AudioStream *st);

MS2_PUBLIC int audio_stream_mixed_record_stop(AudioStream *st);

538 539 540 541 542 543
/**
 * Open a player to play an audio/video file to remote end.
 * The player is returned as a MSFilter so that application can make usual player controls on it using the MSPlayerInterface.
**/
MS2_PUBLIC MSFilter * audio_stream_open_remote_play(AudioStream *stream, const char *filename);

544 545
MS2_PUBLIC void audio_stream_close_remote_play(AudioStream *stream);

Simon Morlat's avatar
Simon Morlat committed
546
MS2_PUBLIC void audio_stream_set_default_card(int cardindex);
aymeric's avatar
aymeric committed
547

Simon Morlat's avatar
Simon Morlat committed
548
/* retrieve RTP statistics*/
549
static MS2_INLINE void audio_stream_get_local_rtp_stats(AudioStream *stream, rtp_stats_t *stats) {
Ghislain MARY's avatar
Ghislain MARY committed
550 551
	media_stream_get_local_rtp_stats(&stream->ms, stats);
}
Simon Morlat's avatar
Simon Morlat committed
552

553
/* returns a realtime indicator of the stream quality between 0 and 5 */
554 555
MS2_PUBLIC float audio_stream_get_quality_rating(AudioStream *stream);

556 557
/* returns the quality rating as an average since the start of the streaming session.*/
MS2_PUBLIC float audio_stream_get_average_quality_rating(AudioStream *stream);
aymeric's avatar
aymeric committed
558

559 560 561 562 563 564
/* returns a realtime indicator of the listening quality of the stream between 0 and 5 */
MS2_PUBLIC float audio_stream_get_lq_quality_rating(AudioStream *stream);

/* returns the listening quality rating as an average since the start of the streaming session.*/
MS2_PUBLIC float audio_stream_get_average_lq_quality_rating(AudioStream *stream);

Guillaume Beraudo's avatar
Guillaume Beraudo committed
565
/* enable ZRTP on the audio stream */
566
MS2_PUBLIC void audio_stream_enable_zrtp(AudioStream *stream, MSZrtpParams *params);
jehan's avatar
jehan committed
567 568 569 570
/**
 * return TRUE if zrtp is enabled, it does not mean that stream is encrypted, but only that zrtp is configured to know encryption status, uses #
 * */
bool_t  audio_stream_zrtp_enabled(const AudioStream *stream);
Guillaume Beraudo's avatar
Guillaume Beraudo committed
571

johan's avatar
johan committed
572 573 574
/* enable DTLS on the audio stream */
MS2_PUBLIC void audio_stream_enable_dtls(AudioStream *stream, MSDtlsSrtpParams *params);

575
/* enable SRTP on the audio stream */
576
static MS2_INLINE bool_t audio_stream_enable_srtp(AudioStream* stream, MSCryptoSuite suite, const char* snd_key, const char* rcv_key) {
Ghislain MARY's avatar
Ghislain MARY committed
577 578
	return media_stream_enable_srtp(&stream->ms, suite, snd_key, rcv_key);
}
Guillaume Beraudo's avatar
Guillaume Beraudo committed
579

580
static MS2_INLINE int audio_stream_set_dscp(AudioStream *stream, int dscp) {
Ghislain MARY's avatar
Ghislain MARY committed
581 582
	return media_stream_set_dscp(&stream->ms, dscp);
}
Simon Morlat's avatar
Simon Morlat committed
583

584 585 586 587 588 589 590 591 592 593
/**
 * Gets the RTP session of an audio stream.
 * @param[in] stream #MediaStream object.
 * @return The RTP session of the audio stream.
 */
static MS2_INLINE RtpSession * audio_stream_get_rtp_session(const AudioStream *stream) {
	return media_stream_get_rtp_session(&stream->ms);
}


Simon Morlat's avatar
Simon Morlat committed
594

595 596 597 598 599 600 601 602 603
/**
 * @}
**/


/**
 * @addtogroup video_stream_api
 * @{
**/
aymeric's avatar
aymeric committed
604

605
typedef void (*VideoStreamRenderCallback)(void *user_pointer, const MSPicture *local_view, const MSPicture *remote_view);
606 607 608
typedef void (*VideoStreamEventCallback)(void *user_pointer, const MSFilter *f, const unsigned int event_id, const void *args);


609 610 611 612 613 614
typedef enum _VideoStreamDir{
	VideoStreamSendRecv,
	VideoStreamSendOnly,
	VideoStreamRecvOnly
}VideoStreamDir;

aymeric's avatar
aymeric committed
615 616
struct _VideoStream
{
Ghislain MARY's avatar
Ghislain MARY committed
617
	MediaStream ms;
aymeric's avatar
aymeric committed
618 619
	MSFilter *source;
	MSFilter *pixconv;
620
	MSFilter *sizeconv;
aymeric's avatar
aymeric committed
621 622
	MSFilter *tee;
	MSFilter *output;
623 624
	MSFilter *tee2;
	MSFilter *jpegwriter;
625
	MSFilter *output2;
Simon Morlat's avatar
Simon Morlat committed
626 627
	MSFilter *tee3;
	MSFilter *itcsink;
628
	MSFilter *local_jpegwriter;
smorlat's avatar
smorlat committed
629
	MSVideoSize sent_vsize;
630
	MSVideoSize preview_vsize;
631 632
	float fps; /*the target fps explicitely set by application, overrides internally selected fps*/
	float configured_fps; /*the fps that was configured to the encoder. It might be different from the one really obtained from camera.*/
smorlat's avatar
smorlat committed
633
	int corner; /*for selfview*/
634 635
	VideoStreamRenderCallback rendercb;
	void *render_pointer;
636 637
	VideoStreamEventCallback eventcb;
	void *event_pointer;
638
	char *display_name;
639 640
	unsigned long window_id;
	unsigned long preview_window_id;
641
	VideoStreamDir dir;
642
	MSWebCam *cam;
643
	int device_orientation; /* warning: meaning of this variable depends on the platform (Android, iOS, ...) */
Simon Morlat's avatar
Simon Morlat committed
644
	uint64_t last_reported_decoding_error_time;
645
	uint64_t last_fps_check;
Ghislain MARY's avatar
Ghislain MARY committed
646
	bool_t use_preview_window;
647
	bool_t freeze_on_error;
Ghislain MARY's avatar
Ghislain MARY committed
648
	bool_t display_filter_auto_rotate_enabled;
649 650
	bool_t source_performs_encoding;
	bool_t output_performs_decoding;
651
	bool_t player_active;
652

aymeric's avatar
aymeric committed
653 654 655
};

typedef struct _VideoStream VideoStream;
smorlat's avatar
smorlat committed
656

657

658
MS2_PUBLIC VideoStream *video_stream_new(int loc_rtp_port, int loc_rtcp_port, bool_t use_ipv6);
jehan's avatar
jehan committed
659 660 661 662 663 664 665 666 667
/**
 * Creates an VideoStream object listening on a RTP port for a dedicated address.
 * @param loc_ip the local ip to listen for RTP packets. Can be ::, O.O.O.O or any ip4/6 addresses
 * @param [in] loc_rtp_port the local UDP port to listen for RTP packets.
 * @param [in] loc_rtcp_port the local UDP port to listen for RTCP packets
 * @return a new AudioStream.
**/
MS2_PUBLIC VideoStream *video_stream_new2(const char* ip, int loc_rtp_port, int loc_rtcp_port);

668
MS2_PUBLIC VideoStream *video_stream_new_with_sessions(const MSMediaStreamSessions *sessions);
Simon Morlat's avatar
Simon Morlat committed
669
MS2_PUBLIC void video_stream_set_direction(VideoStream *vs, VideoStreamDir dir);
670
static MS2_INLINE void video_stream_enable_adaptive_bitrate_control(VideoStream *stream, bool_t enabled) {
Ghislain MARY's avatar
Ghislain MARY committed
671 672
	media_stream_enable_adaptive_bitrate_control(&stream->ms, enabled);
}
673
static MS2_INLINE void video_stream_enable_adaptive_jittcomp(VideoStream *stream, bool_t enabled) {
Ghislain MARY's avatar
Ghislain MARY committed
674 675
	media_stream_enable_adaptive_jittcomp(&stream->ms, enabled);
}
Simon Morlat's avatar
Simon Morlat committed
676 677 678
MS2_PUBLIC void video_stream_set_render_callback(VideoStream *s, VideoStreamRenderCallback cb, void *user_pointer);
MS2_PUBLIC void video_stream_set_event_callback(VideoStream *s, VideoStreamEventCallback cb, void *user_pointer);
MS2_PUBLIC void video_stream_set_display_filter_name(VideoStream *s, const char *fname);
679 680
MS2_PUBLIC int video_stream_start_with_source(VideoStream *stream, RtpProfile *profile, const char *rem_rtp_ip, int rem_rtp_port,
		const char *rem_rtcp_ip, int rem_rtcp_port, int payload, int jitt_comp, MSWebCam* cam, MSFilter* source);
681
MS2_PUBLIC int video_stream_start(VideoStream * stream, RtpProfile *profile, const char *rem_rtp_ip, int rem_rtp_port, const char *rem_rtcp_ip, int rem_rtcp_port,
aymeric's avatar
aymeric committed
682
		int payload, int jitt_comp, MSWebCam *device);
683 684
MS2_PUBLIC void video_stream_prepare_video(VideoStream *stream);
MS2_PUBLIC void video_stream_unprepare_video(VideoStream *stream);
685 686


Simon Morlat's avatar
Simon Morlat committed
687
MS2_PUBLIC void video_stream_set_relay_session_id(VideoStream *stream, const char *relay_session_id);
688
static MS2_INLINE void video_stream_set_rtcp_information(VideoStream *st, const char *cname, const char *tool) {
Ghislain MARY's avatar
Ghislain MARY committed
689 690
	media_stream_set_rtcp_information(&st->ms, cname, tool);
}
Simon Morlat's avatar
Simon Morlat committed
691
MS2_PUBLIC void video_stream_change_camera(VideoStream *stream, MSWebCam *cam);
692 693
/* Calling video_stream_set_sent_video_size() or changing the bitrate value in the used PayloadType during a stream is running does nothing.
The following function allows to take into account new parameters by redrawing the sending graph*/
Simon Morlat's avatar
Simon Morlat committed
694
MS2_PUBLIC void video_stream_update_video_params(VideoStream *stream);
aymeric's avatar
aymeric committed
695
/*function to call periodically to handle various events */
Simon Morlat's avatar
Simon Morlat committed
696
MS2_PUBLIC void video_stream_iterate(VideoStream *stream);
697 698 699 700 701 702 703 704 705 706 707

/**
 * Ask the video stream to send a Full-Intra Request.
 * @param[in] stream The videostream object.
 */
MS2_PUBLIC void video_stream_send_fir(VideoStream *stream);

/**
 * Ask the video stream to generate a Video Fast Update (generally after receiving a Full-Intra Request.
 * @param[in] stream The videostream object.
 */
Simon Morlat's avatar
Simon Morlat committed
708
MS2_PUBLIC void video_stream_send_vfu(VideoStream *stream);
709

Simon Morlat's avatar
Simon Morlat committed
710 711
MS2_PUBLIC void video_stream_stop(VideoStream * stream);
MS2_PUBLIC void video_stream_set_sent_video_size(VideoStream *stream, MSVideoSize vsize);
712 713 714 715 716 717

/**
 * Gets the size of the video that is sent.
 * @param[in] stream The videostream for which to get the sent video size.
 * @return The sent video size or MS_VIDEO_SIZE_UNKNOWN if not available.
 */
Ghislain MARY's avatar
Ghislain MARY committed
718
MS2_PUBLIC MSVideoSize video_stream_get_sent_video_size(const VideoStream *stream);
719 720 721 722 723 724

/**
 * Gets the size of the video that is received.
 * @param[in] stream The videostream for which to get the received video size.
 * @return The received video size or MS_VIDEO_SIZE_UNKNOWN if not available.
 */
Ghislain MARY's avatar
Ghislain MARY committed
725
MS2_PUBLIC MSVideoSize video_stream_get_received_video_size(const VideoStream *stream);
726

727 728 729 730 731 732 733 734 735 736 737 738 739 740
/**
 * Gets the framerate of the video that is sent.
 * @param[in] stream The videostream.
 * @return The actual framerate, 0 if not available..
 */
MS2_PUBLIC float video_stream_get_sent_framerate(const VideoStream *stream);

/**
 * Gets the framerate of the video that is received.
 * @param[in] stream The videostream.
 * @return The received framerate or 0 if not available.
 */
MS2_PUBLIC float video_stream_get_received_framerate(const VideoStream *stream);

Simon Morlat's avatar
Simon Morlat committed
741 742 743 744 745
/**
 * Returns the name of the video display filter on the current platform.
**/
const char *video_stream_get_default_video_renderer(void);

Simon Morlat's avatar
Simon Morlat committed
746 747 748 749 750 751
MS2_PUBLIC void video_stream_enable_self_view(VideoStream *stream, bool_t val);
MS2_PUBLIC unsigned long video_stream_get_native_window_id(VideoStream *stream);
MS2_PUBLIC void video_stream_set_native_window_id(VideoStream *stream, unsigned long id);
MS2_PUBLIC void video_stream_set_native_preview_window_id(VideoStream *stream, unsigned long id);
MS2_PUBLIC unsigned long video_stream_get_native_preview_window_id(VideoStream *stream);
MS2_PUBLIC void video_stream_use_preview_video_window(VideoStream *stream, bool_t yesno);
752
MS2_PUBLIC void video_stream_set_device_rotation(VideoStream *stream, int orientation);
753
MS2_PUBLIC void video_stream_show_video(VideoStream *stream, bool_t show);
754
MS2_PUBLIC void video_stream_set_freeze_on_error(VideoStream *stream, bool_t yesno);
755

756 757 758 759 760 761 762 763 764 765 766
/**
 * @brief Gets the camera sensor rotation.
 *
 * This is needed on some mobile platforms to get the number of degrees the camera sensor
 * is rotated relative to the screen.
 *
 * @param stream The video stream related to the operation
 * @return The camera sensor rotation in degrees (0 to 360) or -1 if it could not be retrieved
 */
MS2_PUBLIC int video_stream_get_camera_sensor_rotation(VideoStream *stream);

767
/*provided for compatibility, use video_stream_set_direction() instead */
Simon Morlat's avatar
Simon Morlat committed
768 769
MS2_PUBLIC int video_stream_recv_only_start(VideoStream *videostream, RtpProfile *profile, const char *addr, int port, int used_pt, int jitt_comp);
MS2_PUBLIC int video_stream_send_only_start(VideoStream *videostream,
770
				RtpProfile *profile, const char *addr, int port, int rtcp_port,
771
				int used_pt, int  jitt_comp, MSWebCam *device);
Simon Morlat's avatar
Simon Morlat committed
772 773
MS2_PUBLIC void video_stream_recv_only_stop(VideoStream *vs);
MS2_PUBLIC void video_stream_send_only_stop(VideoStream *vs);
aymeric's avatar
aymeric committed
774

Guillaume Beraudo's avatar
Guillaume Beraudo committed
775
/* enable ZRTP on the video stream using information from the audio stream */
776
MS2_PUBLIC void video_stream_enable_zrtp(VideoStream *vstream, AudioStream *astream, MSZrtpParams *param);
777

778 779 780
/* enable DTLS on the video stream */
MS2_PUBLIC void video_stream_enable_dtls(VideoStream *stream, MSDtlsSrtpParams *params);

781
/* enable SRTP on the video stream */
782
static MS2_INLINE bool_t video_stream_enable_strp(VideoStream* stream, MSCryptoSuite suite, const char* snd_key, const char* rcv_key) {
Ghislain MARY's avatar
Ghislain MARY committed
783 784 785
	return media_stream_enable_srtp(&stream->ms, suite, snd_key, rcv_key);
}

786 787
/* if enabled, the display filter will internaly rotate the video, according to the device orientation */
MS2_PUBLIC void video_stream_enable_display_filter_auto_rotate(VideoStream* stream, bool_t enable);
788

789
/* retrieve RTP statistics*/
790
static MS2_INLINE void video_stream_get_local_rtp_stats(VideoStream *stream, rtp_stats_t *stats) {
Ghislain MARY's avatar
Ghislain MARY committed
791 792
	media_stream_get_local_rtp_stats(&stream->ms, stats);
}
793

794
static MS2_INLINE int video_stream_set_dscp(VideoStream *stream, int dscp) {
Ghislain MARY's avatar
Ghislain MARY committed
795 796
	return media_stream_set_dscp(&stream->ms, dscp);
}
Simon Morlat's avatar
Simon Morlat committed
797

798 799 800 801 802 803 804 805 806
/**
 * Gets the RTP session of a video stream.
 * @param[in] stream #MediaStream object.
 * @return The RTP session of the video stream.
 */
static MS2_INLINE RtpSession * video_stream_get_rtp_session(const VideoStream *stream) {
	return media_stream_get_rtp_session(&stream->ms);
}

807 808 809 810 811 812 813 814 815 816 817 818 819 820
/**
 * Ask the video stream whether a decoding error should be reported (eg. to send a VFU request).
 * @param[in] stream The VideoStream object.
 * @param[in] ms The minimum interval in milliseconds between to decoding error report.
 * @return TRUE if the decoding error should be reported, FALSE otherwise.
 */
MS2_PUBLIC bool_t video_stream_is_decoding_error_to_be_reported(VideoStream *stream, uint32_t ms);

/**
 * Tell the video stream that a decoding error has been reported.
 * @param[in] stream The VideoStream object.
 */
MS2_PUBLIC void video_stream_decoding_error_reported(VideoStream *stream);

821 822 823 824 825 826
/**
 * Tell the video stream that a decoding error has been recovered so that new decoding can be reported sooner.
 * @param[in] stream The VideoStream object.
 */
MS2_PUBLIC void video_stream_decoding_error_recovered(VideoStream *stream);

Simon Morlat's avatar
Simon Morlat committed
827

828 829 830 831 832 833 834
/**
 * Force a resolution for the preview.
 * @param[in] stream The VideoStream object.
 * @param[in] vsize video resolution.
**/
MS2_PUBLIC void video_stream_set_preview_size(VideoStream *stream, MSVideoSize vsize);

835 836 837 838 839 840 841
/**
 * Force a resolution for the preview.
 * @param[in] stream The VideoStream object.
 * @param[in] fps the frame rate in frame/seconds. A value of zero means "use encoder default value".
**/
MS2_PUBLIC void video_stream_set_fps(VideoStream *stream, float fps);

Simon Morlat's avatar
Simon Morlat committed
842 843 844 845 846 847 848 849 850 851 852 853
/**
 * Link the audio stream with an existing video stream.
 * This is necessary to enable recording of audio & video into a multimedia file.
 */
MS2_PUBLIC void audio_stream_link_video(AudioStream *stream, VideoStream *video);

/**
 * Unlink the audio stream from the video stream.
 * This must be done if the video stream is about to be stopped.
**/
MS2_PUBLIC void audio_stream_unlink_video(AudioStream *stream, VideoStream *video);

854 855 856 857
/**
 * Small API to display a local preview window.
**/

Simon Morlat's avatar
Simon Morlat committed
858 859
typedef VideoStream VideoPreview;

860
MS2_PUBLIC VideoPreview * video_preview_new(void);
861
#define video_preview_set_size(p,s)			video_stream_set_sent_video_size(p,s)
Simon Morlat's avatar
Simon Morlat committed
862
#define video_preview_set_display_filter_name(p,dt)	video_stream_set_display_filter_name(p,dt)
863 864 865
#define video_preview_set_native_window_id(p,id)	video_stream_set_native_preview_window_id(p,id)
#define video_preview_get_native_window_id(p)		video_stream_get_native_preview_window_id(p)
#define video_preview_set_fps(p,fps)			video_stream_set_fps((VideoStream*)p,fps)
866
#define video_preview_set_device_rotation(p, r) video_stream_set_device_rotation(p, r)
Simon Morlat's avatar
Simon Morlat committed
867
MS2_PUBLIC void video_preview_start(VideoPreview *stream, MSWebCam *device);
Simon Morlat's avatar
Simon Morlat committed
868
MS2_PUBLIC MSVideoSize video_preview_get_current_size(VideoPreview *stream);
Simon Morlat's avatar
Simon Morlat committed
869
MS2_PUBLIC void video_preview_stop(VideoPreview *stream);
aymeric's avatar
aymeric committed
870

871 872 873 874 875 876 877 878
/**
 * Stops the video preview graph but keep the source filter for reuse.
 * This is useful when transitioning from a preview-only to a duplex video.
 * The filter needs to be passed to the #video_stream_start_with_source function,
 * otherwise you should detroy it.
 */
MS2_PUBLIC MSFilter* video_preview_stop_reuse_source(VideoPreview *stream);

879 880 881 882
/**
 * @}
**/

jehan's avatar
jehan committed
883

aymeric's avatar
aymeric committed
884

Guillaume Beraudo's avatar
Guillaume Beraudo committed
885

aymeric's avatar
aymeric committed
886 887 888 889 890
#ifdef __cplusplus
}
#endif

#endif