mediastream.h 15.2 KB
Newer Older
aymeric's avatar
aymeric committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23
/*
mediastreamer2 library - modular sound and video processing and streaming
Copyright (C) 2006  Simon MORLAT (simon.morlat@linphone.org)

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
*/


#ifndef MEDIASTREAM_H
#define MEDIASTREAM_H

Simon Morlat's avatar
Simon Morlat committed
24 25 26 27 28
#include <mediastreamer2/msfilter.h>
#include <mediastreamer2/msticker.h>
#include <mediastreamer2/mssndcard.h>
#include <mediastreamer2/mswebcam.h>
#include <mediastreamer2/msvideo.h>
29 30
#include <mediastreamer2/bitratecontrol.h>
#include <mediastreamer2/qualityindicator.h>
Simon Morlat's avatar
Simon Morlat committed
31 32
#include <ortp/ortp.h>
#include <ortp/event.h>
Guillaume Beraudo's avatar
Guillaume Beraudo committed
33
#include <ortp/zrtp.h>
34
#include <ortp/ortp_srtp.h>
35

36
#include <ortp/zrtp.h>
aymeric's avatar
aymeric committed
37

Simon Morlat's avatar
Simon Morlat committed
38 39 40
#define PAYLOAD_TYPE_FLAG_CAN_RECV	PAYLOAD_TYPE_USER_FLAG_1
#define PAYLOAD_TYPE_FLAG_CAN_SEND	PAYLOAD_TYPE_USER_FLAG_2

41 42 43
typedef enum EchoLimiterType{
	ELInactive,
	ELControlMic,
44
	ELControlFull
45 46
} EchoLimiterType;

aymeric's avatar
aymeric committed
47 48 49 50 51 52 53 54 55 56 57
struct _AudioStream
{
	MSTicker *ticker;
	RtpSession *session;
	MSFilter *soundread;
	MSFilter *soundwrite;
	MSFilter *encoder;
	MSFilter *decoder;
	MSFilter *rtprecv;
	MSFilter *rtpsend;
	MSFilter *dtmfgen;
58
	MSFilter *dtmfgen_rtp;
aymeric's avatar
aymeric committed
59
	MSFilter *ec;/*echo canceler*/
60
	MSFilter *volsend,*volrecv; /*MSVolumes*/
61 62
	MSFilter *read_resampler;
	MSFilter *write_resampler;
smorlat's avatar
smorlat committed
63
	MSFilter *equalizer;
aymeric's avatar
aymeric committed
64
	uint64_t last_packet_count;
aymeric's avatar
aymeric committed
65
	time_t last_packet_time;
66
	EchoLimiterType el_type; /*use echo limiter: two MSVolume, measured input level controlling local output level*/
67
	OrtpEvQueue *evq;
68
	MSBitrateController *rc;
69
	MSQualityIndicator *qi;
70
	time_t start_time;
smorlat's avatar
smorlat committed
71
	bool_t play_dtmfs;
smorlat's avatar
smorlat committed
72
	bool_t use_gc;
73
	bool_t use_agc;
smorlat's avatar
smorlat committed
74
	bool_t eq_active;
75
	bool_t use_ng;/*noise gate*/
76
	bool_t use_rc;
77
	bool_t is_beginning;
Guillaume Beraudo's avatar
Guillaume Beraudo committed
78
	OrtpZrtpContext *ortpZrtpContext;
79
	srtp_t srtp_session;
aymeric's avatar
aymeric committed
80 81 82 83 84 85
};

#ifdef __cplusplus
extern "C" {
#endif

86 87 88 89 90 91 92 93
/**
 * @addtogroup audio_stream_api
 * @{
**/

/**
 * The AudioStream holds all resources to create and run typical VoIP audiostream.
**/
aymeric's avatar
aymeric committed
94 95 96 97 98 99
typedef struct _AudioStream AudioStream;

struct _RingStream
{
	MSTicker *ticker;
	MSFilter *source;
100
	MSFilter *gendtmf;
Simon Morlat's avatar
Simon Morlat committed
101
	MSFilter *write_resampler;
aymeric's avatar
aymeric committed
102 103 104 105 106 107 108 109
	MSFilter *sndwrite;
};

typedef struct _RingStream RingStream;



/* start a thread that does sampling->encoding->rtp_sending|rtp_receiving->decoding->playing */
Simon Morlat's avatar
Simon Morlat committed
110
MS2_PUBLIC AudioStream *audio_stream_start (RtpProfile * prof, int locport, const char *remip,
aymeric's avatar
aymeric committed
111 112
				 int remport, int payload_type, int jitt_comp, bool_t echo_cancel);

Simon Morlat's avatar
Simon Morlat committed
113
MS2_PUBLIC AudioStream *audio_stream_start_with_sndcards(RtpProfile * prof, int locport, const char *remip4, int remport, int payload_type, int jitt_comp, MSSndCard *playcard, MSSndCard *captcard, bool_t echocancel);
aymeric's avatar
aymeric committed
114

115

Simon Morlat's avatar
Simon Morlat committed
116
MS2_PUBLIC int audio_stream_start_with_files (AudioStream * stream, RtpProfile * prof,
aymeric's avatar
aymeric committed
117 118 119 120
					    const char *remip, int remport, int rem_rtcp_port,
					    int pt, int jitt_comp,
					    const char * infile,  const char * outfile);

121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141
/**
 * Starts an audio stream from/to local wav files or soundcards.
 * 
 * This method starts the processing of the audio stream, that is playing from wav file or soundcard, voice processing, encoding,
 * sending through RTP, receiving from RTP, decoding, voice processing and wav file recording or soundcard playback.
 * 
 * 
 * @param stream an AudioStream previously created with audio_stream_new().
 * @param prof a RtpProfile containing all PayloadType possible during the audio session.
 * @param remip remote IP address where to send the encoded audio.
 * @param remport remote IP port where to send the encoded audio
 * @param rem_rtcp_port remote port for RTCP.
 * @param payload_type payload type index to use for the sending stream. This index must point to a valid PayloadType in the RtpProfile.
 * @param jitt_comp Nominal jitter buffer size in milliseconds.
 * @param infile path to wav file to play out (can be NULL)
 * @param outfile path to wav file to record into (can be NULL)
 * @param playcard The soundcard to be used for playback (can be NULL)
 * @param captcard The soundcard to be used for catpure. (can be NULL)
 * @param echo_cancel whether echo cancellation is to be performed.
 * @returns 0 if sucessful, -1 otherwise.
**/
Simon Morlat's avatar
Simon Morlat committed
142
MS2_PUBLIC int audio_stream_start_full(AudioStream *stream, RtpProfile *profile, const char *remip,int remport,
143 144 145
	int rem_rtcp_port, int payload,int jitt_comp, const char *infile, const char *outfile,
	MSSndCard *playcard, MSSndCard *captcard, bool_t use_ec);

Simon Morlat's avatar
Simon Morlat committed
146 147
MS2_PUBLIC void audio_stream_play(AudioStream *st, const char *name);
MS2_PUBLIC void audio_stream_record(AudioStream *st, const char *name);
aymeric's avatar
aymeric committed
148

Simon Morlat's avatar
Simon Morlat committed
149
MS2_PUBLIC void audio_stream_set_rtcp_information(AudioStream *st, const char *cname, const char *tool);
aymeric's avatar
aymeric committed
150

Simon Morlat's avatar
Simon Morlat committed
151
MS2_PUBLIC void audio_stream_play_received_dtmfs(AudioStream *st, bool_t yesno);
smorlat's avatar
smorlat committed
152

153 154 155 156 157 158
/**
 * Creates an AudioStream object listening on a RTP port.
 * @param locport the local UDP port to listen for RTP packets.
 * @param ipv6 TRUE if ipv6 must be used.
 * @returns a new AudioStream.
**/
Simon Morlat's avatar
Simon Morlat committed
159
MS2_PUBLIC AudioStream *audio_stream_new(int locport, bool_t ipv6);
160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176
/**
 * Starts an audio stream from local soundcards.
 * 
 * This method starts the processing of the audio stream, that is capture from soundcard, voice processing, encoding,
 * sending through RTP, receiving from RTP, decoding, voice processing and soundcard playback.
 * 
 * @param stream an AudioStream previously created with audio_stream_new().
 * @param prof a RtpProfile containing all PayloadType possible during the audio session.
 * @param remip remote IP address where to send the encoded audio.
 * @param remport remote IP port where to send the encoded audio
 * @param rem_rtcp_port remote port for RTCP.
 * @param payload_type payload type index to use for the sending stream. This index must point to a valid PayloadType in the RtpProfile.
 * @param jitt_comp Nominal jitter buffer size in milliseconds.
 * @param playcard The soundcard to be used for playback
 * @param captcard The soundcard to be used for catpure.
 * @param echo_cancel whether echo cancellation is to be performed.
**/
Simon Morlat's avatar
Simon Morlat committed
177 178
MS2_PUBLIC int audio_stream_start_now(AudioStream * stream, RtpProfile * prof,  const char *remip, int remport, int rem_rtcp_port, int payload_type, int jitt_comp,MSSndCard *playcard, MSSndCard *captcard, bool_t echo_cancel);
MS2_PUBLIC void audio_stream_set_relay_session_id(AudioStream *stream, const char *relay_session_id);
aymeric's avatar
aymeric committed
179
/*returns true if we are still receiving some data from remote end in the last timeout seconds*/
Simon Morlat's avatar
Simon Morlat committed
180
MS2_PUBLIC bool_t audio_stream_alive(AudioStream * stream, int timeout);
aymeric's avatar
aymeric committed
181

182 183 184 185
/**
 * Executes background low priority tasks related to audio processing (RTP statistics analysis).
 * It should be called periodically, for example with an interval of 100 ms or so.
 */
186 187
MS2_PUBLIC void audio_stream_iterate(AudioStream *stream);

smorlat's avatar
smorlat committed
188
/*enable echo-limiter dispositve: one MSVolume in input branch controls a MSVolume in the output branch*/
Simon Morlat's avatar
Simon Morlat committed
189
MS2_PUBLIC void audio_stream_enable_echo_limiter(AudioStream *stream, EchoLimiterType type);
190

smorlat's avatar
smorlat committed
191
/*enable gain control, to be done before start() */
Simon Morlat's avatar
Simon Morlat committed
192
MS2_PUBLIC void audio_stream_enable_gain_control(AudioStream *stream, bool_t val);
smorlat's avatar
smorlat committed
193

194
/*enable automatic gain control, to be done before start() */
Simon Morlat's avatar
Simon Morlat committed
195
MS2_PUBLIC void audio_stream_enable_automatic_gain_control(AudioStream *stream, bool_t val);
196

197
/*to be done before start */
Simon Morlat's avatar
Simon Morlat committed
198
MS2_PUBLIC void audio_stream_set_echo_canceller_params(AudioStream *st, int tail_len_ms, int delay_ms, int framesize);
199

200 201
/*enable adaptive rate control */
MS2_PUBLIC void audio_stream_enable_adaptive_bitrate_control(AudioStream *st, bool_t enabled);
202 203


Simon Morlat's avatar
Simon Morlat committed
204
MS2_PUBLIC void audio_stream_set_mic_gain(AudioStream *stream, float gain);
smorlat's avatar
smorlat committed
205

206
/* enable/disable rtp stream */
Simon Morlat's avatar
Simon Morlat committed
207
MS2_PUBLIC void audio_stream_mute_rtp(AudioStream *stream, bool_t val);
Simon Morlat's avatar
Simon Morlat committed
208

209
/*enable noise gate, must be done before start()*/
Simon Morlat's avatar
Simon Morlat committed
210
MS2_PUBLIC void audio_stream_enable_noise_gate(AudioStream *stream, bool_t val);
211

smorlat's avatar
smorlat committed
212
/*enable parametric equalizer in the stream that goes to the speaker*/
Simon Morlat's avatar
Simon Morlat committed
213
MS2_PUBLIC void audio_stream_enable_equalizer(AudioStream *stream, bool_t enabled);
smorlat's avatar
smorlat committed
214

Simon Morlat's avatar
Simon Morlat committed
215
MS2_PUBLIC void audio_stream_equalizer_set_gain(AudioStream *stream, int frequency, float gain, int freq_width);
smorlat's avatar
smorlat committed
216

smorlat's avatar
smorlat committed
217
/* stop the audio streaming thread and free everything*/
Simon Morlat's avatar
Simon Morlat committed
218
MS2_PUBLIC void audio_stream_stop (AudioStream * stream);
aymeric's avatar
aymeric committed
219

Simon Morlat's avatar
Simon Morlat committed
220 221 222
MS2_PUBLIC RingStream *ring_start (const char * file, int interval, MSSndCard *sndcard);
MS2_PUBLIC RingStream *ring_start_with_cb(const char * file, int interval, MSSndCard *sndcard, MSFilterNotifyFunc func, void * user_data);
MS2_PUBLIC void ring_stop (RingStream * stream);
aymeric's avatar
aymeric committed
223 224 225


/* send a dtmf */
Simon Morlat's avatar
Simon Morlat committed
226
MS2_PUBLIC int audio_stream_send_dtmf (AudioStream * stream, char dtmf);
aymeric's avatar
aymeric committed
227

Simon Morlat's avatar
Simon Morlat committed
228
MS2_PUBLIC void audio_stream_set_default_card(int cardindex);
aymeric's avatar
aymeric committed
229

Simon Morlat's avatar
Simon Morlat committed
230
/* retrieve RTP statistics*/
Simon Morlat's avatar
Simon Morlat committed
231
MS2_PUBLIC void audio_stream_get_local_rtp_stats(AudioStream *stream, rtp_stats_t *stats);
Simon Morlat's avatar
Simon Morlat committed
232

233
/* returns a realtime indicator of the stream quality between 0 and 5 */
234 235
MS2_PUBLIC float audio_stream_get_quality_rating(AudioStream *stream);

236 237
/* returns the quality rating as an average since the start of the streaming session.*/
MS2_PUBLIC float audio_stream_get_average_quality_rating(AudioStream *stream);
aymeric's avatar
aymeric committed
238

Guillaume Beraudo's avatar
Guillaume Beraudo committed
239 240 241
/* enable ZRTP on the audio stream */
MS2_PUBLIC void audio_stream_enable_zrtp(AudioStream *stream, OrtpZrtpParams *params);

242 243
/* enable SRTP on the audio stream */
MS2_PUBLIC bool_t audio_stream_enable_strp(AudioStream* stream, enum ortp_srtp_crypto_suite_t suite, const char* snd_key, const char* rcv_key);
Guillaume Beraudo's avatar
Guillaume Beraudo committed
244

aymeric's avatar
aymeric committed
245 246 247 248
/*****************
  Video Support
 *****************/

249
typedef void (*VideoStreamRenderCallback)(void *user_pointer, const MSPicture *local_view, const MSPicture *remote_view);
250 251 252
typedef void (*VideoStreamEventCallback)(void *user_pointer, const MSFilter *f, const unsigned int event_id, const void *args);


aymeric's avatar
aymeric committed
253

254 255 256 257 258 259
typedef enum _VideoStreamDir{
	VideoStreamSendRecv,
	VideoStreamSendOnly,
	VideoStreamRecvOnly
}VideoStreamDir;

aymeric's avatar
aymeric committed
260 261 262 263 264 265
struct _VideoStream
{
	MSTicker *ticker;
	RtpSession *session;
	MSFilter *source;
	MSFilter *pixconv;
266
	MSFilter *sizeconv;
aymeric's avatar
aymeric committed
267 268 269 270 271 272
	MSFilter *tee;
	MSFilter *output;
	MSFilter *encoder;
	MSFilter *decoder;
	MSFilter *rtprecv;
	MSFilter *rtpsend;
273 274
	MSFilter *tee2;
	MSFilter *jpegwriter;
275
	MSFilter *output2;
aymeric's avatar
aymeric committed
276
	OrtpEvQueue *evq;
smorlat's avatar
smorlat committed
277
	MSVideoSize sent_vsize;
smorlat's avatar
smorlat committed
278
	int corner; /*for selfview*/
279 280
	VideoStreamRenderCallback rendercb;
	void *render_pointer;
281 282
	VideoStreamEventCallback eventcb;
	void *event_pointer;
283
	char *display_name;
284 285
	unsigned long window_id;
	unsigned long preview_window_id;
286
	VideoStreamDir dir;
287
	MSWebCam *cam;
288
	bool_t use_preview_window;
289 290
	bool_t use_rc;
	bool_t pad[2];
291
    bool_t display_filter_auto_rotate_enabled;
292
	int device_orientation; /* warning: meaning of this variable depends on the platform (Android, iOS, ...) */
Guillaume Beraudo's avatar
Guillaume Beraudo committed
293
	OrtpZrtpContext *ortpZrtpContext;
294
	srtp_t srtp_session;
295
	MSBitrateController *rc;
aymeric's avatar
aymeric committed
296 297 298
};

typedef struct _VideoStream VideoStream;
smorlat's avatar
smorlat committed
299

300 301


Simon Morlat's avatar
Simon Morlat committed
302 303 304 305 306 307 308
MS2_PUBLIC VideoStream *video_stream_new(int locport, bool_t use_ipv6);
MS2_PUBLIC void video_stream_set_direction(VideoStream *vs, VideoStreamDir dir);
MS2_PUBLIC void video_stream_enable_adaptive_bitrate_control(VideoStream *s, bool_t yesno);
MS2_PUBLIC void video_stream_set_render_callback(VideoStream *s, VideoStreamRenderCallback cb, void *user_pointer);
MS2_PUBLIC void video_stream_set_event_callback(VideoStream *s, VideoStreamEventCallback cb, void *user_pointer);
MS2_PUBLIC void video_stream_set_display_filter_name(VideoStream *s, const char *fname);
MS2_PUBLIC int video_stream_start(VideoStream * stream, RtpProfile *profile, const char *remip, int remport, int rem_rtcp_port,
aymeric's avatar
aymeric committed
309
		int payload, int jitt_comp, MSWebCam *device);
310 311


Simon Morlat's avatar
Simon Morlat committed
312 313 314
MS2_PUBLIC void video_stream_set_relay_session_id(VideoStream *stream, const char *relay_session_id);
MS2_PUBLIC void video_stream_set_rtcp_information(VideoStream *st, const char *cname, const char *tool);
MS2_PUBLIC void video_stream_change_camera(VideoStream *stream, MSWebCam *cam);
315 316
/* Calling video_stream_set_sent_video_size() or changing the bitrate value in the used PayloadType during a stream is running does nothing.
The following function allows to take into account new parameters by redrawing the sending graph*/
Simon Morlat's avatar
Simon Morlat committed
317
MS2_PUBLIC void video_stream_update_video_params(VideoStream *stream);
aymeric's avatar
aymeric committed
318
/*function to call periodically to handle various events */
Simon Morlat's avatar
Simon Morlat committed
319 320 321 322 323 324 325 326 327 328
MS2_PUBLIC void video_stream_iterate(VideoStream *stream);
MS2_PUBLIC void video_stream_send_vfu(VideoStream *stream);
MS2_PUBLIC void video_stream_stop(VideoStream * stream);
MS2_PUBLIC void video_stream_set_sent_video_size(VideoStream *stream, MSVideoSize vsize);
MS2_PUBLIC void video_stream_enable_self_view(VideoStream *stream, bool_t val);
MS2_PUBLIC unsigned long video_stream_get_native_window_id(VideoStream *stream);
MS2_PUBLIC void video_stream_set_native_window_id(VideoStream *stream, unsigned long id);
MS2_PUBLIC void video_stream_set_native_preview_window_id(VideoStream *stream, unsigned long id);
MS2_PUBLIC unsigned long video_stream_get_native_preview_window_id(VideoStream *stream);
MS2_PUBLIC void video_stream_use_preview_video_window(VideoStream *stream, bool_t yesno);
329
MS2_PUBLIC void video_stream_set_device_rotation(VideoStream *stream, int orientation);
330
MS2_PUBLIC void video_stream_show_video(VideoStream *stream, bool_t show);
331

332
/*provided for compatibility, use video_stream_set_direction() instead */
Simon Morlat's avatar
Simon Morlat committed
333 334
MS2_PUBLIC int video_stream_recv_only_start(VideoStream *videostream, RtpProfile *profile, const char *addr, int port, int used_pt, int jitt_comp);
MS2_PUBLIC int video_stream_send_only_start(VideoStream *videostream,
335
				RtpProfile *profile, const char *addr, int port, int rtcp_port,
336
				int used_pt, int  jitt_comp, MSWebCam *device);
Simon Morlat's avatar
Simon Morlat committed
337 338
MS2_PUBLIC void video_stream_recv_only_stop(VideoStream *vs);
MS2_PUBLIC void video_stream_send_only_stop(VideoStream *vs);
aymeric's avatar
aymeric committed
339

Guillaume Beraudo's avatar
Guillaume Beraudo committed
340 341
/* enable ZRTP on the video stream using information from the audio stream */
MS2_PUBLIC void video_stream_enable_zrtp(VideoStream *vstream, AudioStream *astream, OrtpZrtpParams *param);
342

343 344
/* enable SRTP on the video stream */
MS2_PUBLIC bool_t video_stream_enable_strp(VideoStream* stream, enum ortp_srtp_crypto_suite_t suite, const char* snd_key, const char* rcv_key);
345 346
/* if enabled, the display filter will internaly rotate the video, according to the device orientation */
MS2_PUBLIC void video_stream_enable_display_filter_auto_rotate(VideoStream* stream, bool_t enable);
347

348 349 350 351
/**
 * Small API to display a local preview window.
**/

Simon Morlat's avatar
Simon Morlat committed
352 353
typedef VideoStream VideoPreview;

Simon Morlat's avatar
Simon Morlat committed
354
MS2_PUBLIC VideoPreview * video_preview_new();
Simon Morlat's avatar
Simon Morlat committed
355 356 357 358
#define video_preview_set_size(p,s) 							video_stream_set_sent_video_size(p,s)
#define video_preview_set_display_filter_name(p,dt)	video_stream_set_display_filter_name(p,dt)
#define video_preview_set_native_window_id(p,id)		video_stream_set_native_preview_window_id (p,id)
#define video_preview_get_native_window_id(p)			video_stream_get_native_preview_window_id (p)
Simon Morlat's avatar
Simon Morlat committed
359 360
MS2_PUBLIC void video_preview_start(VideoPreview *stream, MSWebCam *device);
MS2_PUBLIC void video_preview_stop(VideoPreview *stream);
aymeric's avatar
aymeric committed
361

Simon Morlat's avatar
Simon Morlat committed
362
MS2_PUBLIC bool_t ms_is_ipv6(const char *address);
aymeric's avatar
aymeric committed
363

Guillaume Beraudo's avatar
Guillaume Beraudo committed
364

aymeric's avatar
aymeric committed
365 366 367 368
#ifdef __cplusplus
}
#endif

369 370 371
/**
 * @}
**/
aymeric's avatar
aymeric committed
372 373

#endif