Commit 9492b114 authored by Simon Morlat's avatar Simon Morlat

- improve v4l2 grabber for frame rate control

- improve rate control of mpeg4/h263 encoder
parent cd5b40be
......@@ -52,14 +52,13 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#endif
typedef struct V4l2State{
int fd;
#ifdef V4L2_THREADED
ms_thread_t thread;
bool_t thread_run;
queue_t rq;
ms_mutex_t mutex;
#endif
char *dev;
char *mmapdbuf;
int msize;/*mmapped size*/
......@@ -72,8 +71,10 @@ typedef struct V4l2State{
int frame_ind;
int frame_max;
float fps;
float start_time;
int frame_count;
unsigned int start_time;
unsigned int last_frame_time;
float mean_inter_frame;
int th_frame_count;
int queued;
bool_t configured;
}V4l2State;
......@@ -276,7 +277,7 @@ static int msv4l2_do_mmap(V4l2State *s){
return 0;
}
static mblk_t * v4lv2_grab_image(V4l2State *s){
static mblk_t * v4lv2_grab_image(V4l2State *s, int poll_timeout_ms){
struct v4l2_buffer buf;
unsigned int k;
memset(&buf,0,sizeof(buf));
......@@ -307,7 +308,7 @@ static mblk_t * v4lv2_grab_image(V4l2State *s){
fds.events=POLLIN;
fds.fd=s->fd;
/*check with poll if there is something to read */
if (poll(&fds,1,0)==1 && fds.revents==POLLIN){
if (poll(&fds,1,poll_timeout_ms)==1 && fds.revents==POLLIN){
if (v4l2_ioctl(s->fd, VIDIOC_DQBUF, &buf)<0) {
switch (errno) {
case EAGAIN:
......@@ -372,25 +373,19 @@ static void msv4l2_init(MSFilter *f){
s->fps=15;
s->configured=FALSE;
f->data=s;
#ifdef V4L2_THREADED
s->thread_run = TRUE;
qinit(&s->rq);
#endif
}
static void msv4l2_uninit(MSFilter *f){
V4l2State *s=(V4l2State*)f->data;
ms_free(s->dev);
#ifdef V4L2_THREADED
flushq(&s->rq,0);
ms_mutex_destroy(&s->mutex);
#endif
ms_free(s);
}
#ifdef V4L2_THREADED
static void *msv4l2_thread(void *ptr){
V4l2State *s=(V4l2State*)ptr;
int err=-1;
ms_message("msv4l2_thread starting");
if (s->fd!=-1)
{
......@@ -413,10 +408,9 @@ static void *msv4l2_thread(void *ptr){
ms_message("V4L2 video capture started.");
while(s->thread_run)
{
mblk_t *m;
if (s->fd!=-1){
mblk_t *m;
m=v4lv2_grab_image(s);
m=v4lv2_grab_image(s,50);
if (m){
mblk_t *om=dupb(m);
mblk_set_marker_info(om,(s->pix_fmt==MS_MJPEG));
......@@ -425,9 +419,7 @@ static void *msv4l2_thread(void *ptr){
ms_mutex_unlock(&s->mutex);
}
}
}
ms_message("thread:%d",s->thread_run);
munmap:
}
msv4l2_do_munmap(s);
close:
msv4l2_close(s);
......@@ -436,45 +428,39 @@ exit:
s->fd = -1;
ms_thread_exit(NULL);
}
#endif
static void msv4l2_preprocess(MSFilter *f){
V4l2State *s=(V4l2State*)f->data;
#ifdef V4L2_THREADED
s->thread_run=TRUE;
ms_thread_create(&s->thread,NULL,msv4l2_thread,s);
#else
if (s->fd==-1 && msv4l2_open(s)!=0) {
return;
}
if (!s->configured && msv4l2_configure(s)!=0){
return;
}
if (msv4l2_do_mmap(s)==0){
ms_message("V4L2 video capture started.");
}else{
msv4l2_close(s);
}
s->start_time=f->ticker->time;
#endif
s->th_frame_count=-1;
s->mean_inter_frame=0;
}
static void msv4l2_process(MSFilter *f){
V4l2State *s=(V4l2State*)f->data;
#ifdef V4L2_THREADED
uint32_t timestamp;
int cur_frame;
if (s->frame_count==-1){
s->start_time=f->ticker->time;
s->frame_count=0;
uint32_t curtime=f->ticker->time;
float elapsed;
if (s->th_frame_count==-1){
s->start_time=curtime;
s->th_frame_count=0;
}
cur_frame=((f->ticker->time-s->start_time)*s->fps/1000.0);
elapsed=((float)(curtime-s->start_time))/1000.0;
cur_frame=elapsed*s->fps;
if (cur_frame>=s->frame_count){
if (cur_frame>=s->th_frame_count){
mblk_t *om=NULL;
ms_mutex_lock(&s->mutex);
/*keep the most recent frame if several frames have been captured */
if (s->fd!=-1){
om=getq(&s->rq);
mblk_t *tmp=NULL;
while((tmp=getq(&s->rq))!=NULL){
if (om!=NULL) freemsg(om);
om=tmp;
}
}
ms_mutex_unlock(&s->mutex);
if (om!=NULL){
......@@ -482,47 +468,32 @@ static void msv4l2_process(MSFilter *f){
mblk_set_timestamp_info(om,timestamp);
mblk_set_marker_info(om,TRUE);
ms_queue_put(f->outputs[0],om);
/*ms_message("picture sent");*/
s->frame_count++;
}
}else{
flushq(&s->rq,0);
}
#else
uint32_t elapsed;
if (s->fd!=-1){
/*see it is necessary to output a frame:*/
elapsed=f->ticker->time-s->start_time;
if (((float)elapsed*s->fps/1000.0)>s->frame_count){
mblk_t *m;
m=v4lv2_grab_image(s);
if (m){
mblk_t *om=dupb(m);
mblk_set_marker_info(om,(s->pix_fmt==MS_MJPEG));
ms_queue_put(f->outputs[0],om);
s->frame_count++;
if (s->last_frame_time!=-1){
float frame_interval=(float)(curtime-s->last_frame_time)/1000.0;
if (s->mean_inter_frame==0){
s->mean_inter_frame=frame_interval;
}else{
s->mean_inter_frame=(0.8*s->mean_inter_frame)+(0.2*frame_interval);
}
}
s->last_frame_time=curtime;
}
s->th_frame_count++;
if (s->th_frame_count%50==0 && s->mean_inter_frame!=0){
ms_message("Captured mean fps=%f, expected=%f",1/s->mean_inter_frame, s->fps);
}
}
#endif
}
static void msv4l2_postprocess(MSFilter *f){
V4l2State *s=(V4l2State*)f->data;
#ifdef V4L2_THREADED
s->thread_run = FALSE;
if(ms_thread_join(s->thread,NULL))
ms_warning("msv4l2 thread was already stopped");
else
ms_message("msv4l2 thread has joined.");
flushq(&s->rq,0);
#else
if (s->fd!=-1){
msv4l2_do_munmap(s);
msv4l2_close(s);
}
#endif
}
static int msv4l2_set_fps(MSFilter *f, void *arg){
......@@ -549,9 +520,7 @@ static int msv4l2_get_pixfmt(MSFilter *f, void *arg){
if (msv4l2_open(s)==0){
msv4l2_configure(s);
*(MSPixFmt*)arg=s->pix_fmt;
#ifdef V4L2_THREADED
msv4l2_close(s);
#endif
return 0;
}else return -1;
}
......
......@@ -35,6 +35,8 @@ Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#include "rfc2429.h"
#define RATE_CONTROL_MARGIN 15000 /*bits/second*/
static bool_t avcodec_initialized=FALSE;
#ifdef ENABLE_LOG_FFMPEG
......@@ -213,13 +215,19 @@ static void prepare(EncState *s){
}
/* put codec parameters */
c->bit_rate=(float)s->maxbr*0.7;
c->bit_rate_tolerance=s->fps!=1?(float)c->bit_rate/(s->fps-1):c->bit_rate;
/* in order to take in account RTP protocol overhead and avoid possible
bitrate peaks especially on low bandwidth, we make a correction on the
codec's target bitrate.
*/
c->bit_rate=(float)s->maxbr*0.92;
if (c->bit_rate>RATE_CONTROL_MARGIN){
c->bit_rate -= RATE_CONTROL_MARGIN;
}
c->bit_rate_tolerance=s->fps>1?(float)c->bit_rate/(s->fps-1):c->bit_rate;
if (s->codec!=CODEC_ID_SNOW && s->maxbr<256000){
/*snow does not like 1st pass rate control*/
/*and rate control eats too much cpu with CIF high fps pictures*/
c->rc_max_rate=(float)s->maxbr*0.8;
c->rc_max_rate=c->bit_rate;
c->rc_min_rate=0;
c->rc_buffer_size=c->rc_max_rate;
}else{
......@@ -232,7 +240,7 @@ static void prepare(EncState *s){
c->height = s->vsize.height;
c->time_base.num = 1;
c->time_base.den = (int)s->fps;
c->gop_size=(int)s->fps*5; /*emit I frame every 5 seconds*/
c->gop_size=(int)s->fps*10; /*emit I frame every 10 seconds*/
c->pix_fmt=PIX_FMT_YUV420P;
s->comp_buf=allocb(c->bit_rate*2,0);
if (s->codec==CODEC_ID_SNOW){
......@@ -279,15 +287,6 @@ static void enc_uninit(MSFilter *f){
EncState *s=(EncState*)f->data;
ms_free(s);
}
#if 0
static void enc_set_rc(EncState *s, AVCodecContext *c){
int factor=c->width/MS_VIDEO_SIZE_QCIF_W;
c->rc_min_rate=0;
c->bit_rate=400; /* this value makes around 100kbit/s at QCIF=2 */
c->rc_max_rate=c->bit_rate+1;
c->rc_buffer_size=20000*factor; /* food recipe */
}
#endif
static void enc_preprocess(MSFilter *f){
EncState *s=(EncState*)f->data;
......@@ -795,19 +794,19 @@ static int enc_set_br(MSFilter *f, void *arg){
if (s->maxbr>=1024000 && s->codec!=CODEC_ID_H263P){
s->vsize.width = MS_VIDEO_SIZE_SVGA_W;
s->vsize.height = MS_VIDEO_SIZE_SVGA_H;
s->fps=17;
s->fps=25;
}else if (s->maxbr>=800000 && s->codec!=CODEC_ID_H263P){
s->vsize.width = MS_VIDEO_SIZE_VGA_W;
s->vsize.height = MS_VIDEO_SIZE_VGA_H;
s->fps=17;
s->fps=25;
}else if (s->maxbr>=512000){
s->vsize.width=MS_VIDEO_SIZE_CIF_W;
s->vsize.height=MS_VIDEO_SIZE_CIF_H;
s->fps=17;
s->fps=25;
}else if (s->maxbr>=256000){
s->vsize.width=MS_VIDEO_SIZE_CIF_W;
s->vsize.height=MS_VIDEO_SIZE_CIF_H;
s->fps=10;
s->fps=15;
s->qmin=3;
}else if (s->maxbr>=128000){
s->vsize.width=MS_VIDEO_SIZE_QCIF_W;
......
......@@ -563,11 +563,13 @@ static int sdl_create_window(SdlDisplay *wd, int w, int h){
if (info->blit_hw_A)
ms_message("Alpha blits between sw to hw surfaces: accelerated");
wd->sdl_screen = SDL_SetVideoMode(wd->screen_size.width,wd->screen_size.height, 0,flags);
wd->sdl_screen = SDL_SetVideoMode(w,h, 0,flags);
if (wd->sdl_screen == NULL ) {
ms_warning("no hardware for video mode: %s\n",
SDL_GetError());
}
wd->screen_size.width = w;
wd->screen_size.height = h;
if (wd->sdl_screen->flags & SDL_HWSURFACE) ms_message("SDL surface created in hardware");
if (once) {
SDL_WM_SetCaption("Video window", NULL);
......@@ -608,12 +610,9 @@ static bool_t sdl_display_init(MSDisplay *obj, MSFilter *f, MSPicture *fbuf, MSP
}
ms_mutex_init(&wd->sdl_mutex,NULL);
ms_mutex_lock(&wd->sdl_mutex);
wd->screen_size.width = fbuf->w;
wd->screen_size.height = fbuf->h;
}else {
ms_mutex_lock(&wd->sdl_mutex);
if (wd->lay!=NULL)
SDL_FreeYUVOverlay(wd->lay);
if (wd->sdl_screen!=NULL)
......@@ -755,551 +754,6 @@ MSDisplayDesc ms_sdl_display_desc={
.pollevent=sdl_poll_event,
};
#elif defined(WIN32)
#include <Vfw.h>
typedef struct _WinDisplay{
MSFilter *filter;
HWND window;
HDRAWDIB ddh;
MSPicture fb;
MSPicture fb_selfview;
uint8_t *rgb_selfview;
int rgb_len_selfview;
struct ms_SwsContext *sws_selfview;
MSDisplayEvent last_rsz;
uint8_t *rgb;
int last_rect_w;
int last_rect_h;
int rgb_len;
struct ms_SwsContext *sws;
bool_t new_ev;
}WinDisplay;
static LRESULT CALLBACK window_proc(
HWND hwnd, // handle to window
UINT uMsg, // message identifier
WPARAM wParam, // first message parameter
LPARAM lParam) // second message parameter
{
switch(uMsg){
case WM_DESTROY:
break;
case WM_SIZE:
if (wParam==SIZE_RESTORED){
int h=(lParam>>16) & 0xffff;
int w=lParam & 0xffff;
MSDisplay *obj;
WinDisplay *wd;
ms_message("Resized to %i,%i",w,h);
obj=(MSDisplay*)GetWindowLongPtr(hwnd,GWLP_USERDATA);
if (obj!=NULL){
wd=(WinDisplay*)obj->data;
wd->last_rsz.evtype=MS_DISPLAY_RESIZE_EVENT;
wd->last_rsz.w=w;
wd->last_rsz.h=h;
wd->new_ev=TRUE;
}else{
ms_error("Could not retrieve MSDisplay from window !");
}
}
break;
default:
return DefWindowProc(hwnd, uMsg, wParam, lParam);
}
return 0;
}
static HWND create_window(int w, int h)
{
WNDCLASS wc;
HINSTANCE hInstance = GetModuleHandle(NULL);
HWND hwnd;
RECT rect;
wc.style = 0 ;
wc.lpfnWndProc = window_proc;
wc.cbClsExtra = 0;
wc.cbWndExtra = 0;
wc.hInstance = NULL;
wc.hIcon = NULL;
wc.hCursor = LoadCursor(hInstance, IDC_ARROW);
wc.hbrBackground = NULL;
wc.lpszMenuName = NULL;
wc.lpszClassName = "Video Window";
if(!RegisterClass(&wc))
{
/* already registred! */
}
rect.left=100;
rect.top=100;
rect.right=rect.left+w;
rect.bottom=rect.top+h;
if (!AdjustWindowRect(&rect,WS_OVERLAPPEDWINDOW|WS_VISIBLE /*WS_CAPTION WS_TILED|WS_BORDER*/,FALSE)){
ms_error("AdjustWindowRect failed.");
}
ms_message("AdjustWindowRect: %li,%li %li,%li",rect.left,rect.top,rect.right,rect.bottom);
hwnd=CreateWindow("Video Window", "Video window",
WS_OVERLAPPEDWINDOW /*WS_THICKFRAME*/ | WS_VISIBLE ,
CW_USEDEFAULT, CW_USEDEFAULT, rect.right-rect.left,rect.bottom-rect.top,
NULL, NULL, hInstance, NULL);
if (hwnd==NULL){
ms_error("Fail to create video window");
}
return hwnd;
}
static bool_t win_display_init(MSDisplay *obj, MSFilter *f, MSPicture *fbuf, MSPicture *fbuf_selfview){
WinDisplay *wd=(WinDisplay*)obj->data;
int ysize,usize;
if (wd!=NULL)
{
wd->filter = NULL;
if (wd->ddh) DrawDibClose(wd->ddh);
wd->ddh=NULL;
if (wd->fb.planes[0]) ms_free(wd->fb.planes[0]);
wd->fb.planes[0]=NULL;
wd->fb.planes[1]=NULL;
wd->fb.planes[2]=NULL;
wd->fb.planes[3]=NULL;
if (wd->rgb) ms_free(wd->rgb);
wd->rgb=NULL;
wd->rgb_len=0;
ms_sws_freeContext(wd->sws);
wd->sws=NULL;
if (wd->fb_selfview.planes[0]) ms_free(wd->fb_selfview.planes[0]);
wd->fb_selfview.planes[0]=NULL;
wd->fb_selfview.planes[1]=NULL;
wd->fb_selfview.planes[2]=NULL;
wd->fb_selfview.planes[3]=NULL;
if (wd->rgb_selfview) ms_free(wd->rgb_selfview);
wd->rgb_selfview=NULL;
wd->rgb_len_selfview=0;
ms_sws_freeContext(wd->sws_selfview);
wd->sws_selfview=NULL;
wd->last_rect_w=0;
wd->last_rect_h=0;
}
else
wd=(WinDisplay*)ms_new0(WinDisplay,1);
wd->filter = f;
obj->data=wd;
wd->fb.w=fbuf->w;
wd->fb.h=fbuf->h;
wd->fb_selfview.w=fbuf_selfview->w;
wd->fb_selfview.h=fbuf_selfview->h;
if (wd->window==NULL){
if (obj->use_external_window && obj->window_id!=0){
void *p;
wd->window=(HWND)obj->window_id;
p=(void*)GetWindowLongPtr(wd->window,GWLP_USERDATA);
if (p!=NULL){
ms_error("Gulp: this externally supplied windows seems to "
"already have a userdata ! resizing will crash !");
}else SetWindowLongPtr(wd->window,GWLP_USERDATA,(LONG_PTR)obj);
}else{
wd->window=create_window(wd->fb.w,wd->fb.h);
obj->window_id=(long)wd->window;
if (wd->window!=NULL) SetWindowLongPtr(wd->window,GWLP_USERDATA,(LONG_PTR)obj);
else return FALSE;
}
}else if (!obj->use_external_window){
/* the window might need to be resized*/
RECT cur;
GetWindowRect(wd->window,&cur);
MoveWindow(wd->window,cur.left, cur.top, wd->fb.w, wd->fb.h,TRUE);
}
if (wd->ddh==NULL) wd->ddh=DrawDibOpen();
if (wd->ddh==NULL){
ms_error("DrawDibOpen() failed.");
return FALSE;
}
/*allocate yuv and rgb buffers*/
if (wd->fb_selfview.planes[0]) ms_free(wd->fb_selfview.planes[0]);
if (wd->rgb_selfview) ms_free(wd->rgb_selfview);
ysize=wd->fb_selfview.w*wd->fb_selfview.h;
usize=ysize/4;
fbuf_selfview->planes[0]=wd->fb_selfview.planes[0]=(uint8_t*)ms_malloc0(ysize+2*usize);
fbuf_selfview->planes[1]=wd->fb_selfview.planes[1]=wd->fb_selfview.planes[0]+ysize;
fbuf_selfview->planes[2]=wd->fb_selfview.planes[2]=wd->fb_selfview.planes[1]+usize;
fbuf_selfview->planes[3]=NULL;
fbuf_selfview->strides[0]=wd->fb_selfview.strides[0]=wd->fb_selfview.w;
fbuf_selfview->strides[1]=wd->fb_selfview.strides[1]=wd->fb_selfview.w/2;
fbuf_selfview->strides[2]=wd->fb_selfview.strides[2]=wd->fb_selfview.w/2;
fbuf_selfview->strides[3]=0;
wd->rgb_len_selfview=ysize*3;
wd->rgb_selfview=(uint8_t*)ms_malloc0(wd->rgb_len_selfview);
if (wd->fb.planes[0]) ms_free(wd->fb.planes[0]);
if (wd->rgb) ms_free(wd->rgb);
ysize=wd->fb.w*wd->fb.h;
usize=ysize/4;
fbuf->planes[0]=wd->fb.planes[0]=(uint8_t*)ms_malloc0(ysize+2*usize);
fbuf->planes[1]=wd->fb.planes[1]=wd->fb.planes[0]+ysize;
fbuf->planes[2]=wd->fb.planes[2]=wd->fb.planes[1]+usize;
fbuf->planes[3]=NULL;
fbuf->strides[0]=wd->fb.strides[0]=wd->fb.w;
fbuf->strides[1]=wd->fb.strides[1]=wd->fb.w/2;
fbuf->strides[2]=wd->fb.strides[2]=wd->fb.w/2;
fbuf->strides[3]=0;
wd->rgb_len=ysize*3;
wd->rgb=(uint8_t*)ms_malloc0(wd->rgb_len);
wd->last_rect_w=0;
wd->last_rect_h=0;
return TRUE;
}
typedef struct rgb{
uint8_t r,g,b;
} rgb_t;
typedef struct yuv{
uint8_t y,u,v;
} yuv_t;
static void yuv420p_to_rgb(WinDisplay *wd, MSPicture *src, uint8_t *rgb){
int rgb_stride=-src->w*3;
uint8_t *p;
p=rgb+(src->w*3*(src->h-1));
if (wd->sws==NULL){
wd->sws=ms_sws_getContext(src->w,src->h,PIX_FMT_YUV420P,
src->w,src->h, PIX_FMT_BGR24,
SWS_FAST_BILINEAR, NULL, NULL, NULL);
}
if (ms_sws_scale(wd->sws,src->planes,src->strides, 0,
src->h, &p, &rgb_stride)<0){
ms_error("Error in 420->rgb ms_sws_scale().");
}
}
static void yuv420p_to_rgb_selfview(WinDisplay *wd, MSPicture *src, uint8_t *rgb){
int rgb_stride=-src->w*3;
uint8_t *p;
p=rgb+(src->w*3*(src->h-1));
if (wd->sws_selfview==NULL){
wd->sws_selfview=ms_sws_getContext(src->w,src->h,PIX_FMT_YUV420P,
src->w,src->h, PIX_FMT_BGR24,
SWS_FAST_BILINEAR, NULL, NULL, NULL);
}
if (ms_sws_scale(wd->sws_selfview,src->planes,src->strides, 0,
src->h, &p, &rgb_stride)<0){
ms_error("Error in 420->rgb ms_sws_scale().");
}
}
static void win_display_update(MSDisplay *obj, int new_image, int new_selfview){
WinDisplay *wd=(WinDisplay*)obj->data;
HDC hdc;
BITMAPINFOHEADER bi;
RECT rect;
bool_t ret;
int ratiow;
int ratioh;
int w;
int h;
int corner;
float sv_scalefactor;
float sv_pos[3];
int color[3];
HDC dd_hdc;
HBITMAP dd_bmp;
HBRUSH brush;
BOOL dont_draw;
if (wd->window==NULL) return;
hdc=GetDC(wd->window);
if (hdc==NULL) {
ms_error("Could not get window dc");
return;
}
if (new_image>0)
yuv420p_to_rgb(wd, &wd->fb, wd->rgb);
memset(&bi,0,sizeof(bi));
bi.biSize=sizeof(bi);
GetClientRect(wd->window,&rect);