diff --git a/ivfenc.c b/ivfenc.c
index b13db652228216f384562408a15a33d709fdab02..e9a49cddd46205694a9068b0326a0c16851f008b 100644
--- a/ivfenc.c
+++ b/ivfenc.c
@@ -950,7 +950,7 @@ int main(int argc, const char **argv_)
             SHOW(kf_max_dist);
         }
 
-        if(pass == 0) {
+        if(pass == (one_pass_only ? one_pass_only - 1 : 0)) {
             if (file_type == FILE_TYPE_Y4M)
                 /*The Y4M reader does its own allocation.
                   Just initialize this here to avoid problems if we never read any