android-camera2-capture.cpp 38.9 KB
Newer Older
Sylvain Berfini's avatar
Sylvain Berfini committed
1
/*
Sylvain Berfini's avatar
Sylvain Berfini committed
2
 * Copyright (c) 2010-2019 Belledonne Communications SARL.
Sylvain Berfini's avatar
Sylvain Berfini committed
3
 *
Sylvain Berfini's avatar
Sylvain Berfini committed
4
 * android-camera2-capture.cpp - Android capture plugin using NDK Camera2 APIs.
Sylvain Berfini's avatar
Sylvain Berfini committed
5
 *
Sylvain Berfini's avatar
Sylvain Berfini committed
6 7 8 9
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
Sylvain Berfini's avatar
Sylvain Berfini committed
10
 *
Sylvain Berfini's avatar
Sylvain Berfini committed
11 12 13 14
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
Sylvain Berfini's avatar
Sylvain Berfini committed
15
 *
Sylvain Berfini's avatar
Sylvain Berfini committed
16 17
 * You should have received a copy of the GNU General Public License
 * along with this program. If not, see <http://www.gnu.org/licenses/>.
Sylvain Berfini's avatar
Sylvain Berfini committed
18 19 20
 */

#include <mediastreamer2/msfilter.h>
21
#include <mediastreamer2/msvideo.h>
Sylvain Berfini's avatar
Sylvain Berfini committed
22 23 24 25 26 27
#include <mediastreamer2/msjava.h>
#include <mediastreamer2/msticker.h>
#include <mediastreamer2/mssndcard.h>
#include <mediastreamer2/mswebcam.h>
#include <mediastreamer2/android_utils.h>

28
#include <android/native_window_jni.h>
Sylvain Berfini's avatar
Sylvain Berfini committed
29
#include <camera/NdkCaptureRequest.h>
Sylvain Berfini's avatar
Sylvain Berfini committed
30 31 32 33 34 35
#include <camera/NdkCameraCaptureSession.h>
#include <camera/NdkCameraDevice.h>
#include <camera/NdkCameraError.h>
#include <camera/NdkCameraManager.h>
#include <camera/NdkCameraMetadata.h>
#include <camera/NdkCameraMetadataTags.h>
36
#include <media/NdkImageReader.h>
Sylvain Berfini's avatar
Sylvain Berfini committed
37

38
#include <jni.h>
Sylvain Berfini's avatar
Sylvain Berfini committed
39 40
#include <math.h>

41 42
struct AndroidCamera2Device {
	AndroidCamera2Device(char *id) : camId(id), orientation(0), back_facing(false) {
Sylvain Berfini's avatar
Sylvain Berfini committed
43 44 45
		
	};

46 47 48 49 50 51 52 53 54 55 56 57
	~AndroidCamera2Device() {
		if (camId) {
			ms_free(camId);
		}
	};

	char *camId;
	int32_t orientation;
	bool back_facing;
};

struct AndroidCamera2Context {
Sylvain Berfini's avatar
Sylvain Berfini committed
58
	AndroidCamera2Context(MSFilter *f) : filter(f), configured(false), capturing(false), device(nullptr), rotation(0), nativeWindowId(nullptr), surface(nullptr),
59
			captureFormat(AIMAGE_FORMAT_YUV_420_888),
60 61
			frame(nullptr), bufAllocator(ms_yuv_buf_allocator_new()), fps(5), 
			cameraDevice(nullptr), captureSession(nullptr), captureSessionOutputContainer(nullptr), 
62
			nativeWindow(nullptr), captureWindow(nullptr), capturePreviewRequest(nullptr), 
63 64
			cameraCaptureOutputTarget(nullptr), cameraPreviewOutputTarget(nullptr),
			sessionCaptureOutput(nullptr), sessionPreviewOutput(nullptr), imageReader(nullptr) 
65 66 67 68 69 70
	{
		captureSize.width = 0;
		captureSize.height = 0;
		previewSize.width = 0;
		previewSize.height = 0;
		ms_mutex_init(&mutex, NULL);
71 72

    	cameraManager = ACameraManager_create();
73
		ms_message("[Camera2 Capture] Context ready");
74 75
	};

Sylvain Berfini's avatar
Sylvain Berfini committed
76
	~AndroidCamera2Context() {
77
		ms_message("[Camera2 Capture] Context destroyed");
78 79 80
		// Don't delete device object in here !
		ms_mutex_destroy(&mutex);
		if (bufAllocator) ms_yuv_buf_allocator_free(bufAllocator);
81 82

		ACameraManager_delete(cameraManager);
Sylvain Berfini's avatar
Sylvain Berfini committed
83
	};
84 85

	MSFilter *filter;
Sylvain Berfini's avatar
Sylvain Berfini committed
86
	bool configured;
87 88 89
	bool capturing;
	AndroidCamera2Device *device;
	int rotation;
90
	jobject nativeWindowId;
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
	jobject surface;

	MSVideoSize captureSize;
	MSVideoSize previewSize;
	int32_t captureFormat;

	ms_mutex_t mutex;
	mblk_t *frame;
	MSYuvBufAllocator *bufAllocator;

	float fps;
	MSFrameRateController fpsControl;
	MSAverageFPS averageFps;
	char fps_context[64];

106
	ACameraManager *cameraManager;
107 108 109 110 111 112
	ACameraDevice *cameraDevice;
	ACameraCaptureSession *captureSession;
	ACaptureSessionOutputContainer *captureSessionOutputContainer;

	ANativeWindow *nativeWindow;
	ANativeWindow *captureWindow;
113 114 115 116 117
	ACaptureRequest *capturePreviewRequest;
	ACameraOutputTarget *cameraCaptureOutputTarget;
	ACameraOutputTarget *cameraPreviewOutputTarget;
	ACaptureSessionOutput *sessionCaptureOutput;
	ACaptureSessionOutput *sessionPreviewOutput;
118
	AImageReader *imageReader;
Sylvain Berfini's avatar
Sylvain Berfini committed
119
	AImageReader_ImageListener *imageReaderListener;
120 121 122

	ACameraDevice_StateCallbacks deviceStateCallbacks;
	ACameraCaptureSession_stateCallbacks captureSessionStateCallbacks;
Sylvain Berfini's avatar
Sylvain Berfini committed
123 124
};

125 126
/* ************************************************************************* */

127 128
static void android_camera2_capture_stop(AndroidCamera2Context *d);

129 130
static void android_camera2_capture_device_on_disconnected(void *context, ACameraDevice *device) {
    ms_message("[Camera2 Capture] Camera %s is diconnected", ACameraDevice_getId(device));
131 132 133

	AndroidCamera2Context *d = (AndroidCamera2Context *)context;
	android_camera2_capture_stop(d);
134 135 136 137
}

static void android_camera2_capture_device_on_error(void *context, ACameraDevice *device, int error) {
    ms_error("[Camera2 Capture] Error %d on camera %s", error, ACameraDevice_getId(device));
138 139 140

	AndroidCamera2Context *d = (AndroidCamera2Context *)context;
	android_camera2_capture_stop(d);
141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156
}

static void android_camera2_capture_session_on_ready(void *context, ACameraCaptureSession *session) {
    ms_message("[Camera2 Capture] Session is ready %p", session);
}

static void android_camera2_capture_session_on_active(void *context, ACameraCaptureSession *session) {
    ms_message("[Camera2 Capture] Session is activated %p", session);
}

static void android_camera2_capture_session_on_closed(void *context, ACameraCaptureSession *session) {
    ms_message("[Camera2 Capture] Session is closed %p", session);
}

/* ************************************************************************* */

157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199
// https://developer.android.com/ndk/reference/group/camera.html
static const char* android_camera2_status_to_string(camera_status_t status) {
	if (status == ACAMERA_OK) {
		return "ACAMERA_OK";
	} else if (status == ACAMERA_ERROR_BASE) {
		return "ACAMERA_ERROR_BASE";
	} else if (status == ACAMERA_ERROR_UNKNOWN) {
		return "ACAMERA_ERROR_UNKNOWN";
	} else if (status == ACAMERA_ERROR_INVALID_PARAMETER) {
		return "ACAMERA_ERROR_INVALID_PARAMETER";
	} else if (status == ACAMERA_ERROR_CAMERA_DISCONNECTED) {
		return "ACAMERA_ERROR_CAMERA_DISCONNECTED";
	} else if (status == ACAMERA_ERROR_NOT_ENOUGH_MEMORY) {
		return "ACAMERA_ERROR_NOT_ENOUGH_MEMORY";
	} else if (status == ACAMERA_ERROR_METADATA_NOT_FOUND) {
		return "ACAMERA_ERROR_METADATA_NOT_FOUND";
	} else if (status == ACAMERA_ERROR_CAMERA_DEVICE) {
		return "ACAMERA_ERROR_CAMERA_DEVICE";
	} else if (status == ACAMERA_ERROR_CAMERA_SERVICE) {
		return "ACAMERA_ERROR_CAMERA_SERVICE";
	} else if (status == ACAMERA_ERROR_SESSION_CLOSED) {
		return "ACAMERA_ERROR_SESSION_CLOSED";
	} else if (status == ACAMERA_ERROR_INVALID_OPERATION) {
		return "ACAMERA_ERROR_INVALID_OPERATION";
	} else if (status == ACAMERA_ERROR_STREAM_CONFIGURE_FAIL) {
		return "ACAMERA_ERROR_STREAM_CONFIGURE_FAIL";
	} else if (status == ACAMERA_ERROR_CAMERA_IN_USE) {
		return "ACAMERA_ERROR_CAMERA_IN_USE";
	} else if (status == ACAMERA_ERROR_MAX_CAMERA_IN_USE) {
		return "ACAMERA_ERROR_MAX_CAMERA_IN_USE";
	} else if (status == ACAMERA_ERROR_CAMERA_DISABLED) {
		return "ACAMERA_ERROR_CAMERA_DISABLED";
	} else if (status == ACAMERA_ERROR_PERMISSION_DENIED) {
		return "ACAMERA_ERROR_PERMISSION_DENIED";
	} else if (status == -10014) { // Can't use ACAMERA_ERROR_UNSUPPORTED_OPERATION, not present in NDK 17c...
		return "ACAMERA_ERROR_UNSUPPORTED_OPERATION";
	}

	return "UNKNOWN";
}

/* ************************************************************************* */

200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251
static int32_t android_camera2_capture_get_orientation(AndroidCamera2Context *d) {
	int32_t orientation = d->device->orientation;
	if (d->device->back_facing) {
		orientation -= d->rotation;
	} else {
		orientation += d->rotation;
	}
	if (orientation < 0) {
		orientation += 360;
	}
	orientation = orientation % 360;
	return orientation;
}

static mblk_t* android_camera2_capture_image_to_mblkt(AndroidCamera2Context *d, AImage *image) {
	int32_t width, height;
	int32_t yStride, uvStride;
	uint8_t *yPixel, *uPixel, *vPixel;
	int32_t yLen, uLen, vLen;
	int32_t yPixelStride, uvPixelStride;
	int32_t orientation = android_camera2_capture_get_orientation(d);

	AImage_getWidth(image, &width);
	AImage_getHeight(image, &height);
	if (orientation % 180 != 0) {
		int32_t tmp = width;
		width = height;
		height = tmp;
	}

	AImage_getPlaneRowStride(image, 0, &yStride);
	AImage_getPlaneRowStride(image, 1, &uvStride);
	AImage_getPlaneData(image, 0, &yPixel, &yLen);
	AImage_getPlaneData(image, 1, &uPixel, &uLen);
	AImage_getPlaneData(image, 2, &vPixel, &vLen);
	AImage_getPlanePixelStride(image, 0, &yPixelStride);
	AImage_getPlanePixelStride(image, 1, &uvPixelStride);

	//ms_message("[Camera2 Capture] Image %p size %d/%d, y is %p, u is %p, v is %p, ystride %d, uvstride %d, ypixelstride %d, uvpixelstride %d", image, width, height, yPixel, uPixel, vPixel, yStride, uvStride, yPixelStride, uvPixelStride);

	mblk_t* yuv_block = nullptr;
	if (uvPixelStride == 1) {
		yuv_block = copy_yuv_with_rotation(d->bufAllocator, yPixel, uPixel, vPixel, orientation, width, height, yStride, uvStride, uvStride);
	} else {
		yuv_block = copy_ycbcrbiplanar_to_true_yuv_with_rotation_and_down_scale_by_2(d->bufAllocator,
	 		yPixel, vPixel < uPixel ? vPixel : uPixel, orientation, width, height, yStride, uvStride, uPixel < vPixel, false);
	}
	return yuv_block;
}

static void android_camera2_capture_on_image_available(void *context, AImageReader *reader) {
	AndroidCamera2Context *d = static_cast<AndroidCamera2Context *>(context);
252 253 254 255
	if (d == nullptr || d->filter == nullptr) {
		ms_error("[Camera2 Capture] Image available callback called after filtre uninit!");
		return;
	}
256

Sylvain Berfini's avatar
Sylvain Berfini committed
257 258 259 260 261 262
	ms_filter_lock(d->filter);
	if (!d->filter || !d->filter->ticker || !d->configured) {
		AImage *image = nullptr;
		media_status_t status = AImageReader_acquireLatestImage(d->imageReader, &image);
		if (status == AMEDIA_OK) AImage_delete(image);
		ms_filter_unlock(d->filter);
Sylvain Berfini's avatar
Sylvain Berfini committed
263 264
		return;
	}
Sylvain Berfini's avatar
Sylvain Berfini committed
265
	ms_filter_unlock(d->filter);
Sylvain Berfini's avatar
Sylvain Berfini committed
266

267 268 269 270
	int32_t format;
  	media_status_t status = AImageReader_getFormat(reader, &format);
	if (format == d->captureFormat) {
		AImage *image = nullptr;
271 272 273
		// Using AImageReader_acquireLatestImage will lead to the following log if maxImages given to ImageReader is 1
		// W/NdkImageReader: Unable to acquire a lockedBuffer, very likely client tries to lock more than maxImages buffers
		status = AImageReader_acquireNextImage(reader, &image);
274
		if (status == AMEDIA_OK) {
Sylvain Berfini's avatar
Sylvain Berfini committed
275 276 277 278 279 280 281
			if (ms_video_capture_new_frame(&d->fpsControl, d->filter->ticker->time)) {
				mblk_t *m = android_camera2_capture_image_to_mblkt(d, image);
				if (m) {
					ms_mutex_lock(&d->mutex);
					if (d->frame) freemsg(d->frame);
					d->frame = m;
					ms_mutex_unlock(&d->mutex);
282 283 284 285 286
				}
			}
			
			AImage_delete(image);
		} else {
287
			ms_error("[Camera2 Capture] Couldn't acquire image, error is %i", status);
288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324
		}
	} else {
		ms_error("[Camera2 Capture] Aquired image is in wrong format %d, expected %d", format, d->captureFormat);
	}
}

/* ************************************************************************* */

static void android_camera2_capture_create_preview(AndroidCamera2Context *d) {
    ms_message("[Camera2 Capture] Creating preview");
   	JNIEnv *jenv = ms_get_jni_env();

	if (!d->surface) {
		ms_error("[Camera2 Capture] Can't create preview window, no surface");
		return;
	}

	d->nativeWindow = ANativeWindow_fromSurface(jenv, d->surface);
}

static void android_camera2_capture_destroy_preview(AndroidCamera2Context *d) {
    ms_message("[Camera2 Capture] Destroying preview");
	if (d->nativeWindow) {
		ANativeWindow_release(d->nativeWindow);
		d->nativeWindow = nullptr;
    	ms_message("[Camera2 Capture] Preview window destroyed");
	}
	if (d->surface) {
		JNIEnv *env = ms_get_jni_env();
		env->DeleteGlobalRef(d->surface);
		d->surface = nullptr;
    	ms_message("[Camera2 Capture] Preview surface destroyed");
	}
}

static void android_camera2_capture_open_camera(AndroidCamera2Context *d) {
	ms_message("[Camera2 Capture] Opening camera");
325
	d->deviceStateCallbacks.context = d;
326
	d->deviceStateCallbacks.onDisconnected = android_camera2_capture_device_on_disconnected;
327
	d->deviceStateCallbacks.onError = android_camera2_capture_device_on_error;
328 329 330 331 332 333 334

	if (!d->device) {
		ms_error("[Camera2 Capture] Can't open camera, no device selected");
		return;
	}

	ACameraDevice *cameraDevice;
335
	camera_status_t camera_status = ACameraManager_openCamera(d->cameraManager, d->device->camId, &d->deviceStateCallbacks, &cameraDevice);
336
	if (camera_status != ACAMERA_OK) {
337
		ms_error("[Camera2 Capture] Failed to open camera %s, error is %s", d->device->camId, android_camera2_status_to_string(camera_status));
338 339
		return;
    }
340 341
	
	ms_message("[Camera2 Capture] Camera %s opened", d->device->camId);
342 343 344 345 346 347 348 349 350
	d->cameraDevice = cameraDevice;
}

static void android_camera2_capture_close_camera(AndroidCamera2Context *d) {
	ms_message("[Camera2 Capture] Closing camera");

    if (d->cameraDevice) {
        camera_status_t camera_status = ACameraDevice_close(d->cameraDevice);
        if (camera_status != ACAMERA_OK) {
351
            ms_error("[Camera2 Capture] Failed to close camera %s, error is %s", d->device->camId, android_camera2_status_to_string(camera_status));
352 353 354 355 356 357 358
        } else {
			ms_message("[Camera2 Capture] Camera closed %s", d->device->camId);
		}
        d->cameraDevice = nullptr;
    }
}

Sylvain Berfini's avatar
Sylvain Berfini committed
359
static void android_camera2_check_configuration_ok(AndroidCamera2Context *d) {
360
	if (d->nativeWindowId == 0) {
Sylvain Berfini's avatar
Sylvain Berfini committed
361
		ms_error("[Camera2 Capture] TextureView wasn't set (was core.setNativePreviewWindowId() called?)");
362 363 364 365 366 367 368
		return;
	}
	if (d->surface == nullptr) {
		ms_error("[Camera2 Capture] Failed to get a valid object to display camera preview from native window id [%p]", d->nativeWindowId);
		return;
	}
	if (d->captureSize.width == 0 || d->captureSize.height == 0) {
Sylvain Berfini's avatar
Sylvain Berfini committed
369
		ms_error("[Camera2 Capture] Capture size hasn't been configured yet");
370 371
		return;
	}
Sylvain Berfini's avatar
Sylvain Berfini committed
372 373 374 375 376 377 378
	d->configured = true;
}

static void android_camera2_capture_start(AndroidCamera2Context *d) {
	ms_message("[Camera2 Capture] Starting capture");
	camera_status_t camera_status = ACAMERA_OK;

379 380 381 382
	if (d->capturing) {
		ms_warning("[Camera2 Capture] Capture was already started, ignoring...");
		return;
	}
Sylvain Berfini's avatar
Sylvain Berfini committed
383 384 385 386
	if (!d->configured) {
		ms_warning("[Camera2 Capture] Filter configuration not finished, ignoring...");
		return;
	}
387 388 389 390
	
	if (!d->nativeWindow && d->surface) {
		android_camera2_capture_create_preview(d);
	}
391 392 393
	if (!d->cameraDevice) {
		android_camera2_capture_open_camera(d);
	}
394

395 396 397 398 399
	if (!d->cameraDevice) {
		ms_error("[Camera2 Capture] Couldn't open camera %s, aborting capture",  d->device->camId);
		return;
	}

400 401
	camera_status = ACaptureSessionOutputContainer_create(&d->captureSessionOutputContainer);
	if (camera_status != ACAMERA_OK) {
402
		ms_error("[Camera2 Capture] Failed to create capture session output container, error is %s", android_camera2_status_to_string(camera_status));
403
	}
404 405 406 407
	d->captureSessionStateCallbacks.onReady = android_camera2_capture_session_on_ready;
	d->captureSessionStateCallbacks.onActive = android_camera2_capture_session_on_active;
	d->captureSessionStateCallbacks.onClosed = android_camera2_capture_session_on_closed;

408
	camera_status = ACameraDevice_createCaptureRequest(d->cameraDevice, TEMPLATE_PREVIEW, &d->capturePreviewRequest);
409
	if (camera_status != ACAMERA_OK) {
410
		ms_error("[Camera2 Capture] Failed to create capture preview request, error is %s", android_camera2_status_to_string(camera_status));
411 412
	}

413 414
	camera_status = ACameraOutputTarget_create(d->nativeWindow, &d->cameraPreviewOutputTarget);
	if (camera_status != ACAMERA_OK) {
415
		ms_error("[Camera2 Capture] Failed to create output target, error is %s", android_camera2_status_to_string(camera_status));
416
	}
417

418 419
	camera_status = ACaptureRequest_addTarget(d->capturePreviewRequest, d->cameraPreviewOutputTarget);
	if (camera_status != ACAMERA_OK) {
420
		ms_error("[Camera2 Capture] Failed to add output target to capture request, error is %s", android_camera2_status_to_string(camera_status));
421
	}
422

423 424
	camera_status = ACaptureSessionOutput_create(d->nativeWindow, &d->sessionPreviewOutput);
	if (camera_status != ACAMERA_OK) {
425
		ms_error("[Camera2 Capture] Failed to create capture session output, error is %s", android_camera2_status_to_string(camera_status));
426
	}
427

428 429
	camera_status = ACaptureSessionOutputContainer_add(d->captureSessionOutputContainer, d->sessionPreviewOutput);
	if (camera_status != ACAMERA_OK) {
430
		ms_error("[Camera2 Capture] Failed to add capture session output to container, error is %s", android_camera2_status_to_string(camera_status));
431
	}
432

433 434
	media_status_t status = AImageReader_new(d->captureSize.width, d->captureSize.height, d->captureFormat, 1, &d->imageReader);
	if (status != AMEDIA_OK) {
435
		ms_error("[Camera2 Capture] Failed to create image reader, error is %i", status);
436 437
		return;
	}
438
	ms_message("[Camera2 Capture] Created image reader for size %ix%i and format %d", d->captureSize.width, d->captureSize.height, d->captureFormat);
439

Sylvain Berfini's avatar
Sylvain Berfini committed
440 441 442 443
	d->imageReaderListener = new AImageReader_ImageListener();
	d->imageReaderListener->context = d;
	d->imageReaderListener->onImageAvailable = android_camera2_capture_on_image_available;
  	status = AImageReader_setImageListener(d->imageReader, d->imageReaderListener);
444 445 446 447
	if (status != AMEDIA_OK) {
		ms_error("[Camera2 Capture] Failed to set image listener, error is %i", status);
		return;
	}
448

449
	status = AImageReader_getWindow(d->imageReader, &d->captureWindow);
450
	if (status != AMEDIA_OK) {
451
		ms_error("[Camera2 Capture] Capture window couldn't be acquired, error is %i", status);
452 453 454 455
		return;
	}
	ANativeWindow_acquire(d->captureWindow);

456 457
	camera_status = ACameraOutputTarget_create(d->captureWindow, &d->cameraCaptureOutputTarget);
	if (camera_status != ACAMERA_OK) {
458
		ms_error("[Camera2 Capture] Couldn't create output target, error is %s", android_camera2_status_to_string(camera_status));
459 460 461
		return;
	}

462
	camera_status = ACaptureRequest_addTarget(d->capturePreviewRequest, d->cameraCaptureOutputTarget);
463
	if (camera_status != ACAMERA_OK) {
464
		ms_error("[Camera2 Capture] Couldn't add output target to capture request, error is %s", android_camera2_status_to_string(camera_status));
465 466 467 468 469
		return;
	}

	camera_status = ACaptureSessionOutput_create(d->captureWindow, &d->sessionCaptureOutput);
	if (camera_status != ACAMERA_OK) {
470
		ms_error("[Camera2 Capture] Couldn't create capture session output, error is %s", android_camera2_status_to_string(camera_status));
471
		return;
472
	}
473

474 475
	camera_status = ACaptureSessionOutputContainer_add(d->captureSessionOutputContainer, d->sessionCaptureOutput);
	if (camera_status != ACAMERA_OK) {
476
		ms_error("[Camera2 Capture] Couldn't add capture session output to container, error is %s", android_camera2_status_to_string(camera_status));
477 478 479 480 481
		return;
	}

	camera_status = ACameraDevice_createCaptureSession(d->cameraDevice, d->captureSessionOutputContainer, &d->captureSessionStateCallbacks, &d->captureSession);
	if (camera_status != ACAMERA_OK) {
482
		ms_error("[Camera2 Capture] Couldn't create capture session, error is %s", android_camera2_status_to_string(camera_status));
483 484 485
		return;
	}

486
	camera_status = ACameraCaptureSession_setRepeatingRequest(d->captureSession, NULL, 1, &d->capturePreviewRequest, NULL);
487
	if (camera_status != ACAMERA_OK) {
488
		ms_error("[Camera2 Capture] Couldn't set capture session repeating request, error is %s", android_camera2_status_to_string(camera_status));
489 490
		return;
	}
491 492 493 494 495 496 497

	d->capturing = true;
	ms_message("[Camera2 Capture] Capture started");
}

static void android_camera2_capture_stop(AndroidCamera2Context *d) {
	ms_message("[Camera2 Capture] Stopping capture");
Sylvain Berfini's avatar
Sylvain Berfini committed
498

Sylvain Berfini's avatar
Sylvain Berfini committed
499 500 501 502
	ms_filter_lock(d->filter);
	d->configured = false;
	ms_filter_unlock(d->filter);

503 504
	if (!d->capturing) {
		ms_warning("[Camera2 Capture] Capture was already stopped, ignoring...");
Sylvain Berfini's avatar
Sylvain Berfini committed
505
		ms_filter_unlock(d->filter);
506 507 508 509
		return;
	}
	d->capturing = false;

510 511 512 513 514 515 516 517 518 519
	if (d->imageReaderListener) {
		delete d->imageReaderListener;
		d->imageReaderListener = nullptr;
	}
	AImageReader_ImageListener nullListener = {nullptr, nullptr};
  	media_status_t status = AImageReader_setImageListener(d->imageReader, &nullListener);
	if (status != AMEDIA_OK) {
		ms_error("[Camera2 Capture] Failed to set null image listener, error is %i", status);
	}

520 521 522
	// Seems to provoke an ANR on some devices if the camera close is done after the capture session
	android_camera2_capture_close_camera(d);

523
	if (d->captureSession) {
Sylvain Berfini's avatar
Sylvain Berfini committed
524 525
		camera_status_t camera_status = ACameraCaptureSession_abortCaptures(d->captureSession);
		if (camera_status != ACAMERA_OK) {
526
			ms_error("[Camera2 Capture] Couldn't abort captures on session, error is %s", android_camera2_status_to_string(camera_status));
Sylvain Berfini's avatar
Sylvain Berfini committed
527 528 529 530
		}

		camera_status = ACameraCaptureSession_stopRepeating(d->captureSession);
		if (camera_status != ACAMERA_OK) {
531
			ms_error("[Camera2 Capture] Couldn't stop repeating session, error is %s", android_camera2_status_to_string(camera_status));
Sylvain Berfini's avatar
Sylvain Berfini committed
532 533
		}

534 535 536 537
		ACameraCaptureSession_close(d->captureSession);
		d->captureSession = nullptr;
	}

538 539 540 541 542
	if (d->capturePreviewRequest) {
		ACaptureRequest_removeTarget(d->capturePreviewRequest, d->cameraCaptureOutputTarget);
		ACaptureRequest_free(d->capturePreviewRequest);
		d->capturePreviewRequest = nullptr;
    }
543

544 545 546 547 548 549 550 551
	if (d->cameraCaptureOutputTarget) {
		ACameraOutputTarget_free(d->cameraCaptureOutputTarget);
		d->cameraCaptureOutputTarget = nullptr;
    }

	if (d->cameraPreviewOutputTarget) {
		ACameraOutputTarget_free(d->cameraPreviewOutputTarget);
		d->cameraPreviewOutputTarget = nullptr;
552 553 554
    }

	if (d->captureSessionOutputContainer) {
Sylvain Berfini's avatar
Sylvain Berfini committed
555 556 557 558 559
		if (d->sessionCaptureOutput) {
			ACaptureSessionOutputContainer_remove(d->captureSessionOutputContainer, d->sessionCaptureOutput);
			ACaptureSessionOutput_free(d->sessionCaptureOutput);
			d->sessionCaptureOutput = nullptr;
		}
560

Sylvain Berfini's avatar
Sylvain Berfini committed
561 562 563 564 565
		if (d->sessionPreviewOutput) {
			ACaptureSessionOutputContainer_remove(d->captureSessionOutputContainer, d->sessionPreviewOutput);
			ACaptureSessionOutput_free(d->sessionPreviewOutput);
			d->sessionPreviewOutput = nullptr;
		}
566

567 568 569 570 571
		if (d->captureWindow) {
			ANativeWindow_release(d->captureWindow);
			d->captureWindow = nullptr;
		}

Sylvain Berfini's avatar
Sylvain Berfini committed
572 573 574
		ACaptureSessionOutputContainer_free(d->captureSessionOutputContainer);
		d->captureSessionOutputContainer = nullptr;
	}
575

576 577 578
	if (d->imageReader) {
		AImageReader_delete(d->imageReader);
		d->imageReader = nullptr;
579 580
	}

Sylvain Berfini's avatar
Sylvain Berfini committed
581 582
	android_camera2_capture_destroy_preview(d);

583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600
	ms_message("[Camera2 Capture] Capture stopped");
}

/* ************************************************************************* */

static void android_camera2_capture_init(MSFilter *f) {
	ms_message("[Camera2 Capture] Filter init");
	AndroidCamera2Context* d = new AndroidCamera2Context(f);
	f->data = d;
}

static void android_camera2_capture_preprocess(MSFilter *f) {
	ms_message("[Camera2 Capture] Filter preprocess");
	AndroidCamera2Context *d = (AndroidCamera2Context *)f->data;

	ms_filter_lock(f);
	ms_video_init_framerate_controller(&d->fpsControl, d->fps);
	ms_video_init_average_fps(&d->averageFps, d->fps_context);
Sylvain Berfini's avatar
Sylvain Berfini committed
601
	ms_filter_unlock(f);
602

Sylvain Berfini's avatar
Sylvain Berfini committed
603
	ms_mutex_lock(&d->mutex);
604 605 606 607
	if (d->frame) {
		freemsg(d->frame);
		d->frame = NULL;
	}
Sylvain Berfini's avatar
Sylvain Berfini committed
608
	ms_mutex_unlock(&d->mutex);
609 610 611 612 613 614
}

static void android_camera2_capture_process(MSFilter *f) {
	AndroidCamera2Context *d = (AndroidCamera2Context *)f->data;
	ms_filter_lock(f);

Sylvain Berfini's avatar
Sylvain Berfini committed
615
	if (!d->capturing && d->configured) {
616 617 618
		android_camera2_capture_start(d);
	}

619 620 621 622 623 624 625 626 627 628 629 630 631 632 633
	ms_mutex_lock(&d->mutex);
	if (d->frame) {
		ms_video_update_average_fps(&d->averageFps, f->ticker->time);
		mblk_set_timestamp_info(d->frame, f->ticker->time * 90);
		ms_queue_put(f->outputs[0], d->frame);
		d->frame = nullptr;
	}
	ms_mutex_unlock(&d->mutex);

	ms_filter_unlock(f);
}

static void android_camera2_capture_postprocess(MSFilter *f) {
	ms_message("[Camera2 Capture] Filter postprocess");
	AndroidCamera2Context *d = (AndroidCamera2Context *)f->data;
634 635 636 637 638
}

static void android_camera2_capture_uninit(MSFilter *f) {
	ms_message("[Camera2 Capture] Filter uninit");
	AndroidCamera2Context *d = (AndroidCamera2Context *)f->data;
Sylvain Berfini's avatar
Sylvain Berfini committed
639

640 641 642
	if (d->capturing) {
		android_camera2_capture_stop(d);
	}
643

Sylvain Berfini's avatar
Sylvain Berfini committed
644
	ms_mutex_lock(&d->mutex);
645 646 647 648
	if (d->frame) {
		freemsg(d->frame);
		d->frame = NULL;
	}
Sylvain Berfini's avatar
Sylvain Berfini committed
649
	ms_mutex_unlock(&d->mutex);
650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678

	ms_filter_lock(f);
	delete d;
	ms_filter_unlock(f);
}

/* ************************************************************************* */

static int android_camera2_capture_set_fps(MSFilter *f, void *arg) {
	AndroidCamera2Context *d = (AndroidCamera2Context *)f->data;
	d->fps = *((float*)arg);
	snprintf(d->fps_context, sizeof(d->fps_context), "Captured mean fps=%%f, expected=%f", d->fps);
	ms_filter_lock(f);
	ms_video_init_framerate_controller(&d->fpsControl, d->fps);
	ms_video_init_average_fps(&d->averageFps, d->fps_context);
	ms_filter_unlock(f);
	return 0;
}

static int android_camera2_capture_get_fps(MSFilter *f, void *arg) {
	AndroidCamera2Context *d = (AndroidCamera2Context *)f->data;
	*((float*)arg) = ms_average_fps_get(&d->averageFps);
	return 0;
}

static void android_camera2_capture_choose_best_configurations(AndroidCamera2Context *d) {
	ms_message("[Camera2 Capture] Listing camera %s configurations", d->device->camId);

    ACameraMetadata *cameraMetadata = nullptr;
679
	camera_status_t camera_status = ACameraManager_getCameraCharacteristics(d->cameraManager, d->device->camId, &cameraMetadata);
680
	if (camera_status != ACAMERA_OK) {
681
		ms_error("[Camera2 Capture] Failed to get camera characteristics, error is %s", android_camera2_status_to_string(camera_status));
682 683
		return;
	}
684

685 686 687 688 689 690 691 692
	ACameraMetadata_const_entry supportedFpsRanges;
	ACameraMetadata_getConstEntry(cameraMetadata, ACAMERA_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES, &supportedFpsRanges);
	for (int i = 0; i < supportedFpsRanges.count; i += 2) {
		int32_t min = supportedFpsRanges.data.i32[i];
		int32_t max = supportedFpsRanges.data.i32[i + 1];
		ms_message("[Camera2 Capture] Supported FPS range: [%d-%d]", min, max);
	}
	
693 694 695 696 697 698 699 700 701 702 703 704 705
	ACameraMetadata_const_entry scaler;
	ACameraMetadata_getConstEntry(cameraMetadata, ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS, &scaler);
	
	MSVideoSize backupSize;
	backupSize.width = 0;
	backupSize.height = 0;
	double askedRatio = d->captureSize.width * d->captureSize.height;
	bool found = false;
	
	for (int i = 0; i < scaler.count; i += 4) {
		int32_t input = scaler.data.i32[i + 3];
		int32_t format = scaler.data.i32[i + 0];
		if (input) continue;
Sylvain Berfini's avatar
Sylvain Berfini committed
706

707 708 709 710 711
		if (format == d->captureFormat) {
			int32_t width = scaler.data.i32[i + 1];
			int32_t height = scaler.data.i32[i + 2];
			double currentSizeRatio = width * height;
			ms_message("[Camera2 Capture] Available size width %d, height %d for requested format %d", width, height, format);
Sylvain Berfini's avatar
Sylvain Berfini committed
712

713 714 715 716 717 718 719 720 721 722
			if (width == d->captureSize.width && height == d->captureSize.height) {
				found = true;
			} else {
				double backupRatio = backupSize.width * backupSize.height;
				if (backupRatio == 0 || fabs(askedRatio - currentSizeRatio) < fabs(askedRatio - backupRatio)) {
					// Current resolution is closer to the one we want than the one in backup, update backup
					backupSize.width = width;
					backupSize.height = height;
				}
			}
723 724 725 726
		} else {
			int32_t width = scaler.data.i32[i + 1];
			int32_t height = scaler.data.i32[i + 2];
			ms_debug("[Camera2 Capture] Available size width %d, height %d for format %d", width, height, format);
727 728
		}
	}
Sylvain Berfini's avatar
Sylvain Berfini committed
729

730 731 732 733 734 735 736 737
	if (found) {
		ms_message("[Camera2 Capture] Found exact match for our required size of %ix%i", d->captureSize.width, d->captureSize.height);
	} else {
		// Asked resolution not found
		d->captureSize.width = backupSize.width;
		d->captureSize.height = backupSize.height;
		ms_warning("[Camera2 Capture] Couldn't find requested resolution, instead using %ix%i", backupSize.width, backupSize.height);
	}
738 739

	ACameraMetadata_free(cameraMetadata);
740
}
Sylvain Berfini's avatar
Sylvain Berfini committed
741

742 743
static void android_camera2_capture_create_surface_from_surface_texture(AndroidCamera2Context *d) {
	JNIEnv *env = ms_get_jni_env();
744 745 746 747 748 749 750 751 752 753 754
	jobject surface = nullptr;
	jobject surfaceTexture = d->nativeWindowId;

	jclass surfaceTextureClass = env->FindClass("android/graphics/SurfaceTexture");
	if (!surfaceTextureClass) {
		ms_error("[Camera2 Capture] Could not find android.graphics.SurfaceTexture class");
		return;
	}

	jclass surfaceClass = env->FindClass("android/view/Surface");
	if (!surfaceClass) {
755 756 757 758
		ms_error("[Camera2 Capture] Could not find android.view.Surface class");
		return;
	}

759 760 761 762 763 764 765 766 767 768 769 770 771
	jclass textureViewClass = env->FindClass("android/view/TextureView");
	if (!textureViewClass) {
		ms_error("[Camera2 Capture] Could not find android.view.TextureView class");
		return;
	}

	if (env->IsInstanceOf(surfaceTexture, surfaceClass)) {
		ms_message("[Camera2 Capture] NativePreviewWindowId %p is a Surface, using it directly", surfaceTexture);
		d->surface = (jobject)env->NewGlobalRef(surfaceTexture);
		return;
	}

	if (env->IsInstanceOf(surfaceTexture, textureViewClass)) {
772 773
		ms_message("[Camera2 Capture] NativePreviewWindowId %p is a TextureView", surfaceTexture);

774 775 776 777 778 779
		jmethodID getSurfaceTexture = env->GetMethodID(textureViewClass, "getSurfaceTexture", "()Landroid/graphics/SurfaceTexture;");
		surfaceTexture = env->CallObjectMethod(d->nativeWindowId, getSurfaceTexture);
		if (surfaceTexture == nullptr) {
			ms_error("[Camera2 Capture] TextureView isn't available !");
			return;
		}
780 781 782 783 784 785 786 787 788 789 790 791 792
		ms_message("[Camera2 Capture] Got SurfaceTexture %p from TextureView %p", surfaceTexture, d->nativeWindowId);
	}

	if (surfaceTexture != nullptr) {
		if (d->captureSize.width != 0 && d->captureSize.height != 0) {
			jmethodID setDefaultBufferSize = env->GetMethodID(surfaceTextureClass, "setDefaultBufferSize", "(II)V");
			env->CallVoidMethod(surfaceTexture, setDefaultBufferSize, d->captureSize.width, d->captureSize.height);
			ms_message("[Camera2 Capture] Set default buffer size for SurfaceTexture %p to %ix%i", surfaceTexture, d->captureSize.width, d->captureSize.height);
		} else {
			ms_warning("[Camera2 Capture] SurfaceTexture buffer size not available yet, aborting for now, will come back later");
			d->surface = nullptr;
			return;
		}
793 794
	}

795 796 797 798 799 800
	if (surfaceTexture == nullptr) {
		ms_error("[Camera2 Capture] SurfaceTexture is null, can't create a Surface!");
		return;
	}

	ms_message("[Camera2 Capture] Creating Surface from SurfaceTexture %p", surfaceTexture);
801 802
	jmethodID ctor = env->GetMethodID(surfaceClass, "<init>", "(Landroid/graphics/SurfaceTexture;)V");
	surface = env->NewObject(surfaceClass, ctor, surfaceTexture);
803 804 805 806
	if (!surface) {
		ms_error("[Camera2 Capture] Could not instanciate android.view.Surface object");
		return;
	}
807
	
808
	d->surface = (jobject)env->NewGlobalRef(surface);
809
	ms_message("[Camera2 Capture] Got Surface %p from SurfaceTexture %p",  d->surface, surfaceTexture);
810 811 812 813 814
}

static int android_camera2_capture_set_surface_texture(MSFilter *f, void *arg) {
	AndroidCamera2Context *d = (AndroidCamera2Context *)f->data;
	unsigned long id = *(unsigned long *)arg;
815
	jobject nativeWindowId = (jobject)id;
Sylvain Berfini's avatar
Sylvain Berfini committed
816
	JNIEnv *env = ms_get_jni_env();
817

Sylvain Berfini's avatar
Sylvain Berfini committed
818
	ms_filter_lock(f);
Sylvain Berfini's avatar
Sylvain Berfini committed
819 820
	jobject currentWindowId = d->nativeWindowId;
	ms_filter_unlock(f);
Sylvain Berfini's avatar
Sylvain Berfini committed
821

Sylvain Berfini's avatar
Sylvain Berfini committed
822
	ms_message("[Camera2 Capture] New native window ptr is %p, current one is %p", nativeWindowId, currentWindowId);
Sylvain Berfini's avatar
Sylvain Berfini committed
823
	if (id == 0) {
Sylvain Berfini's avatar
Sylvain Berfini committed
824
		if (currentWindowId) {
Sylvain Berfini's avatar
Sylvain Berfini committed
825
			android_camera2_capture_stop(d);
Sylvain Berfini's avatar
Sylvain Berfini committed
826
			env->DeleteGlobalRef(currentWindowId);
Sylvain Berfini's avatar
Sylvain Berfini committed
827 828
			d->nativeWindowId = nullptr;
		}
Sylvain Berfini's avatar
Sylvain Berfini committed
829 830
	} else if (!env->IsSameObject(currentWindowId, nativeWindowId)) {
		if (currentWindowId) {
Sylvain Berfini's avatar
Sylvain Berfini committed
831
			android_camera2_capture_stop(d);
Sylvain Berfini's avatar
Sylvain Berfini committed
832
			env->DeleteGlobalRef(currentWindowId);
Sylvain Berfini's avatar
Sylvain Berfini committed
833
		}
Sylvain Berfini's avatar
Sylvain Berfini committed
834
		
Sylvain Berfini's avatar
Sylvain Berfini committed
835
		d->nativeWindowId = env->NewGlobalRef(nativeWindowId);
836
		android_camera2_capture_create_surface_from_surface_texture(d);
Sylvain Berfini's avatar
Sylvain Berfini committed
837 838 839 840
		
		ms_filter_lock(f);
		android_camera2_check_configuration_ok(d);
		ms_filter_unlock(f);
841 842 843 844

		if (d->previewSize.width != 0 && d->previewSize.height != 0) {
			ms_filter_notify(f, MS_CAMERA_PREVIEW_SIZE_CHANGED, &d->previewSize);
		}
845 846
	} else {
		ms_message("[Camera2 Capture] New native window is the same as the current one, skipping...");
847
	}
Sylvain Berfini's avatar
Sylvain Berfini committed
848

849 850 851
	return 0; 
}

852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875
static int android_camera2_capture_set_vsize(MSFilter *f, void* arg) {
	AndroidCamera2Context *d = (AndroidCamera2Context *)f->data;

	MSVideoSize requestedSize = *(MSVideoSize*)arg;
	if (d->captureSize.width == requestedSize.width && d->captureSize.height == requestedSize.height) {
		return -1;
	}
	
	MSVideoSize oldSize;
	oldSize.width = d->captureSize.width;
	oldSize.height = d->captureSize.height;
	d->captureSize = requestedSize;

	android_camera2_capture_stop(d);
	android_camera2_capture_choose_best_configurations(d);

	int orientation = android_camera2_capture_get_orientation(d);
	if (orientation % 180 == 0) {
		d->previewSize.width = d->captureSize.width;
		d->previewSize.height = d->captureSize.height;
	} else {
		d->previewSize.width = d->captureSize.height;
		d->previewSize.height = d->captureSize.width;
	}
876 877 878 879

	if (d->previewSize.width != 0 && d->previewSize.height != 0) {
		ms_filter_notify(f, MS_CAMERA_PREVIEW_SIZE_CHANGED, &d->previewSize);
	}
880 881 882 883

	ms_message("[Camera2 Capture] Previous preview size was %i/%i, new size is %i/%i", 
		oldSize.width, oldSize.height, d->previewSize.width, d->previewSize.height);

884 885
	if (d->surface == nullptr && d->nativeWindowId != 0) {
		ms_warning("[Camera2 Capture] Video size has changed after video window id has been set, have to recreate Surface object...");
886 887 888
		android_camera2_capture_create_surface_from_surface_texture(d);
	}

Sylvain Berfini's avatar
Sylvain Berfini committed
889 890
	ms_filter_lock(f);
	android_camera2_check_configuration_ok(d);
891
	ms_filter_unlock(f);
Sylvain Berfini's avatar
Sylvain Berfini committed
892

893 894 895 896 897 898
	return 0;
}

static int android_camera2_capture_get_vsize(MSFilter *f, void* arg) {
	AndroidCamera2Context *d = (AndroidCamera2Context *)f->data;

Sylvain Berfini's avatar
Sylvain Berfini committed
899
	ms_filter_lock(f);
900 901 902 903 904 905 906 907
	int orientation = android_camera2_capture_get_orientation(d);
	if (orientation % 180 == 0) {
		d->previewSize.width = d->captureSize.width;
		d->previewSize.height = d->captureSize.height;
	} else {
		d->previewSize.width = d->captureSize.height;
		d->previewSize.height = d->captureSize.width;
	}
Sylvain Berfini's avatar
Sylvain Berfini committed
908
	ms_filter_unlock(f);
909 910 911 912 913 914 915 916 917 918

	*(MSVideoSize*)arg = d->previewSize;
	ms_message("[Camera2 Capture] Getting preview size: %ix%i", d->previewSize.width, d->previewSize.height);

	return 0;
}

static int android_camera2_capture_set_device_rotation(MSFilter* f, void* arg) {
	AndroidCamera2Context *d = (AndroidCamera2Context *)f->data;
	ms_filter_lock(f);
Sylvain Berfini's avatar
Sylvain Berfini committed
919 920
	int rotation = *((int*)arg);
	d->rotation = rotation;
921
	ms_filter_unlock(f);
Sylvain Berfini's avatar
Sylvain Berfini committed
922 923

	ms_message("[Camera2 Capture] Device rotation is %i", rotation);
924 925 926
	return 0;
}

927 928 929 930 931 932
static int android_camera2_capture_get_pix_fmt(MSFilter *f, void *data){
	*(MSPixFmt*)data = MS_YUV420P;
	return 0;
}

/* ************************************************************************* */
Sylvain Berfini's avatar
Sylvain Berfini committed
933 934

static MSFilterMethod android_camera2_capture_methods[] = {
935 936 937 938 939 940 941 942
		{ MS_FILTER_SET_FPS, &android_camera2_capture_set_fps },
		{ MS_FILTER_GET_FPS, &android_camera2_capture_get_fps },
		{ MS_FILTER_SET_VIDEO_SIZE, &android_camera2_capture_set_vsize },
		{ MS_FILTER_GET_VIDEO_SIZE, &android_camera2_capture_get_vsize },
		{ MS_VIDEO_CAPTURE_SET_DEVICE_ORIENTATION, &android_camera2_capture_set_device_rotation },
		{ MS_VIDEO_DISPLAY_SET_NATIVE_WINDOW_ID, &android_camera2_capture_set_surface_texture },
		{ MS_FILTER_GET_PIX_FMT, &android_camera2_capture_get_pix_fmt },
		{ 0, 0 }
Sylvain Berfini's avatar
Sylvain Berfini committed
943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962
};

MSFilterDesc ms_android_camera2_capture_desc={
		MS_ANDROID_VIDEO_READ_ID,
		"MSAndroidCamera2Capture",
		"A filter that captures Android video using camera2 APIs.",
		MS_FILTER_OTHER,
		NULL,
		0,
		1,
		android_camera2_capture_init,
		android_camera2_capture_preprocess,
		android_camera2_capture_process,
		android_camera2_capture_postprocess,
		android_camera2_capture_uninit,
		android_camera2_capture_methods
};

static void android_camera2_capture_detect(MSWebCamManager *obj);

963
static void android_camera2_capture_cam_init(MSWebCam *cam) { }
Sylvain Berfini's avatar
Sylvain Berfini committed
964 965

static MSFilter *android_camera2_capture_create_reader(MSWebCam *obj) {
966 967
	ms_message("[Camera2 Capture] Creating filter for camera %s", obj->id);

Sylvain Berfini's avatar
Sylvain Berfini committed
968
	MSFilter* filter = ms_factory_create_filter_from_desc(ms_web_cam_get_factory(obj), &ms_android_camera2_capture_desc);
969 970 971
	AndroidCamera2Context *d = (AndroidCamera2Context *)filter->data;
	d->device = (AndroidCamera2Device *)obj->data;

Sylvain Berfini's avatar
Sylvain Berfini committed
972 973 974
	return filter;
}

975
MSWebCamDesc ms_android_camera2_capture_webcam_desc = {
Sylvain Berfini's avatar
Sylvain Berfini committed
976 977 978 979 980 981 982
		"AndroidCamera2Capture",
		&android_camera2_capture_detect,
		&android_camera2_capture_cam_init,
		&android_camera2_capture_create_reader,
		NULL
};

983 984
extern void android_video_capture_detect_cameras_legacy(MSWebCamManager *obj);

Sylvain Berfini's avatar
Sylvain Berfini committed
985 986 987
void android_camera2_capture_detect(MSWebCamManager *obj) {
	ms_message("[Camera2 Capture] Detecting cameras");

988
	ACameraIdList *cameraIdList = nullptr;
989
	ACameraMetadata *cameraMetadata = nullptr;
Sylvain Berfini's avatar
Sylvain Berfini committed
990 991

	camera_status_t camera_status = ACAMERA_OK;
992
	ACameraManager *cameraManager = ACameraManager_create();
Sylvain Berfini's avatar
Sylvain Berfini committed
993

994 995 996 997 998 999
	camera_status = ACameraManager_getCameraIdList(cameraManager, &cameraIdList);
	if (camera_status != ACAMERA_OK) {
		ms_error("[Camera2 Capture] Failed to get camera(s) list : %d", camera_status);
		android_video_capture_detect_cameras_legacy(obj);
		return;
	}
Sylvain Berfini's avatar
Sylvain Berfini committed
1000 1001

	if (cameraIdList->numCameras < 1) {
1002 1003 1004 1005
		ms_warning("[Camera2 Capture] No camera detected !");
		android_video_capture_detect_cameras_legacy(obj);
		return;
	}
Sylvain Berfini's avatar
Sylvain Berfini committed
1006

1007 1008 1009 1010
	bool front_facing_found = false;
	bool back_facing_found = false;

	const char *camId = nullptr;
Sylvain Berfini's avatar
Sylvain Berfini committed
1011
	for (int i = 0; i < cameraIdList->numCameras; i++) {
1012
		camId = cameraIdList->cameraIds[i];
Sylvain Berfini's avatar
Sylvain Berfini committed
1013 1014 1015 1016 1017

		camera_status = ACameraManager_getCameraCharacteristics(cameraManager, camId, &cameraMetadata);
		if (camera_status != ACAMERA_OK) {
			ms_error("[Camera2 Capture] Failed to get camera %s characteristics", camId);
		} else {
1018
			AndroidCamera2Device *device = new AndroidCamera2Device(ms_strdup(camId));
1019

Sylvain Berfini's avatar
Sylvain Berfini committed
1020
  			ACameraMetadata_const_entry orientation;
1021
			ACameraMetadata_getConstEntry(cameraMetadata, ACAMERA_SENSOR_ORIENTATION, &orientation);
Sylvain Berfini's avatar
Sylvain Berfini committed
1022
			int32_t angle = orientation.data.i32[0];
1023
			device->orientation = angle;
Sylvain Berfini's avatar
Sylvain Berfini committed
1024

1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
			ACameraMetadata_const_entry hardwareLevel;
			ACameraMetadata_getConstEntry(cameraMetadata, ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL, &hardwareLevel);
			std::string supportedHardwareLevel = "unknown";
			switch (hardwareLevel.data.u8[0]) {
				case ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL_LIMITED:
					supportedHardwareLevel = "limited";
					break;
				case ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL_FULL:
					supportedHardwareLevel = "full";
					break;
				case ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY:
					supportedHardwareLevel = "legacy";
					break;
				case ACAMERA_INFO_SUPPORTED_HARDWARE_LEVEL_3:
					supportedHardwareLevel = "3";
					break;
			}

Sylvain Berfini's avatar
Sylvain Berfini committed
1043
  			ACameraMetadata_const_entry face;
1044 1045
			ACameraMetadata_getConstEntry(cameraMetadata, ACAMERA_LENS_FACING, &face);
			bool back_facing = face.data.u8[0] == ACAMERA_LENS_FACING_BACK;
1046 1047
			std::string facing = std::string(!back_facing ? "front" : "back");
			ms_message("[Camera2 Capture] Camera %s is facing %s with angle %d, hardware level is %s", camId, facing.c_str(), angle, supportedHardwareLevel.c_str());
1048
			device->back_facing = back_facing;
Sylvain Berfini's avatar
Sylvain Berfini committed
1049 1050

			MSWebCam *cam = ms_web_cam_new(&ms_android_camera2_capture_webcam_desc);
1051 1052 1053
			std::string idstring = std::string(!back_facing ? "Front" : "Back") + std::string("FacingCamera");
			cam->id = ms_strdup(idstring.c_str());
			cam->name = ms_strdup(idstring.c_str());
1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
			cam->data = device;

			if ((back_facing && !back_facing_found) || (!back_facing && !front_facing_found)) {
				ms_web_cam_manager_prepend_cam(obj, cam);
				if (back_facing) {
					back_facing_found = true;
				} else {
					front_facing_found = true;
				}
			} else {
				ms_warning("[Camera2 Capture] A camera with the same direction as already been added, skipping this one");
			}
Sylvain Berfini's avatar
Sylvain Berfini committed
1066 1067 1068 1069 1070 1071

			ACameraMetadata_free(cameraMetadata);
		}
	}

	ACameraManager_deleteCameraIdList(cameraIdList);
1072
	ACameraManager_delete(cameraManager);
Sylvain Berfini's avatar
Sylvain Berfini committed
1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083
}

#ifdef _MSC_VER
#define MS_PLUGIN_DECLARE(type) extern "C" __declspec(dllexport) type
#else
#define MS_PLUGIN_DECLARE(type) extern "C" type
#endif

MS_PLUGIN_DECLARE(void) libmsandroidcamera2_init(MSFactory* factory) {
	ms_factory_register_filter(factory, &ms_android_camera2_capture_desc);
	ms_message("[Camera2 Capture] libmsandroidcamera2 plugin loaded");
1084 1085 1086

	MSWebCamManager *manager = ms_factory_get_web_cam_manager(factory);
	ms_web_cam_manager_register_desc(manager, &ms_android_camera2_capture_webcam_desc);
1087
}