前言
直播、短视频、在线会议等应用越来越多地进入人们的生活，随之诞生的是丰富的各类创意玩法与新鲜体验，其中大量应用了以AI检测和图形渲染为基础的AR技术。
而随着Web技术的不断成熟，AR技术在Web上的实现成为了一种可能。今天就总结了在Web端实现此功能的几个技术要点，跟大家一起探讨一下。
架构和概念
抽象整体的实现思路如下
调取Camera获得相机画面
使用tensorflow加载人脸识别模型生成FaceMesh
根据FaceMesh生成三角网格并进行UV贴图
FaceMesh
MediaPipe Face Mesh是一种脸部几何解决方案，即使在移动设备上，也可以实时估计468个3D脸部界标。它采用 机器学习 （ML）来推断3D表面几何形状，只需要单个摄像机输入，而无需专用的深度传感器。该解决方案利用轻量级的模型架构以及整个管线中的GPU加速，可提供对实时体验至关重要的实时性能。
UVMap
UV是二维纹理坐标，U代表水平方向，V代表垂直方向。UV Map用来描述三维物体表面与图像纹理(Texture) 的映射关系，有了UV Map，我们就可以将二维的图像纹理粘贴到三维的物体表面。
矩形贴图和球面的映射图
技术实现
调取Camera获得相机画面
通过navigator.mediaDevices.getUserMedia获取stream，放到video查看。
async function setupWebcam() {return new Promise( ( resolve, reject ) => {const webcamElement = document.getElementById( "webcam" );const navigatorAny = navigator;navigator.getUserMedia = navigator.getUserMedia ||navigatorAny.webkitGetUserMedia || navigatorAny.mozGetUserMedia ||navigatorAny.msGetUserMedia;if( navigator.getUserMedia ) {navigator.getUserMedia( { video: true },stream => {webcamElement.srcObject = stream;webcamElement.addEventListener( "loadeddata", resolve, false );},error => reject());}else {reject();}});
}
复制代码人脸识别
//创建模型
createModel() {return new Promise(async resolve => {await tf.setBackend('webgl')const model = faceLandmarksDetection.SupportedModels.MediaPipeFaceMesh;const detectorConfig = {maxFaces: 1, //检测到的最大面部数量refineLandmarks: true, //可以完善眼睛和嘴唇周围的地标坐标，并在虹膜周围输出其他地标runtime: 'mediapipe',solutionPath: 'https://unpkg.com/@mediapipe/face_mesh', //WASM二进制文件和模型文件所在的路径};this.model = await faceLandmarksDetection.createDetector(model, detectorConfig);resolve(this.model);})
},
//识别
async recognition() {try {const video = this.$refs.video;const faces = await this.model.estimateFaces(video, {flipHorizontal: false, //镜像});if (faces.length > 0) {const keypoints = faces[0].keypoints;this.render3D({scaledMesh:keypoints.reduce((acc, pos) =>{acc.push([pos.x,pos.y,pos.z])return acc}, [])});}else{this.render3D({scaledMesh:[]})}} catch (error) {console.log(error);}
}
复制代码3D场景贴图
- TRIANGULATION
- UV_COORDS
//3D场景const scene = new THREE.Scene();//添加一些光照scene.add( new THREE.AmbientLight( 0xcccccc, 0.4 ) );camera.add( new THREE.PointLight( 0xffffff, 0.8 ) );//正交相机scene camera = new THREE.PerspectiveCamera( 45, 1, 0.1, 2000 );camera.position.x = videoWidth / 2;camera.position.y = -videoHeight / 2;camera.position.z = -( videoHeight / 2 ) / Math.tan( 45 / 2 )scene.add( camera ); //渲染器const renderer = new THREE.WebGLRenderer({canvas: document.getElementById( "overlay" ),alpha: true});//创建geometry，将468个人脸特征点按照一定的顺序(TRIANGULATION)组成三角网格，并加载UV_COORDSconst geometry = new THREE.BufferGeometry()geometry.setIndex(TRIANGULATION)geometry.setAttribute('uv', new THREE.Float32BufferAttribute(UV_COORDS.map((item, index) => index % 2 ? item : 1 - item), 2))geometry.computeVertexNormals()//创建materialconst textureLoader = new THREE.TextureLoader();const meshImg = this.meshList[meshIndex].src;//材质图片地址textureLoader.load(meshImg,texture=>{texture.encoding = THREE.sRGBEncodingtexture.anisotropy = 16const material = new THREE.MeshBasicMaterial({map: texture,transparent: true,color: new THREE.Color(0xffffff),reflectivity: 0.5});const mesh = new THREE.Mesh(geometry, material)scene.add(mesh)})// 根据face mesh实时更新geometryupdateGeometry(prediction){let w = canvasWidth;let h = canvasWidth;const faceMesh = resolveMesh(prediction.scaledMesh, w, h)const positionBuffer = faceMesh.reduce((acc, pos) => acc.concat(pos), [])geometry.setAttribute('position', new THREE.Float32BufferAttribute(positionBuffer, 3))geometry.attributes.position.needsUpdate = true}resolveMesh(faceMesh, vw, vh){return faceMesh.map(p => [p[0] - vw / 2, vh / 2 - p[1], -p[2]])}//渲染render3D(prediction){if (prediction) {updateGeometry(prediction)}renderer.render(scene, threeCamera)}
复制代码加载3D模型
//加载3D模型
const loader = new GLTFLoader();
const Object3D = new THREE.Object3D();
loader.load(modelUrl, (gltf) => {const object = gltf.sceneconst box = new THREE.Box3().setFromObject(object)const size = box.getSize(new THREE.Vector3()).length()const center = box.getCenter(new THREE.Vector3())object.position.x += (object.position.x - center.x);object.position.y += (object.position.y - center.y + 1);object.position.z += (object.position.z - center.z - 15);Object3D.add(object)this.scene.add(Object3D)
})//计算Matrix
const position = prediction.midwayBetweenEyes[0]
const scale = this.getScale(prediction.scaledMesh, 234, 454)
const rotation = this.getRotation(prediction.scaledMesh, 10, 50, 280)
object.position.set(...position)
object.scale.setScalar(scale / 20)
object.scale.x *= -1
object.rotation.setFromRotationMatrix(rotation)
object.rotation.y = -object.rotation.y
object.rotateZ(Math.PI)
object.rotateX(-Math.PI * .05)
if (this.morphTarget) {// flippedthis.morphTarget['leftEye'] && this.morphTarget['leftEye'](1 - prediction.faceRig.eye.r)this.morphTarget['rightEye'] && this.morphTarget['rightEye'](1 - prediction.faceRig.eye.l)this.morphTarget['mouth'] && this.morphTarget['mouth'](prediction.faceRig.mouth.shape.A)
}
