Build TPS model from FRGC Spring 2003.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:0bac92bb64fed29a880c903ddd1b9fa826e287556b88abb25903e6b274ac8f99"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import os.path\n",
      "import numpy as np\n",
      "import matplotlib.pyplot as plt\n",
      "import cPickle\n",
      "import menpo.io as mio\n",
      "from menpo.image import MaskedImage, BooleanImage\n",
      "from menpo.landmark import labeller, ibug_face_65_closed_mouth\n",
      "from menpo.shape import PointCloud, TriMesh\n",
      "from menpo.transform import Translation, ThinPlateSplines, PiecewiseAffine\n",
      "from menpo.transform import GeneralizedProcrustesAnalysis\n",
      "from scipy.spatial import Delaunay"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def print_replace_line(string):\n",
      "    import sys\n",
      "    # Cheeky carriage return so we print on the same line\n",
      "    sys.stdout.write('\\r' + string)\n",
      "    sys.stdout.flush()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Load all the aligned shapes from the dataset"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "RECREATE_MESHES = True"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from sfs_io import load_frgc\n",
      "\n",
      "images = load_frgc('spring2003', RECREATE_MESHES, max_images=5, \n",
      "                   output_base_path='/home/pts08/Desktop')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def approximate_normals(image):\n",
      "    im = MaskedImage(image.pixels[..., 2], mask=image.mask)\n",
      "    g = im.gradient()\n",
      "\n",
      "    normals = np.concatenate([g.masked_pixels, \n",
      "                              np.ones([g.n_true_pixels, 1])], axis=1)\n",
      "    mag = np.sqrt(np.sum(normals ** 2, axis=1))\n",
      "    return np.clip(normals / mag[..., None], -1., 1.)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def extract_normals(images):\n",
      "    normals = np.zeros([len(images), images[0].n_true_pixels, 3])\n",
      "    for i, im in enumerate(images):\n",
      "        normals[i, ...] = approximate_normals(im)\n",
      "    return normals"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "def create_feature_space(feature_matrix, example_image, feature_space_name, output_path='/home/pts08/Desktop'):\n",
      "    feature_space_images = []\n",
      "    N = feature_matrix.shape[0]\n",
      "    for i, n in enumerate(feature_matrix):\n",
      "        new_im = MaskedImage.blank(example_image.shape, mask=example_image.mask, n_channels=n.shape[1])\n",
      "        new_im.from_vector_inplace(n.flatten())\n",
      "        new_im.landmarks = example_image.landmarks\n",
      "        feature_space_images.append(new_im)\n",
      "        print_replace_line('Image {0} of {1}'.format(i + 1, N))\n",
      "\n",
      "    cPickle.dump(images, open(os.path.join(output_path, 'frgc_spring2003_68_{0}.pkl'.format(feature_space_name)), 'wb'), protocol=2)\n",
      "    return feature_space_images"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Generate the frame of reference (the celebrity image)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ref_frame = mio.import_image('/home/pts08/Dropbox/phd/research/sfs/celebrities/clint_eastwood.png')\n",
      "labeller(ref_frame, 'PTS', ibug_face_65_closed_mouth)\n",
      "ref_frame.crop_to_landmarks_inplace(boundary=2, group='ibug_face_65_closed_mouth')\n",
      "ref_frame.constrain_mask_to_landmarks(group='ibug_face_65_closed_mouth')\n",
      "\n",
      "reference_shape = ref_frame.landmarks['ibug_face_65_closed_mouth'][None]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Pull out the landmarks\n",
      "shapes = []\n",
      "for img in images:\n",
      "    labeller(img, 'PTS', ibug_face_65_closed_mouth)\n",
      "    shapes.append(img.landmarks['ibug_face_65_closed_mouth'][None])"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from menpo.transform import GeneralizedProcrustesAnalysis\n",
      "\n",
      "# center them\n",
      "cent_shapes = [Translation(-s.centre).apply(s) for s in shapes]\n",
      "\n",
      "gpa = GeneralizedProcrustesAnalysis(cent_shapes)\n",
      "aligned_shapes = [p.aligned_source for p in gpa.transforms]\n",
      "\n",
      "aligned_points = [p.points for p in aligned_shapes]\n",
      "# Calculate the mean shape as the reference shape\n",
      "mean_shape = PointCloud(np.mean(aligned_points, axis=0))\n",
      "reference_shape = Translation(-mean_shape.bounds(boundary=2)[0]).apply(mean_shape)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "ref_frame = MaskedImage.blank(reference_shape.range(boundary=2))\n",
      "ref_frame.landmarks['ibug_face_65_closed_mouth'] = reference_shape\n",
      "ref_frame.constrain_mask_to_landmarks()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%matplotlib inline\n",
      "\n",
      "ref_frame.view()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "tps_transforms = [ThinPlateSplines(reference_shape, shape) for shape in shapes]\n",
      "warped_images = [img.warp_to(ref_frame.mask, t) for img, t in zip(images, tps_transforms)]\n",
      "\n",
      "for k, im in enumerate(warped_images):\n",
      "    new_im = im.copy()\n",
      "    new_im.pixels[new_im.pixels[..., 2] == 0, 2] = np.nanmin(new_im.pixels[..., 2])\n",
      "    warped_images[k] = new_im"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%matplotlib inline\n",
      "import matplotlib.pyplot as plt\n",
      "\n",
      "warped_images[0].view(channels=2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "example_image = warped_images[0]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Calculate the normal matrix for all the images"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "normal_matrix = extract_normals(warped_images)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from copy import deepcopy\n",
      "# Save memory by deleting all the images since we don't need them any more. \n",
      "# Keep one around that we can query for it's size etc\n",
      "example_image = deepcopy(warped_images[0])\n",
      "del warped_images[:]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Calculate the normals (for LS and Cosine)"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "normal_images = create_feature_space(normal_matrix, example_image, 'normals')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "%matplotlib inline\n",
      "normal_images[0].view(channels='all')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Calculate the Spherical feature space"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from cosine_normals import Spherical\n",
      "spherical_matrix = Spherical().logmap(normal_matrix)\n",
      "spherical_images = create_feature_space(spherical_matrix, example_image, 'spherical')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Calculate the AEP feature space"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from vector_utils import normalise_vector\n",
      "mean_normals = normalise_vector(np.mean(normal_matrix, 0))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from logmap_utils import partial_logmap\n",
      "from aep import AEP\n",
      "\n",
      "aep_matrix = AEP(mean_normals).logmap(normal_matrix)\n",
      "aep_images = create_feature_space(aep_matrix, example_image, 'aep')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Calculate the PGA feature space"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from pga import PGA, intrinsic_mean\n",
      "mu = intrinsic_mean(normal_matrix, PGA, max_iters=50)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pga_matrix = PGA(mu).logmap(normal_matrix)\n",
      "pga_images = create_feature_space(pga_matrix, example_image, 'pga')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "markdown",
     "metadata": {},
     "source": [
      "## Calculate the PCA for LS, Spherical, Cosine and AEP"
     ]
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "# Create the template image\n",
      "template = ref_frame"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "from menpo.model import PCAModel\n",
      "\n",
      "normal_model = PCAModel(normal_images, center=True)\n",
      "normal_model.trim_components(200)\n",
      "\n",
      "cosine_model = PCAModel(normal_images, center=False)\n",
      "normal_model.trim_components(200)\n",
      "\n",
      "spherical_model = PCAModel(spherical_images, center=False)\n",
      "normal_model.trim_components(200)\n",
      "\n",
      "aep_model = PCAModel(aep_images, center=False)\n",
      "normal_model.trim_components(200)\n",
      "\n",
      "pga_model = PCAModel(pga_images, center=False)\n",
      "normal_model.trim_components(200)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "mean_normals_image = normal_model.mean\n",
      "mu_image = mean_normals_image.from_vector(mu)\n",
      "\n",
      "with open('/vol/data/frgc_spring2003_sfs_normal_tps', 'wb') as f:\n",
      "    cPickle.dump({'appearance_model': normal_model,\n",
      "                 'template': template,\n",
      "                 'mean_normals': mean_normals_image},\n",
      "                 f, protocol=2)\n",
      "with open('/vol/data/frgc_spring2003_sfs_cosine_tps', 'wb') as f:\n",
      "    cPickle.dump({'appearance_model': cosine_model,\n",
      "                 'template': template,\n",
      "                 'mean_normals': mean_normals_image},\n",
      "                 f, protocol=2)\n",
      "with open('/vol/data/frgc_spring2003_sfs_spherical_tps', 'wb') as f:\n",
      "    cPickle.dump({'appearance_model': spherical_model,\n",
      "                 'template': template,\n",
      "                 'mean_normals': mean_normals_image},\n",
      "                 f, protocol=2)\n",
      "with open('/vol/data/frgc_spring2003_sfs_aep_tps', 'wb') as f:\n",
      "    cPickle.dump({'appearance_model': aep_model,\n",
      "                 'template': template,\n",
      "                 'mean_normals': mean_normals_image},\n",
      "                 f, protocol=2)\n",
      "with open('/vol/data/frgc_spring2003_sfs_pga_tps', 'wb') as f:\n",
      "    cPickle.dump({'appearance_model': pga_model,\n",
      "                 'template': template,\n",
      "                 'mean_normals': mean_normals_image,\n",
      "                 'intrinsic_mean_normals': mu_image},\n",
      "                 f, protocol=2)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}