{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pose-classifier-final.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jVZj7qrM1EdH",
        "colab_type": "text"
      },
      "source": [
        "### Pose Classifier Final\n",
        "\n",
        "This notebook creates the final pose classification using the best model we have observed."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HVrzi92TzlmU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import random\n",
        "import cv2\n",
        "from os import listdir\n",
        "from os.path import isfile, join\n",
        "import os\n",
        "import json\n",
        "import matplotlib.pyplot as plt\n",
        "import tensorflow as tf\n",
        "\n",
        "\n",
        "class PoseDataGenerator(tf.keras.utils.Sequence):\n",
        "    \"\"\"\n",
        "    Custom Keras generator for raw pose keypoint data.\n",
        "\n",
        "    Only body keypoints are extracted and normalized.\n",
        "    \"\"\"\n",
        "    def __init__(self, keyframe_dir, batch_size=32, frames_to_use=-1, is_test=False, shuffle=True):\n",
        "        self.frames_to_use = frames_to_use\n",
        "        self.batch_size = batch_size\n",
        "        self.keyframe_dir = keyframe_dir\n",
        "        self.shuffle = shuffle\n",
        "        self.is_test = is_test\n",
        "        self.classes = self.find_classes()\n",
        "        self.video_names, self.video_map, self.video_to_class, self.num_samples, self.min_frames = self.find_samples()\n",
        "        self.on_epoch_end()\n",
        "        if self.is_test:\n",
        "            print(f\"Found {self.num_samples} frames belonging to {len(self.video_names)} videos (test-mode).\")\n",
        "        else:\n",
        "            print(f\"Found {self.num_samples} frames belonging to {len(self.video_names)} videos belonging to {len(self.classes)} classes.\")\n",
        "        print(f\"Min frames determined to be {self.min_frames}\")\n",
        "\n",
        "    def find_classes(self):\n",
        "        if self.is_test:\n",
        "            return []\n",
        "        else:\n",
        "            category_folders = [f for f in listdir(self.keyframe_dir) if not isfile(join(self.keyframe_dir, f))]\n",
        "            return sorted(list(set(category_folders)))\n",
        "\n",
        "\n",
        "    def find_samples(self):\n",
        "        \"\"\"\n",
        "        \"\"\"\n",
        "        num_samples = 0\n",
        "        min_frames = -1\n",
        "        video_map = {}\n",
        "        vid_to_cat = {}\n",
        "\n",
        "        if self.is_test:\n",
        "            category_folders = [self.keyframe_dir]\n",
        "        else:\n",
        "            category_folders = [f for f in listdir(self.keyframe_dir) if not isfile(join(self.keyframe_dir, f))]\n",
        "        print(category_folders)\n",
        "        for category_folder in category_folders:\n",
        "            if self.is_test:\n",
        "                cat_path = category_folder\n",
        "            else:\n",
        "                cat_path = join(self.keyframe_dir, category_folder)\n",
        "            frames = [f for f in listdir(cat_path) if isfile(join(cat_path, f))]\n",
        "            for frame in frames:\n",
        "                frame_arr = frame.split(\".mp4_\")\n",
        "                vid_name = frame_arr[0]\n",
        "                if vid_name not in video_map:\n",
        "                    video_map[vid_name] = []\n",
        "                    vid_to_cat[vid_name] = category_folder\n",
        "                video_map[vid_name].append(frame)\n",
        "                \n",
        "            for k in video_map.keys():\n",
        "                # make sure the frames for each video are in sorted order\n",
        "                video_map[vid_name] = sorted(video_map[vid_name])\n",
        "                if min_frames == -1 or len(video_map[vid_name]) < min_frames:\n",
        "                    min_frames = len(video_map[vid_name])\n",
        "\n",
        "        return list(video_map.keys()), video_map, vid_to_cat, len(vid_to_cat), min_frames\n",
        "\n",
        "    def get_body_joints(self, x):\n",
        "        body_parts = [\n",
        "            1, # neck       --\n",
        "            2, # r shoulder\n",
        "            3, # r elbow\n",
        "            4, # r wrist\n",
        "            5, # l shoulder\n",
        "            6, # l elbow\n",
        "            7, # l wrist\n",
        "            9, # r hip\n",
        "            10, # r knee\n",
        "            11, # r ankle\n",
        "            12, # l hip\n",
        "            13, # l knee\n",
        "            14, # l ankle   -- \n",
        "        ]\n",
        "        body_parts_xy = []\n",
        "        for b in body_parts:\n",
        "            body_parts_xy.append(b * 3)\n",
        "            body_parts_xy.append(b * 3 + 1)\n",
        "        return x[body_parts_xy]\n",
        "\n",
        "    def normalize(self, x_input):\n",
        "        # Separate original data into x_list and y_list\n",
        "        lx = []\n",
        "        ly = []\n",
        "        N = len(x_input)\n",
        "        i = 0\n",
        "        while i<N:\n",
        "            lx.append(x_input[i])\n",
        "            ly.append(x_input[i+1])\n",
        "            i+=2\n",
        "        lx = np.array(lx)\n",
        "        ly = np.array(ly)\n",
        "\n",
        "        # Get rid of undetected data (=0)\n",
        "        non_zero_x = []\n",
        "        non_zero_y = []\n",
        "        for i in range(int(N/2)):\n",
        "            if lx[i] != 0:\n",
        "                non_zero_x.append(lx[i])\n",
        "            if ly[i] != 0:\n",
        "                non_zero_y.append(ly[i])\n",
        "        if len(non_zero_x) == 0 or len(non_zero_y) == 0:\n",
        "            return np.array([0] * N)\n",
        "\n",
        "        # Normalization x/y data according to the bounding box\n",
        "        origin_x = np.min(non_zero_x)\n",
        "        origin_y = np.min(non_zero_y)\n",
        "        len_x = np.max(non_zero_x) - np.min(non_zero_x)\n",
        "        len_y = np.max(non_zero_y) - np.min(non_zero_y)\n",
        "        x_new = []\n",
        "        for i in range(int(N/2)):\n",
        "            if (lx[i] + ly[i]) == 0:\n",
        "                x_new.append(-1)\n",
        "                x_new.append(-1)\n",
        "            else:\n",
        "                x_new.append((lx[i] - origin_x) / len_x)\n",
        "                x_new.append((ly[i] - origin_y) / len_y)\n",
        "        return x_new\n",
        "\n",
        "    def __len__(self):\n",
        "        \"\"\"\n",
        "        Denotes the number of batches per epoch\n",
        "        \"\"\"\n",
        "        return int(np.floor(self.num_samples / self.batch_size))\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        \"\"\"\n",
        "        Generate one batch of data\n",
        "        \"\"\"\n",
        "        video_names = self.video_names[index*self.batch_size:(index+1)*self.batch_size]\n",
        "        num_frames = self.min_frames if self.frames_to_use == -1 else self.frames_to_use\n",
        "        X = np.zeros((len(video_names), num_frames, 13 * 2 + 1), dtype=np.float64)\n",
        "        y = []\n",
        "        i = 0\n",
        "        for vid in video_names:\n",
        "            j = 0\n",
        "            for frame in self.video_map[vid]:\n",
        "                if self.is_test:\n",
        "                    keypoint_file = join(self.keyframe_dir, frame)\n",
        "                else:\n",
        "                    keypoint_file = join(join(self.keyframe_dir, self.video_to_class[vid]), frame)\n",
        "                with open(keypoint_file) as json_file:\n",
        "                    keypoint_data = json.load(json_file)\n",
        "\n",
        "                    # Extract some features from the keypoint data like averaging\n",
        "                    arrs = []\n",
        "\n",
        "                    for person in keypoint_data[\"people\"]:\n",
        "                        # Each person is assigned the label of the video\n",
        "                        kp = np.array(person[\"pose_keypoints_2d\"])\n",
        "                        kp = self.get_body_joints(kp)\n",
        "                        kp = self.normalize(kp)\n",
        "                        arrs.append(kp)\n",
        "                        # if i == 0 and j == 0:\n",
        "                        #     print(kp)\n",
        "                    \n",
        "                    if len(arrs) > 0:\n",
        "                        arrs = np.array(arrs)\n",
        "                        features = []\n",
        "                        features.extend(np.average(arrs, axis=0).tolist())\n",
        "                        features.append(len(keypoint_data[\"people\"]))\n",
        "                        features = np.array(features)\n",
        "                        features[np.isnan(features)] = -1\n",
        "                        X[i, j, :] = np.array(features)\n",
        "                    \n",
        "                j += 1\n",
        "                if j >= num_frames:\n",
        "                    break\n",
        "\n",
        "            if self.is_test:\n",
        "                y.append(0)\n",
        "            else:\n",
        "                y.append(int(self.video_to_class[vid]) - 1)\n",
        "            i += 1\n",
        "        y = np.array(y)\n",
        "        return X, tf.keras.utils.to_categorical(y, num_classes=len(self.classes))\n",
        "\n",
        "    def on_epoch_end(self):\n",
        "        if self.shuffle == True:\n",
        "            np.random.shuffle(self.video_names)\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fpl8T7NAqeNB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import zipfile\n",
        "import cv2\n",
        "from concurrent.futures import ThreadPoolExecutor\n",
        "from pathlib import Path\n",
        "import os\n",
        "from os.path import isfile, join\n",
        "import tensorflow as tf\n",
        "import tempfile\n",
        "import shutil\n",
        "\n",
        "\n",
        "\n",
        "class PoseClassifier:\n",
        "    \"\"\"\n",
        "    Classifies sentiment based on poses extracted from video frames\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, pose_folder, model_location=None, is_test=None, is_zip=True, frames_to_use=12, batch_size=32):\n",
        "        \"\"\"\n",
        "        @param pose_folder    The folder where the list of poses are stored. If \n",
        "                              `is_zip` is set to True, this should be a single zip \n",
        "                              file containing the poses. Paths can either by a local \n",
        "                              folder or a GDrive mounted path.\n",
        "        @param model_location The pre-trained model to perform predictions\n",
        "        @param is_test        If set to True, we assume that `pose_folder` contains a flat\n",
        "                              list of videos. If False, we assume that `pose_folder` first \n",
        "                              contains subdirectories corresponding to category labels. \n",
        "        @param is_zip         If set to True, the `pose_folder` will be unzipped prior to accessing\n",
        "        @param frames_to_use  The number of frames to use per video\n",
        "        @param batch_size     The batch size used to feed into the model evaluation\n",
        "        \"\"\"\n",
        "        self.is_zip = is_zip\n",
        "        self.pose_folder = pose_folder\n",
        "        self.is_test = is_test\n",
        "        self.model_location = model_location\n",
        "        self.frames_to_use = frames_to_use\n",
        "        self.batch_size = batch_size\n",
        "        print(f\"PoseClassifier created with is_zip = {is_zip}, pose_folder = {pose_folder} , is_test = {is_test} , model_location = {model_location}\")\n",
        "\n",
        "    def predict(self):\n",
        "        folder = self.unzip_folder()\n",
        "        generator = PoseDataGenerator(folder, is_test=self.is_test, frames_to_use=self.frames_to_use, batch_size=self.batch_size)\n",
        "        model = tf.keras.models.load_model(self.model_location)\n",
        "        return model.predict(generator)\n",
        "\n",
        "    def evaluate(self):\n",
        "        if self.is_test:\n",
        "            print(\"Evaluation cannot be done in test-mode\")\n",
        "            return\n",
        "\n",
        "        folder = self.unzip_folder()\n",
        "        generator = PoseDataGenerator(folder, is_test=self.is_test, frames_to_use=self.frames_to_use, batch_size=self.batch_size)\n",
        "        model = tf.keras.models.load_model(self.model_location)\n",
        "        return model.evaluate(generator)\n",
        "\n",
        "    def unzip_folder(self):\n",
        "        if self.is_zip:\n",
        "              # Unzips files to a temp directory\n",
        "              tmp_output_folder = \"pose_tmp\"\n",
        "              if os.path.exists(tmp_output_folder) and os.path.isdir(tmp_output_folder):\n",
        "                  print(\"Removing existing dir...\")\n",
        "                  shutil.rmtree(tmp_output_folder)\n",
        "\n",
        "              print(f\"Unzipping files to temp dir {tmp_output_folder}...\")\n",
        "              Path(f\"{tmp_output_folder}\").mkdir(parents=True, exist_ok=True)\n",
        "              with zipfile.ZipFile(self.pose_folder, 'r') as zip_ref:\n",
        "                  zip_ref.extractall(tmp_output_folder)\n",
        "              print(\"Finished unzipping files\")\n",
        "        else:\n",
        "            tmp_output_folder = self.pose_folder\n",
        "            print(\"Skipping unzipping files as input is a folder\")\n",
        "        return tmp_output_folder\n",
        "\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GxshK3rYJFcX",
        "colab_type": "code",
        "outputId": "f852659a-729c-4049-cfde-6d3c83098fe6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qbh349UdM0uU",
        "colab_type": "code",
        "outputId": "31640bd5-2c7b-4ee5-f3b8-89572cec0932",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 224
        }
      },
      "source": [
        "HOME_DIR = \"drive/My Drive/\"\n",
        "pose_classifier = PoseClassifier(\n",
        "    pose_folder = HOME_DIR + \"cs231n-project/datasets/emotiw/train-tiny-pose.zip\", \n",
        "    is_test = False,\n",
        "    model_location = HOME_DIR + \"cs231n-project/models/pose-classifier-v5.h5\"\n",
        ")\n",
        "pose_classifier.evaluate()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "PoseClassifier created with is_zip = True, pose_folder = drive/My Drive/cs231n-project/datasets/emotiw/train-tiny-pose.zip , is_test = False , model_location = drive/My Drive/cs231n-project/models/pose-classifier-v5.h5\n",
            "Removing existing dir...\n",
            "Unzipping files to temp dir pose_tmp...\n",
            "Finished unzipping files\n",
            "['1', '3', '2']\n",
            "Found 50 frames belonging to 50 videos belonging to 3 classes.\n",
            "Min frames determined to be 12\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:133: RuntimeWarning: invalid value encountered in double_scalars\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:134: RuntimeWarning: invalid value encountered in double_scalars\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "1/1 [==============================] - 0s 1ms/step - loss: 0.8743 - accuracy: 0.5938\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.8743239045143127, 0.59375]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "djHyqsMl7mK6",
        "colab_type": "code",
        "outputId": "946d8b73-2d02-46bc-a5ca-381307a6188c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 717
        }
      },
      "source": [
        "HOME_DIR = \"drive/My Drive/\"\n",
        "pose_classifier = PoseClassifier(\n",
        "    pose_folder = HOME_DIR + \"cs231n-project/datasets/emotiw/test-tiny-pose.zip\", \n",
        "    is_test = True,\n",
        "    model_location = HOME_DIR + \"cs231n-project/models/pose-classifier-v5.h5\"\n",
        ")\n",
        "pose_classifier.predict()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "PoseClassifier created with is_zip = True, pose_folder = drive/My Drive/cs231n-project/datasets/emotiw/test-tiny-pose.zip , is_test = True , model_location = drive/My Drive/cs231n-project/models/pose-classifier-v5.h5\n",
            "Unzipping files to temp dir pose_tmp...\n",
            "Finished unzipping files\n",
            "['pose_tmp']\n",
            "Found 50 frames belonging to 50 videos (test-mode).\n",
            "Min frames determined to be 15\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:133: RuntimeWarning: invalid value encountered in double_scalars\n",
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:134: RuntimeWarning: invalid value encountered in double_scalars\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[0.6961418 , 0.26301387, 0.04084435],\n",
              "       [0.32225487, 0.3230175 , 0.35472766],\n",
              "       [0.5572255 , 0.3802794 , 0.06249508],\n",
              "       [0.4602124 , 0.5062294 , 0.03355821],\n",
              "       [0.41146445, 0.5709377 , 0.01759779],\n",
              "       [0.15125038, 0.2793723 , 0.56937736],\n",
              "       [0.357679  , 0.6239649 , 0.01835612],\n",
              "       [0.15157497, 0.13344376, 0.71498126],\n",
              "       [0.35371315, 0.26059487, 0.385692  ],\n",
              "       [0.308326  , 0.39899087, 0.29268312],\n",
              "       [0.44515494, 0.5332356 , 0.02160949],\n",
              "       [0.3417638 , 0.44532776, 0.21290845],\n",
              "       [0.46070442, 0.5080671 , 0.03122848],\n",
              "       [0.20537141, 0.20358285, 0.59104574],\n",
              "       [0.461516  , 0.507118  , 0.03136603],\n",
              "       [0.3698711 , 0.5790788 , 0.05105008],\n",
              "       [0.4006774 , 0.5714115 , 0.02791112],\n",
              "       [0.32452822, 0.6155299 , 0.0599419 ],\n",
              "       [0.38561583, 0.59843326, 0.01595092],\n",
              "       [0.37565228, 0.36649704, 0.25785065],\n",
              "       [0.4620367 , 0.4182856 , 0.11967767],\n",
              "       [0.16377105, 0.75288755, 0.08334146],\n",
              "       [0.47101715, 0.48570663, 0.04327624],\n",
              "       [0.25243807, 0.3249194 , 0.42264256],\n",
              "       [0.43151718, 0.43775362, 0.13072917],\n",
              "       [0.39344105, 0.52768934, 0.07886965],\n",
              "       [0.293047  , 0.45168144, 0.25527152],\n",
              "       [0.38797402, 0.17140009, 0.44062588],\n",
              "       [0.40990496, 0.50914514, 0.08094983],\n",
              "       [0.38617408, 0.597309  , 0.01651692],\n",
              "       [0.44925192, 0.5310126 , 0.01973541],\n",
              "       [0.4089486 , 0.5669616 , 0.02408979]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "abak6XDwKded",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}