{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"FINAL-ensemble_fc_predictions_v2-with-laugh_modality_association.ipynb","provenance":[],"collapsed_sections":[],"toc_visible":true},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU","widgets":{"application/vnd.jupyter.widget-state+json":{"71c781a6e1f949dc988c795e1cbed930":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_815ccf4c91104e0db1a4065a7f943725","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_41caaab83dc44ea9923817256beeea60","IPY_MODEL_d3dd8f3c8b154cc39338fb8ad1b11e74"]}},"815ccf4c91104e0db1a4065a7f943725":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"41caaab83dc44ea9923817256beeea60":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_581232e286c04db4989bdbe11f119e49","_dom_classes":[],"description":"Processing 2661 examples on 2 cores: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":2661,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":2661,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_2dd5f4b0fb4643cab2f7d8b2932f7d81"}},"d3dd8f3c8b154cc39338fb8ad1b11e74":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_6c9d548da24541aa8eada24a5b5985d6","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 2661/2661 [05:31<00:00, 8.04it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_d73d30e03fd84384a2a6f5e88b8d371e"}},"581232e286c04db4989bdbe11f119e49":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"2dd5f4b0fb4643cab2f7d8b2932f7d81":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"6c9d548da24541aa8eada24a5b5985d6":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"d73d30e03fd84384a2a6f5e88b8d371e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"db9f9863216b4616a4f8d46bbfa7c3ce":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","state":{"_view_name":"HBoxView","_dom_classes":[],"_model_name":"HBoxModel","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.5.0","box_style":"","layout":"IPY_MODEL_c32e02a584554cada71c0529d1eeeb58","_model_module":"@jupyter-widgets/controls","children":["IPY_MODEL_ba25bd32b1ed460aa025d3a4dbc628c1","IPY_MODEL_e8486c84c97d4db383515c5207c1790e"]}},"c32e02a584554cada71c0529d1eeeb58":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"ba25bd32b1ed460aa025d3a4dbc628c1":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","state":{"_view_name":"ProgressView","style":"IPY_MODEL_92799b3ca797492f83ad922eda616a88","_dom_classes":[],"description":"Processing 766 examples on 2 cores: 100%","_model_name":"FloatProgressModel","bar_style":"success","max":766,"_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":766,"_view_count":null,"_view_module_version":"1.5.0","orientation":"horizontal","min":0,"description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_ea273c48ba6846508cf8c013a2018c66"}},"e8486c84c97d4db383515c5207c1790e":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","state":{"_view_name":"HTMLView","style":"IPY_MODEL_9f279e4610cd4e208e7e333f44474461","_dom_classes":[],"description":"","_model_name":"HTMLModel","placeholder":"","_view_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","value":" 766/766 [02:36<00:00, 4.88it/s]","_view_count":null,"_view_module_version":"1.5.0","description_tooltip":null,"_model_module":"@jupyter-widgets/controls","layout":"IPY_MODEL_1c7d408c45844205b2d85d5b3d7ba48f"}},"92799b3ca797492f83ad922eda616a88":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","state":{"_view_name":"StyleView","_model_name":"ProgressStyleModel","description_width":"initial","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","bar_color":null,"_model_module":"@jupyter-widgets/controls"}},"ea273c48ba6846508cf8c013a2018c66":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}},"9f279e4610cd4e208e7e333f44474461":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","state":{"_view_name":"StyleView","_model_name":"DescriptionStyleModel","description_width":"","_view_module":"@jupyter-widgets/base","_model_module_version":"1.5.0","_view_count":null,"_view_module_version":"1.2.0","_model_module":"@jupyter-widgets/controls"}},"1c7d408c45844205b2d85d5b3d7ba48f":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","state":{"_view_name":"LayoutView","grid_template_rows":null,"right":null,"justify_content":null,"_view_module":"@jupyter-widgets/base","overflow":null,"_model_module_version":"1.2.0","_view_count":null,"flex_flow":null,"width":null,"min_width":null,"border":null,"align_items":null,"bottom":null,"_model_module":"@jupyter-widgets/base","top":null,"grid_column":null,"overflow_y":null,"overflow_x":null,"grid_auto_flow":null,"grid_area":null,"grid_template_columns":null,"flex":null,"_model_name":"LayoutModel","justify_items":null,"grid_row":null,"max_height":null,"align_content":null,"visibility":null,"align_self":null,"height":null,"min_height":null,"padding":null,"grid_auto_rows":null,"grid_gap":null,"max_width":null,"order":null,"_view_module_version":"1.2.0","grid_template_areas":null,"object_position":null,"object_fit":null,"grid_auto_columns":null,"margin":null,"display":null,"left":null}}}}},"cells":[{"cell_type":"markdown","metadata":{"id":"view-in-github","colab_type":"text"},"source":["![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\")
"]},{"cell_type":"markdown","metadata":{"id":"0rWTS99OI8l-","colab_type":"text"},"source":["## Video Sentiment Analysis in the Wild\n","### Ensembling Notebook | FC | CS231n\n","\n","This notebook runs a basic FC classifier on the predicted outputs of each individual modality. Assumes the preprocessing has already been completed.\n","\n","Contains:\n","- Frame\n","- Audio\n","- Pose\n"]},{"cell_type":"markdown","metadata":{"id":"AKBv4jZKrhQ0","colab_type":"text"},"source":["### Copy Pre-Processed Files"]},{"cell_type":"code","metadata":{"id":"QdlpDjXpdiqz","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1593061101124,"user_tz":420,"elapsed":1384,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"69c30f32-cd9b-4164-b15a-f102ca4cf22c"},"source":["!ls"],"execution_count":null,"outputs":[{"output_type":"stream","text":["sample_data\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"C40EhGh9kopZ","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":357},"executionInfo":{"status":"ok","timestamp":1593061103542,"user_tz":420,"elapsed":1370,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"3ab2dbcb-3bb1-4b89-b3cf-b8cde8912b18"},"source":["!nvidia-smi"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Thu Jun 25 04:58:22 2020 \n","+-----------------------------------------------------------------------------+\n","| NVIDIA-SMI 450.36.06 Driver Version: 418.67 CUDA Version: 10.1 |\n","|-------------------------------+----------------------+----------------------+\n","| GPU Name Persistence-M| Bus-Id Disp.A | Volatile Uncorr. ECC |\n","| Fan Temp Perf Pwr:Usage/Cap| Memory-Usage | GPU-Util Compute M. |\n","| | | MIG M. |\n","|===============================+======================+======================|\n","| 0 Tesla P100-PCIE... Off | 00000000:00:04.0 Off | 0 |\n","| N/A 37C P0 25W / 250W | 0MiB / 16280MiB | 0% Default |\n","| | | ERR! |\n","+-------------------------------+----------------------+----------------------+\n"," \n","+-----------------------------------------------------------------------------+\n","| Processes: |\n","| GPU GI CI PID Type Process name GPU Memory |\n","| ID ID Usage |\n","|=============================================================================|\n","| No running processes found |\n","+-----------------------------------------------------------------------------+\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"XQnFc3BdeCVL","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":513},"executionInfo":{"status":"ok","timestamp":1592690537191,"user_tz":420,"elapsed":235759,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"2a1c8c64-618c-4a6c-b729-f83849e1df83"},"source":["\n","# FULL_PATH = 'My Drive/Machine-Learning-Projects/cs231n-project/datasets/emotiw'\n","FULL_PATH = 'My Drive/cs231n-project/datasets/emotiw'\n","\n","print(\"Using final dataset...\")\n","!cp /content/drive/'$FULL_PATH'/train-final-* .\n","!cp /content/drive/'$FULL_PATH'/val-final-* .\n","\n","!wget https://storage.googleapis.com/cs231n-emotiw/data/Train_labels.txt\n","!wget https://storage.googleapis.com/cs231n-emotiw/data/Val_labels.txt\n"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n","\n","Enter your authorization code:\n","··········\n","Mounted at /content/drive\n","Using final dataset...\n","--2020-06-20 22:02:14-- https://storage.googleapis.com/cs231n-emotiw/data/Train_labels.txt\n","Resolving storage.googleapis.com (storage.googleapis.com)... 64.233.189.128, 2404:6800:4008:c03::80\n","Connecting to storage.googleapis.com (storage.googleapis.com)|64.233.189.128|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 21653 (21K) [text/plain]\n","Saving to: ‘Train_labels.txt’\n","\n","Train_labels.txt 100%[===================>] 21.15K --.-KB/s in 0s \n","\n","2020-06-20 22:02:14 (99.3 MB/s) - ‘Train_labels.txt’ saved [21653/21653]\n","\n","--2020-06-20 22:02:15-- https://storage.googleapis.com/cs231n-emotiw/data/Val_labels.txt\n","Resolving storage.googleapis.com (storage.googleapis.com)... 64.233.189.128, 2404:6800:4008:c06::80\n","Connecting to storage.googleapis.com (storage.googleapis.com)|64.233.189.128|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 6265 (6.1K) [text/plain]\n","Saving to: ‘Val_labels.txt’\n","\n","Val_labels.txt 100%[===================>] 6.12K --.-KB/s in 0s \n","\n","2020-06-20 22:02:16 (52.0 MB/s) - ‘Val_labels.txt’ saved [6265/6265]\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"yXxCkE_DUvEv","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":104},"outputId":"9812567a-52b0-43ce-e203-74cb990d8404"},"source":["!ls"],"execution_count":null,"outputs":[{"output_type":"stream","text":["drive\t\t train-final-frames.zip val-final-fer.zip\n","sample_data\t train-final-pose.zip val-final-frames.zip\n","train-final-audio.zip Train_labels.txt val-final-pose.zip\n","train-final-faces.zip val-final-audio.zip Val_labels.txt\n","train-final-fer.zip val-final-faces.zip\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"sqHRPvB5Cm4F","colab_type":"code","colab":{}},"source":["# RUN THIS FOR FINAL FILES (zip includes root folder)\n","\n","!unzip train-final-audio.zip\n","!unzip train-final-faces.zip\n","!unzip train-final-frames.zip\n","!unzip train-final-pose.zip\n","!unzip -d train-final-fer train-final-fer.zip\n","\n","!unzip val-final-audio.zip\n","!unzip val-final-faces.zip\n","!unzip val-final-frames.zip\n","!unzip val-final-pose.zip\n","!unzip -d val-final-fer val-final-fer.zip\n","\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"gF6Fy4-SLJVP","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":170},"executionInfo":{"status":"ok","timestamp":1592691050508,"user_tz":420,"elapsed":3599,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"87905680-7a90-4957-8a5c-23d4b0f9e198"},"source":["!ls"],"execution_count":null,"outputs":[{"output_type":"stream","text":["drive\t\t\t train-final-frames\t val-final-faces.zip\n","sample_data\t\t train-final-frames.zip val-final-fer\n","train-final-audio\t train-final-pose\t val-final-fer.zip\n","train-final-audio.zip\t train-final-pose.zip\t val-final-frames\n","train-final-captions.pkl Train_labels.txt\t val-final-frames.zip\n","train-final-faces\t val-final-audio\t val-final-pose\n","train-final-faces.zip\t val-final-audio.zip\t val-final-pose.zip\n","train-final-fer\t\t val-final-captions.pkl Val_labels.txt\n","train-final-fer.zip\t val-final-faces\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"0jEaDQAsS8XQ","colab_type":"text"},"source":["### Run Classifiers"]},{"cell_type":"code","metadata":{"id":"w0_gfpBjXP6S","colab_type":"code","colab":{}},"source":["%tensorflow_version 2.x"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"dPXBx950Xd8z","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1593061444738,"user_tz":420,"elapsed":1595,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"3e966e59-6423-4cc4-d0fa-90f3589ee79f"},"source":["import tensorflow\n","print(tensorflow.__version__)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["2.2.0\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"pp_USHxwRAVB","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1593061445517,"user_tz":420,"elapsed":2107,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"d5598b6b-e77c-47c7-e65c-7f84417947b4"},"source":["!pwd"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/content\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"_Nybv6fMJV4X","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":51},"executionInfo":{"status":"ok","timestamp":1593061465377,"user_tz":420,"elapsed":21530,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"991026cd-79db-49a9-bca4-8bd07c038149"},"source":["import urllib\n","from getpass import getpass\n","import os\n","user = input('User name: ')\n","password = getpass('Password: ')\n","password = urllib.parse.quote(password) # your password is converted into url format\n","\n","cmd_string = 'git clone https://{0}:{1}@github.com/kevincong95/cs231n-emotiw.git'.format(user, password)\n","\n","os.system(cmd_string)\n","cmd_string, password = \"\", \"\" # removing the password from the variable"],"execution_count":null,"outputs":[{"output_type":"stream","text":["User name: tbj128\n","Password: ··········\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Z7kRhNvrVnbt","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":51},"executionInfo":{"status":"ok","timestamp":1592980340742,"user_tz":420,"elapsed":12876,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"112cc328-561c-4d2c-953f-e64c9747d55b"},"source":["!mv train-* cs231n-emotiw\n","!mv val-* cs231n-emotiw"],"execution_count":null,"outputs":[{"output_type":"stream","text":["mv: cannot stat 'train-*': No such file or directory\n","mv: cannot stat 'val-*': No such file or directory\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"tauiFf5eVtpP","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":51},"executionInfo":{"status":"ok","timestamp":1592980342470,"user_tz":420,"elapsed":14279,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"34e2e143-8e02-4caf-b001-8f858e18e565"},"source":["!mv Train* cs231n-emotiw\n","!mv Val* cs231n-emotiw"],"execution_count":null,"outputs":[{"output_type":"stream","text":["mv: cannot stat 'Train*': No such file or directory\n","mv: cannot stat 'Val*': No such file or directory\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"qiVbcNhfVwa2","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1592980343244,"user_tz":420,"elapsed":12548,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"9a8bc85c-a5c1-4e1f-ca9d-dc9c4a7e1f81"},"source":["!pwd"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/content\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"hPM9Vm7gVxuR","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1593061466498,"user_tz":420,"elapsed":14327,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"9bd291d2-a67a-4fe4-8b7e-1b643f7c44a6"},"source":["import os\n","os.chdir('/content/cs231n-emotiw')\n","!pwd"],"execution_count":null,"outputs":[{"output_type":"stream","text":["/content/cs231n-emotiw\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"oxC5BZIRKyrq","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":632},"executionInfo":{"status":"ok","timestamp":1593061472155,"user_tz":420,"elapsed":16478,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"75e6acd2-72ac-4d65-ed84-b70d3ada50ed"},"source":["!pip install pytorch-transformers"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Collecting pytorch-transformers\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/a3/b7/d3d18008a67e0b968d1ab93ad444fc05699403fa662f634b2f2c318a508b/pytorch_transformers-1.2.0-py3-none-any.whl (176kB)\n","\u001b[K |████████████████████████████████| 184kB 3.4MB/s \n","\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (2019.12.20)\n","Requirement already satisfied: boto3 in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (1.14.5)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (1.18.5)\n","Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (2.23.0)\n","Requirement already satisfied: torch>=1.0.0 in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (1.5.1+cu101)\n","Collecting sacremoses\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n","\u001b[K |████████████████████████████████| 890kB 8.6MB/s \n","\u001b[?25hRequirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (from pytorch-transformers) (4.41.1)\n","Collecting sentencepiece\n","\u001b[?25l Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n","\u001b[K |████████████████████████████████| 1.1MB 17.0MB/s \n","\u001b[?25hRequirement already satisfied: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-transformers) (0.10.0)\n","Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-transformers) (0.3.3)\n","Requirement already satisfied: botocore<1.18.0,>=1.17.5 in /usr/local/lib/python3.6/dist-packages (from boto3->pytorch-transformers) (1.17.5)\n","Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers) (3.0.4)\n","Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers) (2.9)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers) (2020.4.5.2)\n","Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->pytorch-transformers) (1.24.3)\n","Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.0.0->pytorch-transformers) (0.16.0)\n","Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->pytorch-transformers) (1.12.0)\n","Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->pytorch-transformers) (7.1.2)\n","Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->pytorch-transformers) (0.15.1)\n","Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.5->boto3->pytorch-transformers) (2.8.1)\n","Requirement already satisfied: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.18.0,>=1.17.5->boto3->pytorch-transformers) (0.15.2)\n","Building wheels for collected packages: sacremoses\n"," Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"," Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=20809d1e9e8974e576c67497eb62ae2c0e2836c4bfc42d7d8074fd8524334765\n"," Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n","Successfully built sacremoses\n","Installing collected packages: sacremoses, sentencepiece, pytorch-transformers\n","Successfully installed pytorch-transformers-1.2.0 sacremoses-0.0.43 sentencepiece-0.1.91\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"9xwIeRF_x40m","colab_type":"code","colab":{}},"source":["# Create the concatenated input layer to feed into FC\n","\n","from src.classifiers.audio_classifier import AudioClassifier\n","from src.classifiers.frames_classifier import FramesClassifier\n","from src.classifiers.pose_classifier import PoseClassifier\n","from src.classifiers.face_classifier import FaceClassifier\n","from src.classifiers.image_captioning_classifier import ImageCaptioningClassifier, FineTuningConfig\n","from src.classifiers.utils import get_num_samples\n","import numpy as np\n","\n","IS_TINY = False\n","def run_classifier(layers_to_extract, audio_folder='train-final-audio', frames_folder='train-final-frames', pose_folder='train-final-pose', face_folder='train-final-fer', image_caption_pkl=\"train-final-captions.pkl\", image_caption_prefix=\"train_\", labels_file=\"Train_labels.txt\"):\n"," audio_classifier = AudioClassifier(audio_folder, model_location='https://storage.googleapis.com/cs231n-emotiw/models/openl3-cnn-lstm-tuned-lr.h5', is_test=False)\n"," frames_classifier = FramesClassifier(frames_folder, model_location='https://storage.googleapis.com/cs231n-emotiw/models/scene-classifier-resnet-lstm-x3.h5', is_test=False)\n"," frames_classifier_vgg = FramesClassifier(frames_folder, location_prefix=\"vgg\", model_location='https://storage.googleapis.com/cs231n-emotiw/models/vgg19-lstm-cp-0003.h5', is_test=False, batch_size=4)\n"," pose_classifier = PoseClassifier(pose_folder, model_location='https://storage.googleapis.com/cs231n-emotiw/models/pose-classifier-64lstm-0.01reg.h5', is_test=False)\n"," face_classifier = FaceClassifier(face_folder, model_location='/content/drive/My Drive/cs231n-project/models/face-classifier-playground/cp-0001.h5', is_test=False)\n"," image_captioning_classifier = ImageCaptioningClassifier(image_caption_pkl, image_caption_prefix, model_metadata_location=\"https://storage.googleapis.com/cs231n-emotiw/models/sentiment-transformer-16.metadata.bin\", model_location='https://storage.googleapis.com/cs231n-emotiw/models/sentiment-transformer_sentiment-transformer_756.pth', is_test=False)\n"," \n"," # classifiers = [audio_classifier, frames_classifier, pose_classifier] # face_classifier]\n"," classifiers = [audio_classifier, frames_classifier, frames_classifier_vgg, pose_classifier, face_classifier, image_captioning_classifier]\n"," # classifiers = [frames_classifier_vgg]\n"," \n"," sample_to_true_label = {}\n"," with open(labels_file) as f:\n"," l = 0\n"," for line in f:\n"," if l == 0:\n"," # Skip headers\n"," l += 1\n"," continue\n"," line_arr = line.split(\" \")\n"," sample_to_true_label[line_arr[0].strip()] = int(line_arr[1].strip()) - 1 # subtract one to make labels from 0 to 2\n"," l += 1\n","\n"," classifier_outputs = []\n"," classifier_samples = []\n"," classifier_dim_sizes = []\n"," output_dim_size = 0\n"," num_samples = 0\n"," sample_to_row = {}\n","\n"," for i, classifier in enumerate(classifiers):\n"," output, samples = classifier.predict(layers_to_extract[i])\n"," output_dim_size += output.shape[1]\n"," classifier_dim_sizes.append(output.shape[1])\n"," num_samples = len(samples)\n"," classifier_outputs.append(output)\n"," classifier_samples.append(samples)\n","\n","\n"," X_train = np.zeros(shape=(num_samples, output_dim_size))\n"," y_train = []\n","\n"," print(f\"Number of samples: {num_samples}\")\n"," print(f\"Dim shapes: \")\n"," print(classifier_dim_sizes)\n","\n"," for i, sample in enumerate(classifier_samples[0]):\n"," sample_to_row[sample] = i\n"," y_train.append(sample_to_true_label[sample])\n","\n"," last_classifier_index = 0\n"," for c, output in enumerate(classifier_outputs):\n"," samples = classifier_samples[c]\n"," print(len(output))\n"," for i, row in enumerate(output):\n"," sample = samples[i]\n"," X_train[sample_to_row[sample], last_classifier_index:last_classifier_index+classifier_dim_sizes[c]] += row\n"," last_classifier_index += classifier_dim_sizes[c]\n","\n"," return X_train, tf.keras.utils.to_categorical(y_train, num_classes=3)\n","\n","\n","\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"1VGA1fl2pNvE","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000,"referenced_widgets":["71c781a6e1f949dc988c795e1cbed930","815ccf4c91104e0db1a4065a7f943725","41caaab83dc44ea9923817256beeea60","d3dd8f3c8b154cc39338fb8ad1b11e74","581232e286c04db4989bdbe11f119e49","2dd5f4b0fb4643cab2f7d8b2932f7d81","6c9d548da24541aa8eada24a5b5985d6","d73d30e03fd84384a2a6f5e88b8d371e","db9f9863216b4616a4f8d46bbfa7c3ce","c32e02a584554cada71c0529d1eeeb58","ba25bd32b1ed460aa025d3a4dbc628c1","e8486c84c97d4db383515c5207c1790e","92799b3ca797492f83ad922eda616a88","ea273c48ba6846508cf8c013a2018c66","9f279e4610cd4e208e7e333f44474461","1c7d408c45844205b2d85d5b3d7ba48f"]},"executionInfo":{"status":"ok","timestamp":1592703526102,"user_tz":420,"elapsed":661289,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"e435e8ea-241c-4e11-d9ef-b60855f0d3d6"},"source":["import tensorflow as tf\n","# For each classifier, extract the specific desired layer \n","# (refer to the model summary for the layer names)\n","\n","layers_to_extract = [\n"," \"dense\", # Audio\n"," \"concatenate_5\", # ResNet\n"," \"global_average_pooling3d_1\", # VGG\n"," \"bidirectional_1\", # Pose\n"," \"dense_27\", # FER\n"," \"classification_head\" # Image Caption\n","]\n","\n","prefix = \"final\"\n","if IS_TINY:\n"," prefix = \"tiny\"\n","\n","# X_train, y_train = run_classifier(layers_to_extract, audio_folder=f\"train-{prefix}-audio\", frames_folder=f\"train-{prefix}-frames\", pose_folder=f\"train-{prefix}-pose\", face_folder=f\"train-{prefix}-fer\" ,labels_file=\"Train_labels.txt\")\n","# X_valid, y_valid = run_classifier(layers_to_extract, audio_folder=f\"val-{prefix}-audio\", frames_folder=f\"val-{prefix}-frames\", pose_folder=f\"val-{prefix}-pose\", face_folder=f\"val-{prefix}-fer\" , labels_file=\"Val_labels.txt\")\n","\n","\n","X_train, y_train = run_classifier(layers_to_extract, audio_folder=f\"train-{prefix}-audio\", frames_folder=f\"train-{prefix}-frames\", pose_folder=f\"train-{prefix}-pose\" , face_folder=f\"train-{prefix}-fer\" , image_caption_pkl=\"train-final-captions.pkl\", image_caption_prefix=\"train_\", labels_file=\"Train_labels.txt\")\n","X_valid, y_valid = run_classifier(layers_to_extract, audio_folder=f\"val-{prefix}-audio\", frames_folder=f\"val-{prefix}-frames\", pose_folder=f\"val-{prefix}-pose\" , face_folder=f\"val-{prefix}-fer\" , image_caption_pkl=\"val-final-captions.pkl\", image_caption_prefix=\"val_\", labels_file=\"Val_labels.txt\")\n","\n","print(X_train.shape)\n","print(y_train.shape)"],"execution_count":null,"outputs":[{"output_type":"stream","text":["AudioClassifier created with audio_folder = train-final-audio , is_test = False , model_location = https://storage.googleapis.com/cs231n-emotiw/models/openl3-cnn-lstm-tuned-lr.h5\n","WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n","WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n","WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n","FramesClassifier created with frames_folder = train-final-frames , is_test = False , model_location = https://storage.googleapis.com/cs231n-emotiw/models/scene-classifier-resnet-lstm-x3.h5\n","FramesClassifier created with frames_folder = train-final-frames , is_test = False , model_location = https://storage.googleapis.com/cs231n-emotiw/models/vgg19-lstm-cp-0003.h5\n","PoseClassifier created with pose_folder = train-final-pose , is_test = False , model_location = https://storage.googleapis.com/cs231n-emotiw/models/pose-classifier-64lstm-0.01reg.h5\n","load data\n","FacesClassifier created with face_folder = train-final-fer , is_test = False , model_location = /content/drive/My Drive/cs231n-project/models/face-classifier-playground/cp-0001.h5\n","ImageCaptioningClassifier created with caption_pkl = train-final-captions.pkl , is_test = False , model_location = https://storage.googleapis.com/cs231n-emotiw/models/sentiment-transformer_sentiment-transformer_756.pth\n","Skipping unzipping files as input is a folder\n","Customizing model by returning layer dense\n","Skipping unzipping files as input is a folder\n","Found 2661 frames belonging to 2661 videos belonging to 3 classes.\n","Min frames determined to be 7\n","Customizing model by returning layer concatenate_5\n","Skipping unzipping files as input is a folder\n","Found 2661 frames belonging to 2661 videos belonging to 3 classes.\n","Min frames determined to be 7\n","Customizing model by returning layer global_average_pooling3d_1\n","Skipping unzipping files as input is a folder\n","['3', '2', '1']\n","Found 2661 frames belonging to 2661 videos belonging to 3 classes.\n","Min frames determined to be 7\n","Customizing model by returning layer bidirectional_1\n"],"name":"stdout"},{"output_type":"stream","text":["/content/cs231n-emotiw/src/generators/pose_generator.py:135: RuntimeWarning: invalid value encountered in double_scalars\n"," x_new.append((lx[i] - origin_x) / len_x)\n","/content/cs231n-emotiw/src/generators/pose_generator.py:136: RuntimeWarning: invalid value encountered in double_scalars\n"," x_new.append((ly[i] - origin_y) / len_y)\n"],"name":"stderr"},{"output_type":"stream","text":["Customizing model by returning layer dense_27\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"71c781a6e1f949dc988c795e1cbed930","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Processing 2661 examples on 2 cores', max=2661.0, style=P…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","Number of samples: 2661\n","Dim shapes: \n","[32, 30, 40, 128, 8, 16]\n","2661\n","2661\n","2661\n","2661\n","2661\n","2661\n","AudioClassifier created with audio_folder = val-final-audio , is_test = False , model_location = https://storage.googleapis.com/cs231n-emotiw/models/openl3-cnn-lstm-tuned-lr.h5\n","WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n","WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n","WARNING:tensorflow:Layer lstm will not use cuDNN kernel since it doesn't meet the cuDNN kernel criteria. It will use generic GPU kernel as fallback when running on GPU\n","FramesClassifier created with frames_folder = val-final-frames , is_test = False , model_location = https://storage.googleapis.com/cs231n-emotiw/models/scene-classifier-resnet-lstm-x3.h5\n","FramesClassifier created with frames_folder = val-final-frames , is_test = False , model_location = https://storage.googleapis.com/cs231n-emotiw/models/vgg19-lstm-cp-0003.h5\n","PoseClassifier created with pose_folder = val-final-pose , is_test = False , model_location = https://storage.googleapis.com/cs231n-emotiw/models/pose-classifier-64lstm-0.01reg.h5\n","load data\n","FacesClassifier created with face_folder = val-final-fer , is_test = False , model_location = /content/drive/My Drive/cs231n-project/models/face-classifier-playground/cp-0001.h5\n","ImageCaptioningClassifier created with caption_pkl = val-final-captions.pkl , is_test = False , model_location = https://storage.googleapis.com/cs231n-emotiw/models/sentiment-transformer_sentiment-transformer_756.pth\n","Skipping unzipping files as input is a folder\n","Customizing model by returning layer dense\n","Skipping unzipping files as input is a folder\n","Found 766 frames belonging to 766 videos belonging to 3 classes.\n","Min frames determined to be 7\n","Customizing model by returning layer concatenate_5\n","Skipping unzipping files as input is a folder\n","Found 766 frames belonging to 766 videos belonging to 3 classes.\n","Min frames determined to be 7\n","Customizing model by returning layer global_average_pooling3d_1\n","Skipping unzipping files as input is a folder\n","['3', '2', '1']\n","Found 766 frames belonging to 766 videos belonging to 3 classes.\n","Min frames determined to be 7\n","Customizing model by returning layer bidirectional_1\n"],"name":"stdout"},{"output_type":"stream","text":["/content/cs231n-emotiw/src/generators/pose_generator.py:135: RuntimeWarning: invalid value encountered in double_scalars\n"," x_new.append((lx[i] - origin_x) / len_x)\n","/content/cs231n-emotiw/src/generators/pose_generator.py:136: RuntimeWarning: invalid value encountered in double_scalars\n"," x_new.append((ly[i] - origin_y) / len_y)\n"],"name":"stderr"},{"output_type":"stream","text":["Customizing model by returning layer dense_27\n"],"name":"stdout"},{"output_type":"display_data","data":{"application/vnd.jupyter.widget-view+json":{"model_id":"db9f9863216b4616a4f8d46bbfa7c3ce","version_minor":0,"version_major":2},"text/plain":["HBox(children=(FloatProgress(value=0.0, description='Processing 766 examples on 2 cores', max=766.0, style=Pro…"]},"metadata":{"tags":[]}},{"output_type":"stream","text":["\n","Number of samples: 766\n","Dim shapes: \n","[32, 30, 40, 128, 8, 16]\n","766\n","766\n","766\n","766\n","766\n","766\n","(2661, 254)\n","(2661, 3)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"kXNhfFvFho1Q","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":295},"outputId":"d6a8b8c7-224d-484e-b907-57dd86d0bd8c"},"source":["!ls"],"execution_count":null,"outputs":[{"output_type":"stream","text":[" data\t\t\t\t\t train-final-frames\n"," LICENSE\t\t\t\t train-final-frames.zip\n"," models\t\t\t\t\t train-final-pose\n"," notebooks\t\t\t\t train-final-pose.zip\n"," README.md\t\t\t\t Train_labels.txt\n"," reports\t\t\t\t val-final-audio\n"," requirements-predictions.txt\t\t val-final-audio.zip\n"," requirements.txt\t\t\t val-final-faces\n","'Screen Shot 2020-05-26 at 8.53.32 PM.png' val-final-faces.zip\n"," src\t\t\t\t\t val-final-fer\n"," train-final-audio\t\t\t val-final-fer.zip\n"," train-final-audio.zip\t\t\t val-final-frames\n"," train-final-faces\t\t\t val-final-frames.zip\n"," train-final-faces.zip\t\t\t val-final-pose\n"," train-final-fer\t\t\t val-final-pose.zip\n"," train-final-fer.zip\t\t\t Val_labels.txt\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"G0zv6L6hhzsP","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":102},"executionInfo":{"status":"ok","timestamp":1592703596350,"user_tz":420,"elapsed":7264,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"e5ae6b5c-4d00-405d-8233-b7590b7f3a24"},"source":["!rm -rf ensemble-scene-scene-pose-audio-face-caption-v1\n","!mkdir ensemble-scene-scene-pose-audio-face-caption-v1\n","np.save(\"ensemble-scene-scene-pose-audio-face-caption-v1/X_train.npy\", X_train)\n","np.save(\"ensemble-scene-scene-pose-audio-face-caption-v1/y_train.npy\", y_train)\n","np.save(\"ensemble-scene-scene-pose-audio-face-caption-v1/X_valid.npy\", X_valid)\n","np.save(\"ensemble-scene-scene-pose-audio-face-caption-v1/y_valid.npy\", y_valid)\n","!zip -r ensemble-scene-scene-pose-audio-face-caption-v1.zip ensemble-scene-scene-pose-audio-face-caption-v1\n","# !cp ensemble-scene-pose-audio-v1.zip ../drive/'My Drive/Machine-Learning-Projects'/cs231n-project/datasets/emotiw\n","!cp ensemble-scene-scene-pose-audio-face-caption-v1.zip ../drive/'My Drive'/cs231n-project/datasets/emotiw\n"],"execution_count":null,"outputs":[{"output_type":"stream","text":["updating: ensemble-scene-scene-pose-audio-face-caption-v1/ (stored 0%)\n","updating: ensemble-scene-scene-pose-audio-face-caption-v1/X_train.npy (deflated 49%)\n","updating: ensemble-scene-scene-pose-audio-face-caption-v1/y_valid.npy (deflated 96%)\n","updating: ensemble-scene-scene-pose-audio-face-caption-v1/y_train.npy (deflated 97%)\n","updating: ensemble-scene-scene-pose-audio-face-caption-v1/X_valid.npy (deflated 49%)\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"xGS2rrUIZ_YX","colab_type":"text"},"source":["## START HERE IF YOU DON'T WANT TO REMAKE THE CONCAT ABOVE"]},{"cell_type":"code","metadata":{"id":"a3uocKM6CN7a","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":124},"executionInfo":{"status":"ok","timestamp":1593125852532,"user_tz":240,"elapsed":14959,"user":{"displayName":"Leila Abdelrahman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiiOBijfsdVdWn4yrdWM-_fgxn37DQ4bPiUdT7W=s64","userId":"10595688673230464384"}},"outputId":"07e11667-1266-4cea-8e30-9beebda9a60a"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n","\n","Enter your authorization code:\n","··········\n","Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"EkbMvFKG4IvD","colab_type":"code","colab":{}},"source":["!cp ../drive/'My Drive'/cs231n-project/datasets/emotiw/train-final-audio.zip .\n","!unzip -q -d train-final-audio train-final-audio.zip\n"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"4t07onWM4QIZ","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":187},"executionInfo":{"status":"ok","timestamp":1593061756804,"user_tz":420,"elapsed":982,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"c40cb00a-50ae-47ef-f548-aafee9c347d3"},"source":["!ls train-final-audio/train-final-audio/audio-pickle | head"],"execution_count":null,"outputs":[{"output_type":"stream","text":["10_10.mp4-openl3.pkl\n","101_10.mp4-openl3.pkl\n","101_11.mp4-openl3.pkl\n","101_12.mp4-openl3.pkl\n","101_13.mp4-openl3.pkl\n","101_14.mp4-openl3.pkl\n","101_15.mp4-openl3.pkl\n","101_16.mp4-openl3.pkl\n","101_17.mp4-openl3.pkl\n","101_18.mp4-openl3.pkl\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"zsnKegnL4gyL","colab_type":"code","colab":{}},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"t74F7g-72k8U","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1593125914953,"user_tz":240,"elapsed":7880,"user":{"displayName":"Leila Abdelrahman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiiOBijfsdVdWn4yrdWM-_fgxn37DQ4bPiUdT7W=s64","userId":"10595688673230464384"}}},"source":["!cp /content/drive/'My Drive/Machine-Learning-Projects'/cs231n-project/datasets/emotiw/train-final-laugh-prob.pkl .\n","!cp /content/drive/'My Drive/Machine-Learning-Projects'/cs231n-project/datasets/emotiw/val-final-laugh-prob.pkl ."],"execution_count":3,"outputs":[]},{"cell_type":"code","metadata":{"id":"9W9gMIWF5ONl","colab_type":"code","colab":{}},"source":["samples = sorted(train_laugh_obj[\"vids\"])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"I_XuLCf_3SzK","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":52},"executionInfo":{"status":"ok","timestamp":1593125916074,"user_tz":240,"elapsed":1110,"user":{"displayName":"Leila Abdelrahman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiiOBijfsdVdWn4yrdWM-_fgxn37DQ4bPiUdT7W=s64","userId":"10595688673230464384"}},"outputId":"8f70307a-8cda-4497-f61f-f0ba16bf47f0"},"source":["import pickle\n","train_vid_to_laugh = {}\n","val_vid_to_laugh = {}\n","train_laugh_vec = []\n","val_laugh_vec = []\n","with open('train-final-laugh-prob.pkl', 'rb') as handle:\n"," train_laugh_obj = pickle.load(handle)\n"," i = 0\n"," for vid in train_laugh_obj[\"vids\"]:\n"," train_vid_to_laugh[vid] = train_laugh_obj[\"actual_preds\"][i]\n"," i += 1\n"," \n"," for vid in sorted(train_laugh_obj[\"vids\"]):\n"," train_laugh_vec.append(train_vid_to_laugh[vid])\n","\n","import pickle\n","with open('val-final-laugh-prob.pkl', 'rb') as handle:\n"," val_laugh_obj = pickle.load(handle)\n"," i = 0\n"," for vid in val_laugh_obj[\"vids\"]:\n"," val_vid_to_laugh[vid] = val_laugh_obj[\"actual_preds\"][i]\n"," i += 1\n"," for vid in sorted(val_laugh_obj[\"vids\"]):\n"," val_laugh_vec.append(val_vid_to_laugh[vid])\n","\n","print(len(train_laugh_vec))\n","print(len(val_laugh_vec))"],"execution_count":4,"outputs":[{"output_type":"stream","text":["2661\n","766\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"SvJo_G-jp8Au","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":139},"executionInfo":{"status":"ok","timestamp":1593125949608,"user_tz":240,"elapsed":4240,"user":{"displayName":"Leila Abdelrahman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiiOBijfsdVdWn4yrdWM-_fgxn37DQ4bPiUdT7W=s64","userId":"10595688673230464384"}},"outputId":"94bbf42f-a993-4c93-d565-eb282bc2c56a"},"source":["import numpy as np\n","\n","!cp /content/drive/'My Drive/Machine-Learning-Projects'/cs231n-project/datasets/emotiw/ensemble-scene-scene-pose-audio-face-caption-v1.zip .\n","!unzip ensemble-scene-scene-pose-audio-face-caption-v1.zip\n","\n","X_train = np.load(\"ensemble-scene-scene-pose-audio-face-caption-v1/X_train.npy\")\n","y_train = np.load(\"ensemble-scene-scene-pose-audio-face-caption-v1/y_train.npy\")\n","X_valid = np.load(\"ensemble-scene-scene-pose-audio-face-caption-v1/X_valid.npy\")\n","y_valid = np.load(\"ensemble-scene-scene-pose-audio-face-caption-v1/y_valid.npy\")\n","X_train.shape"],"execution_count":8,"outputs":[{"output_type":"stream","text":["Archive: ensemble-scene-scene-pose-audio-face-caption-v1.zip\n"," creating: ensemble-scene-scene-pose-audio-face-caption-v1/\n"," inflating: ensemble-scene-scene-pose-audio-face-caption-v1/X_train.npy \n"," inflating: ensemble-scene-scene-pose-audio-face-caption-v1/y_valid.npy \n"," inflating: ensemble-scene-scene-pose-audio-face-caption-v1/y_train.npy \n"," inflating: ensemble-scene-scene-pose-audio-face-caption-v1/X_valid.npy \n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":["(2661, 254)"]},"metadata":{"tags":[]},"execution_count":8}]},{"cell_type":"code","metadata":{"id":"d4zPhqy06aey","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1593125953119,"user_tz":240,"elapsed":826,"user":{"displayName":"Leila Abdelrahman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiiOBijfsdVdWn4yrdWM-_fgxn37DQ4bPiUdT7W=s64","userId":"10595688673230464384"}},"outputId":"4d750377-10d5-4611-9b64-0a502c83317c"},"source":["len(train_laugh_vec)"],"execution_count":9,"outputs":[{"output_type":"execute_result","data":{"text/plain":["2661"]},"metadata":{"tags":[]},"execution_count":9}]},{"cell_type":"code","metadata":{"id":"bRqCC6cr5xvS","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1593125955572,"user_tz":240,"elapsed":764,"user":{"displayName":"Leila Abdelrahman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiiOBijfsdVdWn4yrdWM-_fgxn37DQ4bPiUdT7W=s64","userId":"10595688673230464384"}}},"source":["# Adding laughter probability as an additional dimension\n","train_laugh_vec = np.expand_dims(train_laugh_vec, 1)\n","val_laugh_vec = np.expand_dims(val_laugh_vec, 1)\n","X_train = np.hstack((X_train, train_laugh_vec))\n","X_valid = np.hstack((X_valid, val_laugh_vec))"],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"id":"OatM9hSd6utC","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1593125956548,"user_tz":240,"elapsed":406,"user":{"displayName":"Leila Abdelrahman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiiOBijfsdVdWn4yrdWM-_fgxn37DQ4bPiUdT7W=s64","userId":"10595688673230464384"}},"outputId":"a623c28b-3048-451f-dce9-a093340bc50b"},"source":["X_train.shape"],"execution_count":11,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(2661, 255)"]},"metadata":{"tags":[]},"execution_count":11}]},{"cell_type":"markdown","metadata":{"id":"0Kstcqbyrq-x","colab_type":"text"},"source":["## Combine Modes Together"]},{"cell_type":"code","metadata":{"id":"Lu9NquN3rttq","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1593129220373,"user_tz":240,"elapsed":768,"user":{"displayName":"Leila Abdelrahman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiiOBijfsdVdWn4yrdWM-_fgxn37DQ4bPiUdT7W=s64","userId":"10595688673230464384"}},"outputId":"e4fb86ba-fd40-4b0e-c555-104a4a0b0ae6"},"source":["sizes = [32, 30, 40, 128, 8, 16, 1]\n","sizes = [32, 62, 102, 230, 238, 254, 255]\n","\n","X_train_scene_1 = X_train[: , :sizes[0]]\n","X_train_scene_2 = X_train[: , sizes[0]:sizes[1]]\n","X_train_pose = X_train[: , sizes[1]:sizes[2]]\n","X_train_audio = X_train[: , sizes[2]:sizes[3]]\n","X_train_face = X_train[: , sizes[3]:sizes[4]]\n","X_train_caption = X_train[: , sizes[4]:sizes[5]]\n","X_train_laugh = X_train[: , sizes[5]:]\n","\n","\n","X_val_scene_1 = X_valid[: , :sizes[0]]\n","X_val_scene_2 = X_valid[: , sizes[0]:sizes[1]]\n","X_val_pose = X_valid[: , sizes[1]:sizes[2]]\n","X_val_audio = X_valid[: , sizes[2]:sizes[3]]\n","X_val_face = X_valid[: , sizes[3]:sizes[4]]\n","X_val_caption = X_valid[: , sizes[4]:sizes[5]]\n","X_val_laugh = X_valid[: , sizes[5]:]\n"],"execution_count":78,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(2661, 8)"]},"metadata":{"tags":[]},"execution_count":78}]},{"cell_type":"markdown","metadata":{"id":"BLwjjNGUyote","colab_type":"text"},"source":["Combine the modalities"]},{"cell_type":"code","metadata":{"id":"IjDR9uXCyapt","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1593131081309,"user_tz":240,"elapsed":814,"user":{"displayName":"Leila Abdelrahman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiiOBijfsdVdWn4yrdWM-_fgxn37DQ4bPiUdT7W=s64","userId":"10595688673230464384"}}},"source":["pose_audio_train_laugh = np.concatenate((X_train_audio , X_train_pose , X_train_laugh) , axis=1)\n","pose_audio_val_laugh = np.concatenate((X_val_audio , X_val_pose , X_val_laugh) , axis=1)\n","\n","audio_train_laugh = np.concatenate((X_train_audio , X_train_laugh) , axis=1)\n","audio_val_laugh = np.concatenate((X_val_audio , X_val_laugh) , axis=1)\n","\n","pose_scene_caption_train = np.concatenate((X_train_pose , X_train_scene_2 , X_train_caption) , axis=1)\n","pose_scene_caption_val = np.concatenate((X_val_pose , X_val_scene_2 , X_val_caption) , axis=1)\n","\n","scene_caption_train = np.concatenate((X_train_scene_2 , X_train_caption) , axis=1)\n","scene_caption_val = np.concatenate((X_val_scene_2 , X_val_caption) , axis=1)\n","\n","\n","laugh_caption_train = np.concatenate((X_train_laugh , X_train_caption) , axis=1)\n","laugh_caption_val = np.concatenate((X_val_laugh, X_val_caption) , axis=1)"],"execution_count":117,"outputs":[]},{"cell_type":"code","metadata":{"id":"utC2w87Bz4-c","colab_type":"code","colab":{}},"source":["# Create Model\n","import tensorflow as tf\n","inputs_1 = tf.keras.layers.Input(shape=[audio_train_laugh.shape[1]])\n","inputs_2 = tf.keras.layers.Input(shape=[scene_caption_train.shape[1]])\n","\n","## The pose, audio , laughter model \n","x_1 = tf.keras.layers.BatchNormalization()(inputs_1)\n","x_1 = tf.keras.layers.Dense(64, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.02))(x_1)\n","\n","## The scene , caption\n","x_2 = tf.keras.layers.BatchNormalization()(inputs_2)\n","x_2 = tf.keras.layers.Dense(64, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.02))(x_2)\n","\n","x_3 = tf.keras.layers.Concatenate()([x_1 , x_2])\n","\n","outputs = tf.keras.layers.Dense(3, activation='softmax', kernel_regularizer=tf.keras.regularizers.l2(0.02))(x_3)\n","\n","model = tf.keras.Model(inputs=[inputs_1 , inputs_2] , outputs=outputs)\n","model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001),\n"," loss = 'categorical_crossentropy',\n"," metrics=['accuracy'])"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"kWl9GJ6H7KJi","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1593131134745,"user_tz":240,"elapsed":753,"user":{"displayName":"Leila Abdelrahman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiiOBijfsdVdWn4yrdWM-_fgxn37DQ4bPiUdT7W=s64","userId":"10595688673230464384"}}},"source":["# Create Model\n","import tensorflow as tf\n","inputs_1 = keras.layers.Input(shape=[audio_train_laugh.shape[1]])\n","inputs_2 = keras.layers.Input(shape=[scene_caption_train.shape[1]])\n","\n","## The pose, audio , laughter model \n","x_1 = keras.layers.BatchNormalization()(inputs_1)\n","#x_1 = keras.layers.Dense(32, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.001))(x_1)\n","x_1 = keras.layers.Dense(64, activation='relu', kernel_regularizer=keras.regularizers.l2(0.02))(x_1)\n","\n","\n","## The scene , caption\n","x_2 = keras.layers.BatchNormalization()(inputs_2)\n","#x_2 = keras.layers.Dense(32, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.001))(x_2)\n","x_2 = keras.layers.Dense(64, activation='relu', kernel_regularizer=keras.regularizers.l2(0.02))(x_2)\n","\n","x_3 = keras.layers.Concatenate()([x_1 , x_2])\n","\n","outputs = keras.layers.Dense(3, activation='softmax', kernel_regularizer=keras.regularizers.l2(0.02))(x_3)\n","\n","model = keras.Model(inputs=[inputs_1 , inputs_2] , outputs=outputs)\n","model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001),\n"," loss = 'categorical_crossentropy',\n"," metrics=['accuracy'])"],"execution_count":120,"outputs":[]},{"cell_type":"code","metadata":{"id":"7t3kYz_C7kzI","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"status":"ok","timestamp":1593131152780,"user_tz":240,"elapsed":17236,"user":{"displayName":"Leila Abdelrahman","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GiiOBijfsdVdWn4yrdWM-_fgxn37DQ4bPiUdT7W=s64","userId":"10595688673230464384"}},"outputId":"2c97548b-9a64-4add-db3e-338a75b72f40"},"source":["model.fit([audio_train_laugh , scene_caption_train] , y_train , validation_data=[[audio_val_laugh , scene_caption_val] , y_valid] , epochs=50)"],"execution_count":121,"outputs":[{"output_type":"stream","text":["Train on 2661 samples, validate on 766 samples\n","Epoch 1/50\n","2661/2661 [==============================] - 1s 190us/step - loss: 2.9855 - accuracy: 0.6712 - val_loss: 2.7196 - val_accuracy: 0.5431\n","Epoch 2/50\n","2661/2661 [==============================] - 0s 118us/step - loss: 1.7767 - accuracy: 0.7659 - val_loss: 1.8316 - val_accuracy: 0.5548\n","Epoch 3/50\n","2661/2661 [==============================] - 0s 131us/step - loss: 1.2226 - accuracy: 0.7734 - val_loss: 1.4142 - val_accuracy: 0.5718\n","Epoch 4/50\n","2661/2661 [==============================] - 0s 118us/step - loss: 0.9285 - accuracy: 0.7850 - val_loss: 1.2253 - val_accuracy: 0.5718\n","Epoch 5/50\n","2661/2661 [==============================] - 0s 117us/step - loss: 0.7898 - accuracy: 0.7847 - val_loss: 1.1199 - val_accuracy: 0.5731\n","Epoch 6/50\n","2661/2661 [==============================] - 0s 115us/step - loss: 0.7192 - accuracy: 0.7820 - val_loss: 1.0573 - val_accuracy: 0.5718\n","Epoch 7/50\n","2661/2661 [==============================] - 0s 119us/step - loss: 0.6728 - accuracy: 0.7948 - val_loss: 1.0341 - val_accuracy: 0.5796\n","Epoch 8/50\n","2661/2661 [==============================] - 0s 111us/step - loss: 0.6619 - accuracy: 0.7832 - val_loss: 1.0268 - val_accuracy: 0.5770\n","Epoch 9/50\n","2661/2661 [==============================] - 0s 120us/step - loss: 0.6388 - accuracy: 0.7922 - val_loss: 1.0120 - val_accuracy: 0.5770\n","Epoch 10/50\n","2661/2661 [==============================] - 0s 129us/step - loss: 0.6354 - accuracy: 0.7896 - val_loss: 0.9881 - val_accuracy: 0.5731\n","Epoch 11/50\n","2661/2661 [==============================] - 0s 118us/step - loss: 0.6292 - accuracy: 0.7896 - val_loss: 0.9842 - val_accuracy: 0.5901\n","Epoch 12/50\n","2661/2661 [==============================] - 0s 112us/step - loss: 0.6197 - accuracy: 0.7918 - val_loss: 0.9996 - val_accuracy: 0.5836\n","Epoch 13/50\n","2661/2661 [==============================] - 0s 119us/step - loss: 0.6180 - accuracy: 0.7843 - val_loss: 0.9949 - val_accuracy: 0.5770\n","Epoch 14/50\n","2661/2661 [==============================] - 0s 114us/step - loss: 0.6125 - accuracy: 0.7896 - val_loss: 0.9843 - val_accuracy: 0.5796\n","Epoch 15/50\n","2661/2661 [==============================] - 0s 121us/step - loss: 0.6130 - accuracy: 0.7922 - val_loss: 0.9963 - val_accuracy: 0.5836\n","Epoch 16/50\n","2661/2661 [==============================] - 0s 123us/step - loss: 0.6117 - accuracy: 0.7884 - val_loss: 1.0248 - val_accuracy: 0.5744\n","Epoch 17/50\n","2661/2661 [==============================] - 0s 114us/step - loss: 0.6059 - accuracy: 0.7911 - val_loss: 1.0013 - val_accuracy: 0.5640\n","Epoch 18/50\n","2661/2661 [==============================] - 0s 114us/step - loss: 0.6094 - accuracy: 0.7850 - val_loss: 0.9893 - val_accuracy: 0.5875\n","Epoch 19/50\n","2661/2661 [==============================] - 0s 120us/step - loss: 0.5983 - accuracy: 0.7884 - val_loss: 0.9879 - val_accuracy: 0.6044\n","Epoch 20/50\n","2661/2661 [==============================] - 0s 122us/step - loss: 0.6017 - accuracy: 0.7959 - val_loss: 1.0014 - val_accuracy: 0.5770\n","Epoch 21/50\n","2661/2661 [==============================] - 0s 112us/step - loss: 0.6014 - accuracy: 0.7869 - val_loss: 0.9839 - val_accuracy: 0.5757\n","Epoch 22/50\n","2661/2661 [==============================] - 0s 115us/step - loss: 0.6041 - accuracy: 0.7835 - val_loss: 0.9878 - val_accuracy: 0.5809\n","Epoch 23/50\n","2661/2661 [==============================] - 0s 122us/step - loss: 0.6002 - accuracy: 0.7918 - val_loss: 0.9710 - val_accuracy: 0.5849\n","Epoch 24/50\n","2661/2661 [==============================] - 0s 115us/step - loss: 0.6012 - accuracy: 0.7865 - val_loss: 0.9852 - val_accuracy: 0.5822\n","Epoch 25/50\n","2661/2661 [==============================] - 0s 113us/step - loss: 0.5921 - accuracy: 0.7914 - val_loss: 0.9983 - val_accuracy: 0.5705\n","Epoch 26/50\n","2661/2661 [==============================] - 0s 122us/step - loss: 0.5924 - accuracy: 0.7963 - val_loss: 0.9971 - val_accuracy: 0.5731\n","Epoch 27/50\n","2661/2661 [==============================] - 0s 113us/step - loss: 0.6008 - accuracy: 0.7813 - val_loss: 0.9993 - val_accuracy: 0.5822\n","Epoch 28/50\n","2661/2661 [==============================] - 0s 117us/step - loss: 0.5986 - accuracy: 0.7854 - val_loss: 0.9976 - val_accuracy: 0.5966\n","Epoch 29/50\n","2661/2661 [==============================] - 0s 126us/step - loss: 0.5980 - accuracy: 0.7862 - val_loss: 0.9834 - val_accuracy: 0.5796\n","Epoch 30/50\n","2661/2661 [==============================] - 0s 117us/step - loss: 0.6035 - accuracy: 0.7877 - val_loss: 0.9903 - val_accuracy: 0.5875\n","Epoch 31/50\n","2661/2661 [==============================] - 0s 114us/step - loss: 0.5920 - accuracy: 0.7869 - val_loss: 0.9991 - val_accuracy: 0.5731\n","Epoch 32/50\n","2661/2661 [==============================] - 0s 120us/step - loss: 0.5882 - accuracy: 0.7903 - val_loss: 0.9734 - val_accuracy: 0.5888\n","Epoch 33/50\n","2661/2661 [==============================] - 0s 115us/step - loss: 0.5928 - accuracy: 0.7884 - val_loss: 0.9973 - val_accuracy: 0.5809\n","Epoch 34/50\n","2661/2661 [==============================] - 0s 117us/step - loss: 0.5914 - accuracy: 0.7896 - val_loss: 1.0069 - val_accuracy: 0.5979\n","Epoch 35/50\n","2661/2661 [==============================] - 0s 118us/step - loss: 0.5923 - accuracy: 0.7922 - val_loss: 0.9962 - val_accuracy: 0.5849\n","Epoch 36/50\n","2661/2661 [==============================] - 0s 119us/step - loss: 0.5977 - accuracy: 0.7892 - val_loss: 1.0262 - val_accuracy: 0.5783\n","Epoch 37/50\n","2661/2661 [==============================] - 0s 119us/step - loss: 0.5907 - accuracy: 0.7869 - val_loss: 1.0049 - val_accuracy: 0.5914\n","Epoch 38/50\n","2661/2661 [==============================] - 0s 115us/step - loss: 0.5819 - accuracy: 0.7918 - val_loss: 0.9792 - val_accuracy: 0.5809\n","Epoch 39/50\n","2661/2661 [==============================] - 0s 117us/step - loss: 0.5909 - accuracy: 0.7911 - val_loss: 0.9866 - val_accuracy: 0.5849\n","Epoch 40/50\n","2661/2661 [==============================] - 0s 112us/step - loss: 0.5880 - accuracy: 0.7944 - val_loss: 0.9673 - val_accuracy: 0.5953\n","Epoch 41/50\n","2661/2661 [==============================] - 0s 115us/step - loss: 0.6013 - accuracy: 0.7790 - val_loss: 0.9990 - val_accuracy: 0.5822\n","Epoch 42/50\n","2661/2661 [==============================] - 0s 122us/step - loss: 0.5862 - accuracy: 0.7929 - val_loss: 0.9746 - val_accuracy: 0.5862\n","Epoch 43/50\n","2661/2661 [==============================] - 0s 124us/step - loss: 0.5887 - accuracy: 0.7862 - val_loss: 0.9567 - val_accuracy: 0.6018\n","Epoch 44/50\n","2661/2661 [==============================] - 0s 115us/step - loss: 0.5806 - accuracy: 0.7971 - val_loss: 0.9945 - val_accuracy: 0.5901\n","Epoch 45/50\n","2661/2661 [==============================] - 0s 125us/step - loss: 0.5915 - accuracy: 0.7798 - val_loss: 0.9674 - val_accuracy: 0.5770\n","Epoch 46/50\n","2661/2661 [==============================] - 0s 118us/step - loss: 0.5924 - accuracy: 0.7865 - val_loss: 0.9853 - val_accuracy: 0.5679\n","Epoch 47/50\n","2661/2661 [==============================] - 0s 114us/step - loss: 0.5888 - accuracy: 0.7869 - val_loss: 0.9669 - val_accuracy: 0.5927\n","Epoch 48/50\n","2661/2661 [==============================] - 0s 122us/step - loss: 0.5868 - accuracy: 0.7922 - val_loss: 0.9807 - val_accuracy: 0.5744\n","Epoch 49/50\n","2661/2661 [==============================] - 0s 113us/step - loss: 0.5824 - accuracy: 0.7847 - val_loss: 1.0026 - val_accuracy: 0.5836\n","Epoch 50/50\n","2661/2661 [==============================] - 0s 116us/step - loss: 0.5857 - accuracy: 0.7911 - val_loss: 0.9703 - val_accuracy: 0.5783\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/plain":[""]},"metadata":{"tags":[]},"execution_count":121}]},{"cell_type":"code","metadata":{"id":"y8rAjjncdrrJ","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"status":"ok","timestamp":1593064789386,"user_tz":420,"elapsed":18474,"user":{"displayName":"Tom Jin","photoUrl":"","userId":"10159799883240069996"}},"outputId":"7523ef11-4757-49cd-db9e-ca0a9343316a"},"source":["#\n","# CONFIGURATION\n","#\n","# Define any constants for the model here\n","#\n","\n","MODEL_NAME = \"ensemble-scene-scene-pose-audio-face-caption-laugh-v1\"\n","\n","from pathlib import Path\n","import tensorflow as tf\n","import matplotlib.pyplot as plt\n","\n","sizes = [32, 30, 40, 128, 8, 16, 1]\n","\n","# # UNCOMMENT IF EXCLUDING FER, IMAGE CAP, AND VGG\n","# mask = []\n","# for x in range(sum(sizes)):\n","# if x >= 32 and x < 62:\n","# mask.append(False)\n","# elif x < 230:\n","# mask.append(True)\n","# else:\n","# mask.append(False)\n","\n","# # UNCOMMENT IF EXCLUDING FER, IMAGE CAP, POSE, AND RESNET\n","# mask = []\n","# for x in range(sum(sizes)):\n","# if x >= 62 and x < 102:\n","# mask.append(False)\n","# elif x < 62:\n","# mask.append(True)\n","# else:\n","# mask.append(False)\n"," \n","# # UNCOMMENT IF EXCLUDING FER, AND VGG\n","# mask = []\n","# for x in range(sum(sizes)):\n","# if x >= 30 and x < 62:\n","# mask.append(False)\n","# elif x < 230:\n","# mask.append(True)\n","# elif x >= 238:\n","# mask.append(True)\n","# else:\n","# mask.append(False)\n","\n","# UNCOMMENT IF EXCLUDING FER, AND RESNET [best]\n","mask = []\n","for x in range(sum(sizes)):\n"," if x >= 62 and x < 102:\n"," mask.append(False)\n"," elif x < 230:\n"," mask.append(True)\n"," elif x >= 238:\n"," mask.append(True)\n"," else:\n"," mask.append(False)\n","\n","# # UNCOMMENT IF EXCLUDING FER, IMAGE CAP, AND RESNET [best]\n","# mask = []\n","# for x in range(sum(sizes)):\n","# if x >= 62 and x < 102:\n","# mask.append(False)\n","# elif x < 230:\n","# mask.append(True)\n","# else:\n","# mask.append(False)\n","\n","# UNCOMMENT IF EXCLUDING FER AND IMAGE CAP\n","# mask = []\n","# for x in range(sum(sizes)):\n","# if x < 230:\n","# mask.append(True)\n","# else:\n","# mask.append(False)\n","\n","# # UNCOMMENT IF EXCLUDING FER\n","# mask = []\n","# for x in range(sum(sizes)):\n","# if x < 230:\n","# mask.append(True)\n","# elif x >= 238:\n","# mask.append(True)\n","# else:\n","# mask.append(False)\n","\n","\n","def build_model():\n"," def create_model(inputs):\n"," # x = tf.keras.layers.Dense(hp.Int('units', min_value=8, max_value=128, step=8), activation='relu', kernel_regularizer=tf.keras.regularizers.l2())(inputs)\n"," # x = tf.keras.layers.Dense(8, activation='relu')(inputs)\n"," # x = tf.keras.layers.Dropout(0.3)(x)\n"," x = tf.keras.layers.BatchNormalization()(inputs)\n"," # x = tf.keras.layers.Dense(64, activation='relu')(inputs)\n"," # x = tf.keras.layers.Dropout(0.2)(x)\n"," # x = tf.keras.layers.Dense(32, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.001))(x)\n"," x = tf.keras.layers.Dense(64, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(0.02))(x)\n"," x = tf.keras.layers.Dense(3, activation='softmax', kernel_regularizer=tf.keras.regularizers.l2(0.02))(x)\n","\n"," model = tf.keras.Model(inputs=inputs, outputs=x)\n"," model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.001),\n"," loss = 'categorical_crossentropy',\n"," metrics=['accuracy'])\n"," return model\n","\n"," # inputs = tf.keras.Input(shape=(X_train.shape[1]))\n"," inputs = tf.keras.Input(shape=(np.count_nonzero(mask)))\n"," model = create_model(inputs)\n"," # model.summary()\n"," return model\n","\n","Path(f\"/content/drive/My Drive/cs231n-project/models/{MODEL_NAME}\").mkdir(parents=True, exist_ok=True)\n","checkpoint_path = \"/content/drive/My Drive/cs231n-project/models/\" + MODEL_NAME + \"/cp-{epoch:04d}.h5\"\n","cp_callback = tf.keras.callbacks.ModelCheckpoint(\n"," filepath=checkpoint_path, \n"," verbose=1,\n"," save_weights_only=False,\n"," save_best_only=False,\n"," period=1\n",")\n","model = build_model()\n","\n","import pickle\n","history = model.fit(\n"," x=X_train[:, mask],\n"," y=y_train,\n"," epochs=50,\n"," callbacks=[cp_callback],\n"," validation_data=(X_valid[:, mask], y_valid)\n",")\n","\n"],"execution_count":null,"outputs":[{"output_type":"stream","text":["WARNING:tensorflow:`period` argument is deprecated. Please use `save_freq` to specify the frequency in number of batches seen.\n","Epoch 1/50\n","81/84 [===========================>..] - ETA: 0s - loss: 2.2381 - accuracy: 0.7593\n","Epoch 00001: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0001.h5\n","84/84 [==============================] - 0s 5ms/step - loss: 2.2256 - accuracy: 0.7614 - val_loss: 2.1822 - val_accuracy: 0.5875\n","Epoch 2/50\n","82/84 [============================>.] - ETA: 0s - loss: 1.3872 - accuracy: 0.8365\n","Epoch 00002: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0002.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 1.3857 - accuracy: 0.8354 - val_loss: 1.6229 - val_accuracy: 0.5888\n","Epoch 3/50\n","83/84 [============================>.] - ETA: 0s - loss: 0.9917 - accuracy: 0.8309\n","Epoch 00003: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0003.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.9910 - accuracy: 0.8313 - val_loss: 1.3493 - val_accuracy: 0.6005\n","Epoch 4/50\n","80/84 [===========================>..] - ETA: 0s - loss: 0.7881 - accuracy: 0.8301\n","Epoch 00004: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0004.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.7867 - accuracy: 0.8305 - val_loss: 1.1943 - val_accuracy: 0.6057\n","Epoch 5/50\n","83/84 [============================>.] - ETA: 0s - loss: 0.6698 - accuracy: 0.8343\n","Epoch 00005: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0005.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.6692 - accuracy: 0.8346 - val_loss: 1.1344 - val_accuracy: 0.6123\n","Epoch 6/50\n","63/84 [=====================>........] - ETA: 0s - loss: 0.6194 - accuracy: 0.8264\n","Epoch 00006: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0006.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.6016 - accuracy: 0.8346 - val_loss: 1.0859 - val_accuracy: 0.6149\n","Epoch 7/50\n","82/84 [============================>.] - ETA: 0s - loss: 0.5591 - accuracy: 0.8399\n","Epoch 00007: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0007.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.5590 - accuracy: 0.8388 - val_loss: 1.0658 - val_accuracy: 0.6162\n","Epoch 8/50\n","66/84 [======================>.......] - ETA: 0s - loss: 0.5403 - accuracy: 0.8366\n","Epoch 00008: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0008.h5\n","84/84 [==============================] - 0s 3ms/step - loss: 0.5403 - accuracy: 0.8377 - val_loss: 1.0413 - val_accuracy: 0.6084\n","Epoch 9/50\n","64/84 [=====================>........] - ETA: 0s - loss: 0.5370 - accuracy: 0.8330\n","Epoch 00009: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0009.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.5300 - accuracy: 0.8384 - val_loss: 1.0618 - val_accuracy: 0.6162\n","Epoch 10/50\n","80/84 [===========================>..] - ETA: 0s - loss: 0.5178 - accuracy: 0.8344\n","Epoch 00010: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0010.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.5180 - accuracy: 0.8346 - val_loss: 1.0066 - val_accuracy: 0.6136\n","Epoch 11/50\n","66/84 [======================>.......] - ETA: 0s - loss: 0.5217 - accuracy: 0.8310\n","Epoch 00011: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0011.h5\n","84/84 [==============================] - 0s 3ms/step - loss: 0.5185 - accuracy: 0.8335 - val_loss: 1.0230 - val_accuracy: 0.6057\n","Epoch 12/50\n","69/84 [=======================>......] - ETA: 0s - loss: 0.5018 - accuracy: 0.8329\n","Epoch 00012: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0012.h5\n","84/84 [==============================] - 0s 3ms/step - loss: 0.5040 - accuracy: 0.8324 - val_loss: 1.0345 - val_accuracy: 0.6110\n","Epoch 13/50\n","84/84 [==============================] - ETA: 0s - loss: 0.5075 - accuracy: 0.8335\n","Epoch 00013: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0013.h5\n","84/84 [==============================] - 0s 5ms/step - loss: 0.5075 - accuracy: 0.8335 - val_loss: 1.0213 - val_accuracy: 0.6227\n","Epoch 14/50\n","67/84 [======================>.......] - ETA: 0s - loss: 0.5087 - accuracy: 0.8293\n","Epoch 00014: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0014.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.5039 - accuracy: 0.8301 - val_loss: 1.0028 - val_accuracy: 0.6266\n","Epoch 15/50\n","63/84 [=====================>........] - ETA: 0s - loss: 0.5135 - accuracy: 0.8264\n","Epoch 00015: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0015.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.5017 - accuracy: 0.8320 - val_loss: 1.0107 - val_accuracy: 0.6162\n","Epoch 16/50\n","67/84 [======================>.......] - ETA: 0s - loss: 0.5019 - accuracy: 0.8368\n","Epoch 00016: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0016.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4997 - accuracy: 0.8395 - val_loss: 0.9776 - val_accuracy: 0.6123\n","Epoch 17/50\n","64/84 [=====================>........] - ETA: 0s - loss: 0.4958 - accuracy: 0.8296\n","Epoch 00017: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0017.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4926 - accuracy: 0.8324 - val_loss: 0.9885 - val_accuracy: 0.6175\n","Epoch 18/50\n","63/84 [=====================>........] - ETA: 0s - loss: 0.4714 - accuracy: 0.8452\n","Epoch 00018: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0018.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4967 - accuracy: 0.8365 - val_loss: 0.9568 - val_accuracy: 0.6371\n","Epoch 19/50\n","84/84 [==============================] - ETA: 0s - loss: 0.4989 - accuracy: 0.8279\n","Epoch 00019: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0019.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4989 - accuracy: 0.8279 - val_loss: 0.9771 - val_accuracy: 0.6253\n","Epoch 20/50\n","63/84 [=====================>........] - ETA: 0s - loss: 0.5026 - accuracy: 0.8254\n","Epoch 00020: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0020.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4889 - accuracy: 0.8339 - val_loss: 0.9889 - val_accuracy: 0.6253\n","Epoch 21/50\n","66/84 [======================>.......] - ETA: 0s - loss: 0.4834 - accuracy: 0.8404\n","Epoch 00021: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0021.h5\n","84/84 [==============================] - 0s 3ms/step - loss: 0.4885 - accuracy: 0.8384 - val_loss: 0.9400 - val_accuracy: 0.6358\n","Epoch 22/50\n","81/84 [===========================>..] - ETA: 0s - loss: 0.4798 - accuracy: 0.8445\n","Epoch 00022: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0022.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4816 - accuracy: 0.8437 - val_loss: 1.0211 - val_accuracy: 0.6149\n","Epoch 23/50\n","82/84 [============================>.] - ETA: 0s - loss: 0.4890 - accuracy: 0.8365\n","Epoch 00023: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0023.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4890 - accuracy: 0.8362 - val_loss: 0.9948 - val_accuracy: 0.6279\n","Epoch 24/50\n","66/84 [======================>.......] - ETA: 0s - loss: 0.4891 - accuracy: 0.8423\n","Epoch 00024: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0024.h5\n","84/84 [==============================] - 0s 3ms/step - loss: 0.4969 - accuracy: 0.8346 - val_loss: 0.9743 - val_accuracy: 0.6123\n","Epoch 25/50\n","68/84 [=======================>......] - ETA: 0s - loss: 0.5075 - accuracy: 0.8277\n","Epoch 00025: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0025.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4985 - accuracy: 0.8298 - val_loss: 1.0207 - val_accuracy: 0.6253\n","Epoch 26/50\n","65/84 [======================>.......] - ETA: 0s - loss: 0.4711 - accuracy: 0.8471\n","Epoch 00026: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0026.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4748 - accuracy: 0.8448 - val_loss: 0.9863 - val_accuracy: 0.6279\n","Epoch 27/50\n","66/84 [======================>.......] - ETA: 0s - loss: 0.5012 - accuracy: 0.8220\n","Epoch 00027: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0027.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4892 - accuracy: 0.8260 - val_loss: 1.0326 - val_accuracy: 0.6279\n","Epoch 28/50\n","68/84 [=======================>......] - ETA: 0s - loss: 0.4912 - accuracy: 0.8318\n","Epoch 00028: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0028.h5\n","84/84 [==============================] - 0s 3ms/step - loss: 0.4852 - accuracy: 0.8369 - val_loss: 0.9891 - val_accuracy: 0.6240\n","Epoch 29/50\n","83/84 [============================>.] - ETA: 0s - loss: 0.4755 - accuracy: 0.8366\n","Epoch 00029: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0029.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4749 - accuracy: 0.8369 - val_loss: 1.0234 - val_accuracy: 0.6214\n","Epoch 30/50\n","78/84 [==========================>...] - ETA: 0s - loss: 0.4761 - accuracy: 0.8345\n","Epoch 00030: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0030.h5\n","84/84 [==============================] - 0s 6ms/step - loss: 0.4757 - accuracy: 0.8343 - val_loss: 1.0163 - val_accuracy: 0.6110\n","Epoch 31/50\n","84/84 [==============================] - ETA: 0s - loss: 0.4811 - accuracy: 0.8354\n","Epoch 00031: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0031.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4811 - accuracy: 0.8354 - val_loss: 0.9816 - val_accuracy: 0.6175\n","Epoch 32/50\n","83/84 [============================>.] - ETA: 0s - loss: 0.4861 - accuracy: 0.8400\n","Epoch 00032: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0032.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4862 - accuracy: 0.8399 - val_loss: 0.9874 - val_accuracy: 0.6371\n","Epoch 33/50\n","68/84 [=======================>......] - ETA: 0s - loss: 0.4868 - accuracy: 0.8323\n","Epoch 00033: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0033.h5\n","84/84 [==============================] - 0s 3ms/step - loss: 0.4862 - accuracy: 0.8324 - val_loss: 0.9726 - val_accuracy: 0.6031\n","Epoch 34/50\n","83/84 [============================>.] - ETA: 0s - loss: 0.4820 - accuracy: 0.8347\n","Epoch 00034: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0034.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4817 - accuracy: 0.8350 - val_loss: 1.0087 - val_accuracy: 0.6188\n","Epoch 35/50\n","77/84 [==========================>...] - ETA: 0s - loss: 0.4811 - accuracy: 0.8308\n","Epoch 00035: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0035.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4795 - accuracy: 0.8331 - val_loss: 0.9782 - val_accuracy: 0.6305\n","Epoch 36/50\n","66/84 [======================>.......] - ETA: 0s - loss: 0.4718 - accuracy: 0.8447\n","Epoch 00036: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0036.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4747 - accuracy: 0.8452 - val_loss: 0.9584 - val_accuracy: 0.6397\n","Epoch 37/50\n","66/84 [======================>.......] - ETA: 0s - loss: 0.4706 - accuracy: 0.8419\n","Epoch 00037: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0037.h5\n","84/84 [==============================] - 0s 3ms/step - loss: 0.4760 - accuracy: 0.8395 - val_loss: 0.9995 - val_accuracy: 0.6136\n","Epoch 38/50\n","81/84 [===========================>..] - ETA: 0s - loss: 0.4849 - accuracy: 0.8329\n","Epoch 00038: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0038.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4816 - accuracy: 0.8339 - val_loss: 0.9746 - val_accuracy: 0.6266\n","Epoch 39/50\n","84/84 [==============================] - ETA: 0s - loss: 0.4821 - accuracy: 0.8369\n","Epoch 00039: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0039.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4821 - accuracy: 0.8369 - val_loss: 0.9933 - val_accuracy: 0.6292\n","Epoch 40/50\n","67/84 [======================>.......] - ETA: 0s - loss: 0.4780 - accuracy: 0.8288\n","Epoch 00040: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0040.h5\n","84/84 [==============================] - 0s 3ms/step - loss: 0.4732 - accuracy: 0.8335 - val_loss: 0.9713 - val_accuracy: 0.6227\n","Epoch 41/50\n","74/84 [=========================>....] - ETA: 0s - loss: 0.4725 - accuracy: 0.8332\n","Epoch 00041: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0041.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4803 - accuracy: 0.8313 - val_loss: 1.0093 - val_accuracy: 0.6292\n","Epoch 42/50\n","79/84 [===========================>..] - ETA: 0s - loss: 0.4798 - accuracy: 0.8331\n","Epoch 00042: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0042.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4875 - accuracy: 0.8313 - val_loss: 1.0273 - val_accuracy: 0.6162\n","Epoch 43/50\n","78/84 [==========================>...] - ETA: 0s - loss: 0.4759 - accuracy: 0.8313\n","Epoch 00043: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0043.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4764 - accuracy: 0.8328 - val_loss: 0.9755 - val_accuracy: 0.6332\n","Epoch 44/50\n","65/84 [======================>.......] - ETA: 0s - loss: 0.4671 - accuracy: 0.8375\n","Epoch 00044: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0044.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4775 - accuracy: 0.8320 - val_loss: 0.9996 - val_accuracy: 0.6188\n","Epoch 45/50\n","63/84 [=====================>........] - ETA: 0s - loss: 0.5026 - accuracy: 0.8229\n","Epoch 00045: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0045.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4870 - accuracy: 0.8324 - val_loss: 1.0069 - val_accuracy: 0.6149\n","Epoch 46/50\n","82/84 [============================>.] - ETA: 0s - loss: 0.4730 - accuracy: 0.8396\n","Epoch 00046: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0046.h5\n","84/84 [==============================] - 0s 5ms/step - loss: 0.4747 - accuracy: 0.8380 - val_loss: 0.9832 - val_accuracy: 0.6292\n","Epoch 47/50\n","78/84 [==========================>...] - ETA: 0s - loss: 0.4630 - accuracy: 0.8438\n","Epoch 00047: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0047.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4667 - accuracy: 0.8429 - val_loss: 0.9919 - val_accuracy: 0.6188\n","Epoch 48/50\n","63/84 [=====================>........] - ETA: 0s - loss: 0.4754 - accuracy: 0.8328\n","Epoch 00048: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0048.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4656 - accuracy: 0.8358 - val_loss: 1.0127 - val_accuracy: 0.6136\n","Epoch 49/50\n","64/84 [=====================>........] - ETA: 0s - loss: 0.4691 - accuracy: 0.8433\n","Epoch 00049: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0049.h5\n","84/84 [==============================] - 0s 3ms/step - loss: 0.4784 - accuracy: 0.8373 - val_loss: 0.9899 - val_accuracy: 0.6305\n","Epoch 50/50\n","66/84 [======================>.......] - ETA: 0s - loss: 0.4678 - accuracy: 0.8395\n","Epoch 00050: saving model to /content/drive/My Drive/cs231n-project/models/ensemble-scene-scene-pose-audio-face-caption-laugh-v1/cp-0050.h5\n","84/84 [==============================] - 0s 4ms/step - loss: 0.4718 - accuracy: 0.8362 - val_loss: 0.9773 - val_accuracy: 0.6084\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"E7Azhd_udPvI","colab_type":"code","colab":{}},"source":[""],"execution_count":null,"outputs":[]}]}