vae-gan.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow.keras as keras\n",
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import datetime\n",
    "import sys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.config.experimental.list_physical_devices('GPU')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Input dataset\n",
    "Dataset used is celebA : https://www.kaggle.com/jessicali9530/celeba-dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(64, 64, 3)\n",
      "len dataset :  202599\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD7CAYAAACscuKmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO19aZQlV3HmF5lvqb2r967uFmrJaAMZWiAjCRlohAVCBmSPjY8tYDS2bLHZxgNjFs8cj42xB/DBxh5jzrQBo+Ox2WxjyRoMCFliM0hqLYCkRmohpN67equu/S2Zd37U67wR8V5mZXdVvWop4zunTt18976bN2/mfRlxI+ILcs7BYDA88xEs9wAMBkN3YIvdYCgIbLEbDAWBLXaDoSCwxW4wFAS22A2GgmBBi52IriGiR4nocSJ672INymAwLD7odO3sRBQCeAzA1QD2ArgXwK845x5ZvOEZDIbFQmkB330RgMedc08AABF9FsB1AFIXe19fj1sx3D9/z04fEjtglUTIC0Lntk6fLHMcKW3b2mV0ycbf9kMrjqljce6QOrfrcJg+kPT+05DVTF+zuLaMFwq/hXlvp0OsPvBfbD8Vf17Sz0WUMacu9QBp94lICs28/6Ctzh+HYTkpl0oV0W54eDgpHz9+TNQ1GrXW52OYmprqOJMLWeybAOxhx3sBXJb1hRXD/bjxpmtaR3I8ccxuWKzqojApN5vNpByGISTStZIw9JfK+49dU7SL2YOkF2MURR3rdLssaalR9+drNCJR12zyYz9GfZ0B+WsJAlknH9oMLc35Osq5yoJA9sd//JyT19JoNJJyHPM5lnMThr7PckmOgwLflsjflyhuiHZRM2BldS/gx8X7KFfkuUolP6cEucj47Ywhr5PPMZ+faqVHtCqXq76uWhV1lXJfUl654qykvGrlJtHuute9Nin/4z99XtTtP/AEAOCv/uqvkYaF6OydnpC2p5yIbiKiHUS0Y3p6dgGnMxgMC8FC3ux7AZzFjjcD2K8bOee2A9gOACMbVxfCET/vWz+OpTgqX7D5VZRlw2lIN1o6yCtVZPXtHJfGZFsKmPgccGnmlE97StDXxY+1iF8KuSTh2w0ODKpevVRx7NhRUZNn720hb/Z7AZxHROcQUQXALwO4dQH9GQyGJcRpv9mdc00i+k0AXwEQAviUc+7hRRuZwWBYVCxEjIdz7ksAvrRIYzEYDEuIBS32haBdx+A6TvouuNb50pDVjuvKsd5TpM7tFgt5/RpOR5ddLOQ9N9+N13PFj3l3+vq5/sp1bwAIiJvU0vcH+Km1PpxmCM3SqdtMqfzc0OPv3Ifun1tUApIWlGq1n9X5JTkyskG0O3zkcFLWVqST5sisp8vcZQ2GgsAWu8FQECybGP9Mxuk63BQd7WY5X47j03br7tjfYqlJwjMuSHdUEsdtqgZ3zPEi/vr160Wrp3Y/kZQbjRnZQ+vcWVdlb3aDoSCwxW4wFAS22A2GgmAZdfYs7aKLZqcsC+BidJ+ps5v+zpGlRstpy7pJ2gU5n06dZfrNAu8zZPp2ls4eBmVRVyn7oJlq1ZdXDA+IdgfuPsA6lNeZnDtjEu3NbjAUBLbYDYaCoOtivHNh678Sb2PeRooopTL/TWKx6NrDTfBaaHGGV6Z7xsWR61huR7rHlRYlRQ27tjhOj43mIextRAiLoWuQcDtTdZ09wfR9ETdNmcYCNkYRYR8ozzIuWgdyPqQJk3+u4uqZN1nWfSfmnQanuRA6e+u19yG/x5+RoOLrOAmF/p4W48PQx7cPDnrRvVKR7Q6N7mP96Xksqatoh73ZDYaCwBa7wVAQLIMYPyeqtXtEpQcRSDm5M1URIB2TSImLqeKNlrgzqMj4mKVIq1USP5A4UqKpoNxSoiQXTx27NXRm7tpL0To9QITvWGsx/nQ82bIClNp3wTvXtVFsnaZno9iND/Ptxgdh+rlHRkaS8vj4CdFuamoqKUdKBeS0WqljnbeFwWB4RsAWu8FQENhiNxgKgu7q7M4hTmiA9e8MM3208XbzY6+vBVrlZXpMpHRlaTLhOmT7GP04ZB+cTEEQYOhzMb1Uq9v8fO16Hffwijt+DkjTm+4jr4feYujKTpgR0z3Xsj7PInzgfWZFEmb1kTYHWXp/lv6eRUrBofcE+HGlLKmqeR9r165Nyvv2HRDtMnMOZJh7kzHM28JgMDwjYIvdYCgIuirGOzhEJ7ODtAUzcLOIHhb/TeIebpKHS4jqWiQULnrpIiyv0llO0kRkbQIU6kWG2UxLklLy4x5oaj5cZ4/CuT7j1DrRe4bou1z8d9ocm8Wxnxdp15mt/uTrD5gvuKZzO20m6+nxwS8bN25Myl/72tdSx5hXXRFjmLeFwWB4RsAWu8FQENhiNxgKgq67y6bxBnIVJIqbqtIXebpbzeGtTWUcaTzykRqQ0A1Jm5r4mJj+FKiLYop/qNxDuT6vM44Sdd4HaFf7uQlGmYKYmy3XgXW0FjfVZOmaMkO24vMXLptKp04lWEzXvfU9itieTLbZKR3ShTr9e3oe1ch8H9D3kzqXdXZd+Ai2MOgVdX29PqdbX583yx0a1aY3P9+lsl66Tv1vx7xvdiL6FBGNEtFD7LNVRHQ7Ee1q/V85Xz8Gg2F5kUeM/zSAa9Rn7wVwh3PuPAB3tI4NBsMZjHnFeOfcN4hoi/r4OgDbWuWbAdwF4D3z9gUgbolgWdYdHRnFIaPZMn6rlDQTp4qPWpTOEjPTRCQ9Xp7SSJuTuAlGe8ZxMw6PnJO9i2izDHOV6LtNF+jMpz533NlTUHfNxdYsU1AQZJnNuHktQ6XKibZrOeUe2r+VlW45zZynxfgy85rjKZ4AYHh4OCnPzDSS8vT0pBoXV8vkOPLM1elu0K13zh0AgNb/dafZj8Fg6BKWfDeeiG4ioh1EtGNmurbUpzMYDCk43d34Q0Q04pw7QEQjAEbTGjrntgPYDgDrNww7v6OY7kGXRf3m0Dk4Yj6k7jin0691aJpPKJRZS2WddMJLDx6RgTC6D74zrTnR5h9Te1064QM3BWjrR8g93FSffOeYhJeZasctC7GszPYYWx5kBvIIL1D5Hq1WPc+cfmp5mqcDB/wOfK1Wzz2OkyrQUmRxvRXADa3yDQBuOc1+DAZDl5DH9PYZAN8BcAER7SWiGwF8EMDVRLQLwNWtY4PBcAYjz278r6RUvWKRx2IwGJYQXfWgIwLCcGF7gpwwIdvzK90klU12sBjgPOayRp5Pz0VnhVsTFIrvqetspp5be9Cl69FpOnus9hjCEp9TbTZLiQokFUnItMxmU+vsvL/0vRquH7fd9xQ+eM2BLz3osrjnMzd1UsHHX6lI8gqusz/88EOsJv0626PvaL4hmG+8wVAU2GI3GAqCZcjienokBJ2RJbTkTN20FBljM7zwsjOQdh5XFGsTDBfjFddZyAJhhJecEltF9qcskot0kg4n7mWb8Y2VmBmu7b7w4858bgtBGgFGHKWL8VlecoHiG4yigJV9XaPREO0mJ703XLU8K+p27NiRlC+99IVJ+f4H7lHn8n1qdSVqXU+WNdre7AZDQWCL3WAoCGyxGwwFQdcJJ5st00umpqzNCtTZ5BCfgrusHgc/kliM3z+ealezSjLTWAbhgzAFqSip2qz/3kD/alHX3+sjqHqq/X4UKipq25VX+T4GBkTdunUbkvLggCdWaDQlqcjRo0eS8u7dPxZ1d951R1I+cHBPUtYmukbTx0v0MJdSQOY2y8plJvcc2vxxE6S5EgOS7LOdaJTp+pBz0Gz6MfMh1qXKjpjtuxw6LPsolVmut42rkvLLtr1YtLv11i8mZUU9n+wztKXVZrA3u8FQENhiNxgKgu6a3ohSueBks/TopzMxEqodGaG8zov4sZPTT8z0FJBv19e7VrS7ettLk/K2l75S1IUhE4V55JwmZGCmpzAsQ6LzfJfLUkSsjngutbPPPlfUXXHFS1j//nOf/msOR44cTsr33H2vqLvzzn9PyuMTx9ng0818bR56Lp85L+u54s9srHSBhvP3muqMsIOkqB6XOAednIPRw36M//Ft/0y89jW/INr95MWXJuWHd35Xjr9l3nRZHH+pNQaD4RkFW+wGQ0HQdQ+6boneWYEwElplOHU6ag25wzwk6uJmX1Luq64Sda+46lVJ+fIX+Z3Y2Rl5m8ol38fkuBQX+fjDkO/oy+vqq3ixsqFEP36ZlWqJfa69tphlJJLzWC5xumQm3iqVYf36s5Pyq165SdRd++qfT8o77vtOUt7+Nx8T7UoVrmpIsb1Wk95qJ3G6z2E7eQVXefy9iNSueMyeCU0vHk15sT4kb/34+te/Kdr97GuuTcq79zwm6o6N7W+NYQFU0gaD4ZkBW+wGQ0Fgi91gKAi6S16BfLqS1jsWQ89P60OrOHkJMbLaxU2fgpfKK0Tdm65/W1I+a+MFom5i3HuMnRjjuqA05TU4p3wsb+HUpNdRGw2vQ+rthuEB711XKmvyQv+9Q6PcM07q/cMrvX65du0aeQJmRgyZ7Y1Imxv9uavVtJRGwCVbfyop/9lHpM7+kY/8r6Q8Pn5C1JVK/lq4F2GWeTeLA19XhYz7X6aXUi504F546tyMYXX0qNfFtZfcV77q7+1b3/JOUfdHf9zK05IRVWlvdoOhILDFbjAUBHQq3OsLxYaNw+4Nv/Gyedt1c0ztpjdxlP41l04gsXHDRUn5P7/xbaLONX3Qic7myfspVxgpAo6KZtOTXiR84P5HRd2+vd4j7dxznp2Uv/Xtb4h2zVkvEr7kJTLg4nnPf25SHlrhx1guy+tkTmEIQ82Bz01g+Tj/morwQaZT6kwSAQAxMzf+6Yc/LOr2H9rJvsf7zyAYyTCvhUq0DkUWV/+5NuHyjLft+QdYABT7tBz0i1bnPOsFSfnC868Qdc/fOufB+Mqrr8WDD36v4yTbm91gKAhssRsMBYEtdoOhIOgyeYXXx7PcWfO6pcaZpJKnB+LphZWLaUBeYYsb3g12oE+6eb7u1b+VlPfvkWazej01LZ6Yg0rZK8TDQ5LUoVL15rzPfeY2UXfkiNfvf+3GNyblN7/lV0W7aslfi9YvK1WvfDaaXs+lULqiUsTNazpyLk03T9fZS+Vqap2AUrejpv/g9b90vaj76F++Pyk7Fi1XqcpxcBfndpdYZurUexNsTngNqZTN/Lj92ecuz76XupOuvk/ue5ydt0/Ubd48l0i52cxw906tOdkp0VlEdCcR7SSih4noHa3PVxHR7US0q/V/5Xx9GQyG5UOeV2gTwLuccxcBuBzA24noOQDeC+AO59x5AO5oHRsMhjMUeXK9HQBwoFWeIKKdADYBuA7AtlazmwHcBeA9SzLKroLnW1bRSSyyK4AXg7e97BrRbmxsPCmXyz2irlJh3nUZXly1mhcr943OiHYBE/tuerM07fEotY2bfFRdU3HPE0vP1Nsr7UnlKjNz1b1Y2XAywi5kao2yhglVLK9alhdZ6ZA3bZIq1XXXeQKI//clz+HmnJxTznHXZh7kx6FKWx2meWamq5ik1MO0PANOE2XEY0n54JFdou5b35zzZpycmEIaTukuENEWAJcAuBvA+tYPwckfhHWn0pfBYOguci92IhoA8E8Afsc5Nz5fe/a9m4hoBxHtmJlOTy5vMBiWFrkWOxGVMbfQ/94598+tjw8R0UirfgRAx21m59x259ylzrlLe/sqnZoYDIYuYF6dneYUmE8C2Omc+zNWdSuAGwB8sPX/ljwnTNNlSHCm5zSpnUIwXJreyFMLz52bj0maT6KmP65WvOltqH9EtOvtGUYaeA4wbSbhc1CtMlfahjRrNVk0W6hIIENmOjxy8EBSXr1GjqnEdPs25002Dh4d11BkkVGd6fBqenmE2WLr7Fkutz090nx39Ss8u8sjjzyclPcffFy0a0Rc11XPn/CM7pwqWY9L8/SLZ1oNP0jhxGw21PMR+H2cmdpBUTc2Pnevm7GOtvPIY2e/EsCbAPyAiB5sffZ7mFvknyeiGwHsBvD6HH0ZDIZlQp7d+G8h/R36isUdjsFgWCp0nbxiMUW6zOzHOdEuETKxTKUQ7mOplX5q608n5YBk+qTarBelGg25KZlFoMBJHuosQkunSi4x0021JOfzwF6faukHD9yXlHfv2SPaNRkRxfp1kpf+uT95YVK+5lVXJ+W+AWlG5CKsTs+kI9O6h3Rz6Y03vjkp/9EH3i3ahdzDTaV44hmcXSk9Ik6QXChyThlOqVNUdV6G7UvF99GMJ0XNoaM/mPu8KU2Kor/UGoPB8IyCLXaDoSDoPm98sAiydwtZwTR5eev0jrvM8Cp/C13kTYerhjcm5dq0FJ0c34kuyf5DltZJ1zUaPvCBB/kcOfSkaPfx//0XSfmR798v6sB2+0MW4FJX/Omlqh9HtSR3+2OWGur3WUDOsy+QnHm//ra3JuVrf+61oo6rJI5ZPHjACSA9/uI4PQAl//2U7Wo1L5JXmffitT/7OtHuS//2BT8OzaPPvOb0brxEVnbg04GeD64Cqvd0ONGqWEAgjMFgeGbAFrvBUBDYYjcYCoLuEk5uWune+OZti9ZflpdSXhOfTunruCkllrrs+c/yxIwXne/TJlcCaZIKhUuU8sJznENdDYbpW+9//+8n5Qfv/7Zo1qx7vdc1pC4eNvz+QYmTNShu+IDp1OVAbt0EzBTUbLB9EEUuURnypsiGus4Pf+RDSfm1P/+apBzF0qzF88fNzkrvr54eP6+nc28BgD8ixFIlT80cFu0+8CfMFBeolNuh/16c8X7Mu5ZOd81l5nFrTf///T/fwMH9Y0Y4aTAUGbbYDYaCoOumt8VEO1fYqZv1pAcUhGwdKVPQyuH1Sbk240W7sorma7J0PqRMMCH7eT16WIqSv/u7v52U9+170vcRTcg+2JjLOjAj9GIyl9wD1a7MOOOqmi+NmcqqvSy1syZTYKmWYqUO/e5b356U//ZvPpGU/3L7x0W7dRs3JOVKVb57OIFHs+mvq69P8q9lw1/3saPHk/KKldLrsafqU1lN16QpNUwJVGk702maB9NTk52KuJ/Bg9+CvdkNhoLAFrvBUBDYYjcYCoJnlM5+Oojb9CJGlEhSFx8Y8GzZoUj6pfvwum0USVPToX27k/I7f+e3Rd342CHfP7y+GijzWqXsb5uiMUeFKZglpqdXK/JaqhVvRiup3/wy66NW99cSq9TRYMdRQ96L2bqfk29//WtJeduVMq/c9k99Oim/5GVXynEwV11OKnno0CHRjpvoBgakLh6w/YghZio8clj2MTToyTnr0XFRF4NHIOZU4E8BeQhdOh2r1icbpbawN7vBUBDYYjcYCoKntRi/GHBORTgxb7K+/iFRR8yzjDPlDvZrUgdfPqLMa7/59rck5RNHpShZYh5e5Hz/Q1UpgnOiiEpFevlxEZxHngWK35wfhtr6yExs1bKvjBVbSIO8ihIofSJuMo598uOdPCp5SX/9TW9Kyu94938VdW/7TW++49eyfv160W6WpZ/et2+fqBsZ8fyA/N4OD68R7c7e4tNb77t7p6hjwXJLkHBssTC/SmtvdoOhILDFbjAUBF0X490i7KAnfWV4GGURW3DEKoNpM/I7zI3JE6KuHnnROmDZTRt12cfM9LGk/Na3/JqoOzq2358bcnc7YmPs7/O3JlI/yZx/vxTKysFeL3NyurRyqG61Y55xdcmTx73VeFBPVakMjlFha+86YrpBkxFquLK8D8fHPd31Bz7wIVG3Z7dXc/6ABQb19amUWox8Y/PmjaJuatJfW5lZMSI1qc8+56KkfO99XxN1YejVhKZTiU4E9fhpPttpj7EmqFhQZ/ZmNxgKA1vsBkNBYIvdYCgIzkjT21IQaqTpU6HSZZtNZkJSRAtHj3gd8qJzt/jvNGSa3N/7Hz5z9dHjR0SdY+mZ4kheJ89ENTnjzx0pTvY60xurirQyYmauCtPny8rxq8n2Kohk/w2m94WMlKKpUmXV2Fwdn5RzUGO893XmRdhUt5YHFrqZY6Lur//6Y0n5h496c9gnPrFdtFu92nu/8Wg+AHDMy+9Hj/8oKZ9/wRbR7nxBpqm45/l+RBvhSOrX0qEf79TvpUdkng7mfbMTUQ8R3UNE3yOih4noD1ufn0NEdxPRLiL6HBFZ1kaD4QxGHjG+BuAq59zzAWwFcA0RXQ7gQwD+3Dl3HoDjAG5cumEaDIaFIk+uNwfgZK6ZcuvPAbgKwPWtz28G8AcAPq6/L/pCPhE9rxifZerIHUSgyCvKLJ1SQ2XRnJ2eYHU+/c5X/u1W0W7nD3220OkZmaZHcMrrbJ5sKFybmFCy7zgLjCkp88zhCS+2EptHPaMN1r8mttCpnJL+1Bzy21RvSP44znsfOz5GNRLu5aeuhXMM3v7V25PyT1/5EtHukksuScrXX/9GUfenH/lwUt648VlJ+e/+7tOi3de/9XU/Qj1ZPGVXSc/B6aic+juptjd1rtM4FUPe/OxhK4PrKIDbAfwIwJhz7qQythfApoUNxWAwLCVyLXbnXOSc2wpgM4AXAbioU7NO3yWim4hoBxHtmJmqdWpiMBi6gFMyvTnnxgDcBeByAMPkt3E3A9if8p3tzrlLnXOX9vZXOzUxGAxdwLw6OxGtBdBwzo0RUS+An8Hc5tydAH4RwGcB3ADglsUaVF63Q60vZX0vTbdyys2zwvTVsCLtVSMbfKTUMRax9tnP3CzazdS8nh6U5RTHzBVVy0I8x5gYrtP8+P46m+r3OmSkFCFzI9Vzw1XPmRlFsNjTm5Q56WOomBc5oURPICMEm8ydeHaGuSA3pW7PXV3XrFop6oaGfJ/r1vpItxUrZLvJSd//nXd+Q9S99a0+H90Pd+5KynfffY9oNzrqo/E4aQYg9zc0+Lzm1t/1c5uis5/KfoAfR/p38tjZRwDcTHMUHQGAzzvnbiOiRwB8log+AOABAJ/MPTKDwdB15NmN/z6ASzp8/gTm9HeDwfA0wBnpQZcXpyLmpPJ8KbGHc8aFJKOrenu92PrhP/pIUj73nLNEu8MT3mtualKKyFyadsrUREw8J3DxUMqRvI+eXu3L5Pvk5q+qSt3UrHPznRTxm0x05w9Ij+Kxi5gHoNKGUA79+SqM3EOnbuop+z4rPfJxrFb992o1L/5PTk6Ldn29nnfukYcfFXUve7nntbvsssuS8r33SjH+tddtS8o7d31L1MXMPOvifM9c1rNJiuPdpaZ61mppRkRbDm488403GAoCW+wGQ0HQXTHeuSUJcvHdZ4hOqTv1UvyJWebWqKm80456frOzzvK0xCv6Voh2dz9wX1Jeu2JY1A1WfOoiUoEwMzWvQsyyTK2zNUklXWdBLDOzsi5tDmrT0pOvhwXJDJSkeD7U78Xi4yfGkvL0CUmxzB3j9Hl5QEpv2YvjoQpUCZj83yRJAz3JiERqFS/61mclgUQ85I8fevghUfcPn/bi/8iW85PyxAkZuPP6617lx+SkyhNVvXXCRVItC7hYz9SttkSzLKOuaxPP/TLk83gqVNInZ7G9bzbW1BqDwfCMgi12g6EgsMVuMBQET2vT26kglXBSmTNc5HXIXkVs0ZhlJilGvviN//i2aBeyKLJ+RY4YMD294SQ5Bk+P3LPC64lBU5pqOLnj3tGDoo6rbH3ME26gKsfRYF5zZyse9h7mhTfAOOsPHZEc+CG7tkZTXsvUpN8jmGZ15VDuD0w7b0aLjkuCT673cpPdwGC/aDfDjlf194q6WTaOY4f8nstTu54S7Q495oktLlgvTak7D/i9iqbcchBGNMdMdMoSKSIQ28kr0vX0vPDPtxFOGgyFhy12g6Eg6D5v/BKSV/Dv5Q2SaWvHzCekySvGxpPyIw98PynXpqVH16o+L0rq1E1R3ZuCQid/axuOBYkwdUJ7bUUNb2paNdgn6jgBRG3ai+qNmgwvvuCcc5NyvxLx+5n4/xMbffqkXSU53kNjnjOutypNmAMlb47kInik5NuIEUM4Nd+Nmr/OEgvC6SepMgwx0fpd73u3qIvZnH7pq54P/rdueINo1zzo+evLR8ZF3UWrvFj/yNhjsn+WR4tlvGqjfOfZsfIK6py8A5DPcLpZzkxvBkPhYYvdYCgIbLEbDAXB09r0lpXPLa/OXlI6TsDUwfUDkpBh9PHdSXlVjzf3nBiVbqSvuPzypLx3tzTxTBz3+mBdkSJMMxNbjZE8RMqsxaPUympPgJh+v27N6qS8on9QtFvJ9hUqgdS3h1i+uAoz5bzw4ueKdvsPeHKi/QclUdH0DCe+9OONVG66cg9zpYXEymGv92/YsC4pX3TRhaLd2Vs8kWRPLF1pqcrcn6f93NdUuux7v+OJLUYP7hF1P7nthUm5pO4ZTxkn3IdlMxGpGOg9qSV0IeewN7vBUBDYYjcYCoIz0vR2un1lifFp5w2U8DhY9uLto3ffL/s44fvYerY3XR0dlZ5lg4yQ4bzN0hurudqLmZr7bZwdTzJTWUOllY7Ycdt1MXPN6gEvumtVYMuI95rrq8goL+5B18NE+oFhGcFXZmbKAcVPV6r4R2uGp4RWUW/Dqzyv38oVUm3awMfIPOPCUJlcWSSac7L/Y8e8Vx43q9aVufT8c89LypHi+g9YjqpIkUQ45ivHUxBopTHI4M7n5CScyILUu9hlElsYeYXBYGjBFrvBUBA8rXfjFwNxLMWhMPa/f8d+JHfSN/dvSMqzo2w3ty6900LO/VaRYmVPj/d4iwZkQEeTidp1Tuqm+OM44dvklPT2IuJZYn1/q4Yl/XJj3I9Ze9CVuFg560Xkxx6S87HpWeck5YpKGTUz61NllctM3B+UovpqZjEYGJQkIKtW+GNiLmgRpEoSs6y245Mya+7BUS+SlxhJR6kqPQ8fe9JbWi5+8YtF3bGqF/nrE0qMZ3NV4sQT7YK8/45KOYZAh8202iHdg84pVYBMjDcYDCdhi91gKAhssRsMBUHhdXat+zSZKSVWaYDWjGxMylPM262/R+rUXI/etG6tqAtY+uWSSv9bgdcpq2xcAVQ6IpYeuTErzXd1ZuaamGCmvHFJTDkx6iPWzjn7HFE3MOiJH8cmve59ZEKSS+wf96mpBwekvt0/4Oekf6U3m82y/gBgcsynXervk+YuH1oAABleSURBVHq02H9gJsXZurwWHmFWKkvdtcLeZ89mkX6k0lQ/9qQnr1j1E5LMY3axV4lT+rwOkUtFekRbMj8Zpu3cb/ZW2uYHiOi21vE5RHQ3Ee0ios8Rkc5WYDAYziCcihj/DgA72fGHAPy5c+48AMcB3LiYAzMYDIuLXAIKEW0G8LMA/hjAO2nOBnAVgOtbTW4G8AcAPr4EY0z1jMvi1c7L5RWT8k4Lff+rNm0SdfuP+oCXYye8OLpx4wbRrsm45ZrKfNLLPMtEFAUAYkEi3CIYNOW1zDLvLwpUlljOksBMdo1IjqM04tWLgw2pCvRMMfGZDbF/g1RJ+nq8uN9XkWbEco8X9JosOGWgIs18deatdvzEUVE3OOz755JvrSaDXQImklerkoNuy4j30Kv0ezVhdkLyxr/pN/y7apykd90jR7yI7zShBFcHcvPHqec2h9ls7uQZPc7PXZH7zf5RAO+G59dbDWDMueSp3gtgU6cvGgyGMwPzLnYieg2AUefcffzjDk07/u4Q0U1EtIOIdsxM1zs1MRgMXUAeMf5KAK8jomsB9AAYwtybfpiISq23+2YA+zt92Tm3HcB2AFi/cbg7gbsGg6ENefKzvw/A+wCAiLYB+G/OuTcQ0RcA/CKAzwK4AcAtSzjOJYPmjZ+NvPSxduNGUffEPk9wcOS4N129/IqrRLuxSV/XjBuirsH2HCqKlz4t6+5Ms556PKsYynnLoNfr0aHSNQPWR6i8NYnlnKswZXmwLMe7ps/3z9MyA0Dc5/XtButjYkzq5TyvXH1WmuUiNndNxlQZRVpv9qbJQO1hlBkxx+FD3sy3edNm0W7Ds/xxyclxHNvzPeRB9j4RpZQzVfFTwPy9LMSp5j2Y26x7HHM6/CcX0JfBYFhinJK7gHPuLgB3tcpPAHjR4g/JYDAsBbruQRe2In7ykksAQJAS7RMps1YWeP+cx1w7LzXYuVZvWifqNp3wdc+5mHl7zUo5mKdlnlV87c0M16MK+dsRsOmYbUhT0CzjQo+U+FZlXn+D3G6mRF9+4bPK9ManhIum0Ywcx5E9/nv9A4p7frU3ea3b4E2TgYoCBDM3rlkvDTozTT+vISMEiRXPHEq+z6mmVJtmT+xNytPT/l4cUV6P47H3enQrpD7VcIwcI5QRd3weHV9O2ktOQD104tlnZuY20Tzu2E72lfusBoPhmQpb7AZDQdBVMZ5welkq075zKj2l9RE7Jd4yfrc6SZFww9lerO9lXm3jKrtps+HFxT6VxbXe4DvM8twOvk6kfAqkbMbVkH4tFjcYHfWEDxipT0tRPWLz0VRZYmM2B/1sV32YEU0AQA8jvXhk505RF+/12WXPZsEvgytlwEwv66OiROuQ7f4T57gLgtR2JTUfR2d88E6j6kXwxoykiyZmCtn31AFR5yqdySXakb7jLo/1OzZOaZf3vPlgb3aDoSCwxW4wFAS22A2GgmAZyCvm9JN2FTrLA6izaSJL/c9K5ywrVDQYU/lmm5KsYXiVj/qaOepNNdVhea5azR/XI0m0UCp7HTWO5Jhipos2GaliFEtzT5mZqxozipSCpZXe/5Q3O61eI82IWy70qZympmQEWE+P3Gc4CVJkHmCpnVdtPltU/fjHjyflfaPec22j7qKPmRvVq0eYSNn9LClyS/EgqPt+vOyvLQr9nsixeEy0i6Z8XalP9l9v+j2YIGPJCELIjMe5PTVZetvFhL3ZDYaCwBa7wVAQdFeMJydz5PAqJspkedMJiS3zZPlEpbLKYAry3ln1kjR97K15sbja66euUtW/mb7PeEqqCbMnvLhYhRSXHfMY4/xrlV7pdsf4NRBPSZNaX8mfexMj1aCSMkkd8qaxOJLXOcnSXIXMQ2xoWJrNKmzqRtbL1FCbN3tP6nKPP/fMrFQZIuJqmcqoKzwdfTlSQT0hM8tVVSqrct3PHVeH2rwvGQlFU6XbKrGUVfrRTDULt6mRHZt1FfZmNxgKAlvsBkNBYIvdYCgIum56oxSdXSKfgpOps6vKIIWbO9Tpc9n4GspdtlH2uhwnkpxRhJDE2CJLsUplPMN0fRUCFzW8Ga3MdO8ZNQ6esjhUU8XTLw8O+pTNmpiynHHrebQVn7ewJOewwsyKpVjWOXbdMeOyr/bIa47ZtehAMc6Bz+1ycZyub+v9nv7YRyDWa368YUXeF+m5LK+Fp1uOSZpBJRmq7yRIYyJZRtib3WAoCGyxGwwFQZfF+ACBOylWaeIJliZYR6JR54D+MFz4b5UeBe+xorWJNO2ibRhe1CPViev1IufsrPR+C0JuXvJ99JQkJ/t03ZuvSoMyZVLIouXKTJQsK767mJn5IjXfs5zvjelDpNJQDbG0x06rSY5fC1NrnBxHwPjjZpXJK2LjIBbNFyp5P2Tj7+mVvPHcZEcs3VasSCgipiqFkbxOYk9JgHTyCq46an75TM00TeRv+0768942ro5tDAZDIWCL3WAoCLosxhPzLMoK7tdZLjv3dgY4JXVEk0t2ZUW00OtFNk0aUebpoNjudj2SIlrAUkgFsRQ5HdtWbjCRPlJprhjdHepNOZMTjJMuYiJzrwpAcWwXPFK74DW2802MXKKp7lovszqI3XcAJS6Cs6nSInI99oEqZWV16GUpsI7N+u+RevKDDNPOmfqcCWRy3s3B3uwGQ0Fgi91gKAhssRsMBUHXCScDzVCQ2pIfpvBqnwL3fBo0N3feHiRRgfxWxHRlUsa9sOyPp5syAmxtxZNjBKyP6Uhyzzt+bmV+nGbmvAbXgfXcsP7jWOvbjKyB8bWXm3LvgOvfpJRe7hlXYXp5ECqPQhZ1WNGkFGxcXE/XUXoRMyNC6fMbVnqSzNGJfX7sajqafA7anitfDoL0aDYxj20PUhYZZWdkEbC0P/vz95c3P/uTACYwR3fZdM5dSkSrAHwOwBYATwL4Jefc8bQ+DAbD8uJUxPiXO+e2OucubR2/F8AdzrnzANzROjYYDGcoFiLGXwdgW6t8M+ZywL1ngeM5JeiYGi72UFumzM5yTtwmUuUMwskgDisz8grNcUcsZWqjJtMpzcx6zrsVNJSUqyp7apN51+nxOybiclNWo6GCafh3lNmGq1ol9j4IlKzY5H0qvnZuluNBLFmqVrMuxxgxtaEn5JladXomf81Tk5OibuW6VUk5ZLciVPPGU4y1keHl1O1kF9orjnnvaY0qX/eZ0A57nZD3ze4AfJWI7iOim1qfrXfOHQCA1v91qd82GAzLjrxv9iudc/uJaB2A24noh3lP0PpxuAkAhlb0zdPaYDAsFXK92Z1z+1v/RwF8EXOpmg8R0QgAtP6Ppnx3u3PuUufcpX391U5NDAZDFzDvm52I+gEEzrmJVvmVAN4P4FYANwD4YOv/LbnO2NJh2zReHjGkySJT9Dyt/4VMT9I6dRvhQftpAQBRTg7vLJ29xMxhTWWu4qpsuSz1utljPidalUWKxf2yHdcvSyrarJeRV9SZCU1nDSP2O1+tSkIJ7prKCSdD1Ul91vdP+p5xsgk2B5WKPFeDjXFyXJFn9nlJsFT10WxRTbrV9rA+x0+Mi7pVbmVSrlY8wWc9UO7DYgNI6fMZRxz8cYxiPeM8tXMbs0pqn3nh91kynssc/awH8MXWw10C8A/OuS8T0b0APk9ENwLYDeD1CxuuwWBYSsy72J1zTwB4fofPjwJ4xVIMymAwLD66HPXmEKYZGpiJqp0uLsX2oSKcUrJEzTVNEb/agoWyop9cZw8pbeaLAnYtiuvMNdmYVehVnXHQnah5cdQp0gie/kib1PgYq8z7Tdt7KOYmNTk33BsuZv01Yik+OxZFVnbKbMYmttHDSC4aKuUVm8iePrmnMzzkeeqbTHRvNqRqVGHpqgL1eNVrflzEePpJc+axVFylkuw/5KbOQI6R8/sTIwR02r7G72HbQ5fuGSdaZXjQ5eF2NN94g6EgsMVuMBQEttgNhoKg60w1/vdF6xgi2Zuqenr9JpHzOl7glL7NuOL7SzJ32hTT/5qR1zWjGWnG4ZpzqKLIuImRmyyrKtcbn2+t//E+ZJSXYohp1FiNHGPIztdoyKg9jt5+r8+vWDEo6rjpbWzGm+X0eGdY3VDfgKgbO+Rn600/9xtJee+JA6LdLbd/ISlTRboxRwHT4Z3ct0hNJ673k/gegTbbcr0/I5oyCye/lmXEe3qtIoPBcNqwxW4wFARdT//k+cQ1N7wXR9t44/Xxya/kNZO1fS9dVMoSnPKKVaXAe3uFTcljfv7G5yXlxw6oEIPIR2zVY192dZVeikdlqTFx7zpu/mnn4mcRgmoiyyzKjnu8tYn7bLZ0GuUS8w4cGPKidUmpEz29XlQfGJAiOI++K7PvxQ3l/SaIOOQ4orpv+9B3H0rKz3ne80S7P3znh5PyH3/sf4q6WnTUj186AKIZcbGek5ao92jQsVmrcd60zzlMbxlrwt7sBkNBYIvdYCgIui/Gp/2+iDgELeek/SZ1l9E7K/hFoObF0YvOeZGomt7nd9kfe3CvqFu3mvHBl5mIrMT4WHCuqTkIO6cSaiPzYDvAsRIJSyGTVSltZ14GHpWgrAJMTWvUGa+7IuIolbj6lm4VaDLufB1cVE65ZgDoKfv5rp/w5CC77t8l2lWqw0n59971EVH3r9/0MV7f/e5XRV0Yeq88kBfpndPpmLjL5Sm4beaGedAZDIYWbLEbDAWBLXaDoSBYBp19/tSy2XngePpmTRCQ0UVKH5p0gZMStgUncasO8+ojlXK3P1yTlHsxJOqaznPFD/ZLUxORj3orlRgBhnLaogwCx4jptqUMjv6QpUrWc8UzJxMbR0l5hXHvPW3y4nz2ZUFkIaPjmnXv/dasSbNcRVgY/aCcylvXZGbFuCxNnYN9XqcuBX6udu97QrTbfPz8pFw9tFrUveYlv5yUX3aljOre9fAjSfk73/haUh4bPyjauRLziAzTTZgl7kUZq4hGXlZepbGKOuwEe7MbDAWBLXaDoSDovhifw8qQbeLiXkTp7dq93VyHUrunEzdRkVITBDee4+1kH+tWbUjKIdJF382bzxJ13//+N5PyoLcEgXQwDTOB6V9rfm11RvhQ0bzuLLWzDqbh4Hx0JcWZVy4zLnfVBz8Wt0ndW8E9r1JT8zRPXD1pNFRq5xIfh/I2ZCoWJ+XYs+cp0Y5z1w1Py/7pmD/uH14v6s5Z4/sfusLXrVqtvAFZ2q+dTz4o6v7trn9JymEfIw7RKi+7GVqFJVRa/9PXhL3ZDYaCwBa7wVAQ2GI3GAqCZTC9LRRZ4UNIrUuPWFNTIHTKdNMe19PDSIZCrej3ppsglnXVio/y2rjxWaJu926v649P+5wbZZJmlZi5mIZKBy4xnbVaZUSMygwXMX2YlBlH5MzjEWUNnbLZo1JV7OqlzrqjjnrjOeFE6mVIt2Be1nqpoNeIpAmQj98x2+kPH9kp2l33ev8cONXH+Jh3sx1Urtzr13g9ffaoNyMeelLy169e4XPOXTxymezjP/k+/uGfP52Ugx45346ZZom0qa01rxb1ZjAYbLEbDAVBV8V45+IkjXBWYL4WOTmyzHJC/FSiqY7YOoko1t5MXlzUHPece54izx/eV5HmmNWDG5NyY1KRRpR9HyuH14qqF112VVL+8u23JeWpKRkdN8RIHnpU6qYmm8cw4KK6HAePFNNBhqWQqwnMW0+LyNzapur4ccB47gOtdnCzmboXMxOMwIOrEErc532UlBw7PeU9Fht1LwZPjksxe3jQezrq6+SpnOp1+b3Zmm+74Swvqh8/LD3opo4xz7hJyT2/cvVIUn7ZZdck5Tt2fFm0489OoD3mgpPXlq7a5nqzE9EwEf0jEf2QiHYS0RVEtIqIbieiXa3/K+fvyWAwLBfyivF/AeDLzrkLMZcKaieA9wK4wzl3HoA7WscGg+EMRZ4srkMAXgrgvwCAc64OoE5E1wHY1mp2M4C7ALwnszPnAya0qM4DKXRQBaXwpbV5vwlPrQ4n74BYibeO0QYTKQ8mRgNNDV8+7/ytolkJPhij1piSdSwARQdtrGBi/auv/YWk/K1//4xod/DAvqQ80K8CPwY9HXPU9NdcVmmoOHedFq3FMWunVag0vjtAEWywdnGo1KsMPsCJCZ/Vlnv5Zal5s7Oz4rjExrF7t7dwrFsjVSjeZ7MuPeia7BmpVtfIOlaemmK79iukoDu+93hSjgN5nZMz/hl54fOvTMpnXfQTot2P9z+WlB/b+ZCoOzF+aO46SFOGe+R5s58L4DCAvyWiB4joE63UzeudcwcAoPV/XY6+DAbDMiHPYi8BeAGAjzvnLgEwhVMQ2YnoJiLaQUQ7pqc1wb7BYOgW8iz2vQD2Oufubh3/I+YW/yEiGgGA1v/RTl92zm13zl3qnLu0r6/SqYnBYOgC8uRnP0hEe4joAufco5jLyf5I6+8GAB9s/b8loxsAczrfyXTDWXp5hzEkZZmaSOn98OYIRzrtLiNTYFFkodIh63V/XKlIvS6qeZPJ1otenJT7FUFFNOF1sEAxT0R876Akz10N+pPyENO9Xvmq60W7ex74TlL+3g/uEXWzzuuNG1aycdXkfARVr+u3ZRdm9yJy3HNNolb3X6wwbz0AcGy/gD9k2tTZZCFxoY7MY/ede/zpKD1OQBnH0iw3zb738M7Hk/K69c8W7Q6NjiXlwU3ypVRZ4b0e9dbHzKQ3DzbYfV85IJ+JidDflxm1r1Bt+k4nmOfdWc+9QLRbueLcpPwzW39JDqQyN1f/8pn7kYa8dvbfAvD3RFQB8ASAX8WcVPB5IroRwG4Ar8/Zl8FgWAbkWuzOuQcBXNqh6hUdPjMYDGcguutBB9cmvp9EkMGrxiHFfZ3SiJ3Lqe0I5y81jphHV1OKjn2hF2/PHj5P1K3ZvDkpTx7x4uGe/ZLP7LwtZydlpSWgyUxBmnyDX1pPjxeLXXWVaPfSl7wyKY9s3CzqvvbVW5Py2JgXF/t75K12znO58xRPgFaPeGbS9My7XMzWx/x+as53oaLVpHib9qxojz/+TExMSVPn8eNePN+732duveBCGYxSm/XzMVCX19LPefSV9x6/HmLqhVZXONFHTWXlrTFT38QxP96BMXnf9+zb48vjcq5WrJ4LvqpNyc85zDfeYCgIbLEbDAWBLXaDoSDoOnlFmoktdx41Bqd1dt6fujTH9PRLnn9FUt56odx3nBr1Os/t//pNUdeoen1q00bvynhcESXu3e3dWc961oioE+6+ynQo9yq4G6mMkgpCf3zxc6XuecGzL0zKX7nt80n56KHdol3cZFFYDeUyLCIQ/edZur3WxXldo5HOaZ5Vl+Yi29RRaWx/4MSx46Lu+BHvcsuHOLxScsNXGCf+kadklOHh3U8m5Ysve4Go4yScUZW5QivVmV+LHn9c93PQOOH1956DR0S7i8/1prj7b/+WqDt6dO57zVr6fNqb3WAoCGyxGwwFAWWZuRb9ZESHATwFYA2AI/M0X2qcCWMAbBwaNg6JUx3H2c65tZ0qurrYk5MS7XDOdXLSKdQYbBw2jm6Ow8R4g6EgsMVuMBQEy7XYty/TeTnOhDEANg4NG4fEoo1jWXR2g8HQfZgYbzAUBF1d7ER0DRE9SkSPE1HX2GiJ6FNENEpED7HPuk6FTURnEdGdLTruh4noHcsxFiLqIaJ7iOh7rXH8Yevzc4jo7tY4PtfiL1hyEFHY4je8bbnGQURPEtEPiOhBItrR+mw5npElo23v2mKnuUTZHwPwagDPAfArRPScLp3+0wCuUZ8tBxV2E8C7nHMXAbgcwNtbc9DtsdQAXOWcez6ArQCuIaLLAXwIwJ+3xnEcwI1LPI6TeAfm6MlPYrnG8XLn3FZm6lqOZ2TpaNudc135A3AFgK+w4/cBeF8Xz78FwEPs+FEAI63yCIBHuzUWNoZbAFy9nGMB0AfgfgCXYc55o9Tpfi3h+Te3HuCrANyGuaCA5RjHkwDWqM+6el8ADAH4MVp7aYs9jm6K8ZsA7GHHe1ufLReWlQqbiLYAuATA3csxlpbo/CDmiEJvB/AjAGPOuZPhIt26Px8F8G54JpLVyzQOB+CrRHQfEd3U+qzb92VJadu7udg7hbUV0hRARAMA/gnA7zjnxudrvxRwzkXOua2Ye7O+CMBFnZot5RiI6DUARp1z9/GPuz2OFq50zr0Ac2rm24nopV04p8aCaNvnQzcX+14AZ7HjzQD2d/H8GrmosBcbRFTG3EL/e+fcPy/nWADAOTeGuWw+lwMYJqKTsZ7duD9XAngdET0J4LOYE+U/ugzjgHNuf+v/KIAvYu4HsNv3ZUG07fOhm4v9XgDntXZaKwB+GcCt83xnKXEr5iiwgZxU2AsFzQWzfxLATufcny3XWIhoLRENt8q9AH4GcxtBdwL4xW6Nwzn3PufcZufcFsw9D//unHtDt8dBRP1ENHiyDOCVAB5Cl++Lc+4ggD1EdDJw/SRt++KMY6k3PtRGw7UAHsOcfvjfu3jezwA4AKCBuV/PGzGnG94BYFfr/6oujOOnMSeSfh/Ag62/a7s9FgDPA/BAaxwPAfj91ufnArgHwOMAvgCg2sV7tA3Abcsxjtb5vtf6e/jks7lMz8hWADta9+ZfAKxcrHGYB53BUBCYB53BUBDYYjcYCgJb7AZDQWCL3WAoCGyxGwwFgS12g6EgsMVuMBQEttgNhoLg/wN79sjNXGOKLgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "IM_DIM = 64\n",
    "@tf.function\n",
    "def im_preprocessing(im_path) :\n",
    "    im_file = tf.io.read_file(im_path)\n",
    "    im = tf.io.decode_jpeg(im_file)\n",
    "    im = tf.image.convert_image_dtype(im, tf.float32)\n",
    "    #im = tf.image.crop_to_bounding_box(im, 20,0,178,178)\n",
    "    im = tf.image.resize(im, [IM_DIM, IM_DIM])\n",
    "    im = tf.image.random_flip_left_right(im)\n",
    "    return(im)\n",
    "\n",
    "batch_size = 32\n",
    "im_patern = \"./data/img_align_celeba/img_align_celeba/*.jpg\"\n",
    "files_dataset = tf.data.Dataset.list_files(im_patern)\n",
    "im_dataset = files_dataset.map(im_preprocessing)\n",
    "im_dataset_repeated = im_dataset.repeat()\n",
    "im_dataset_batch = im_dataset_repeated.batch(batch_size)\n",
    "\n",
    "for x in im_dataset.take(1) :\n",
    "    print(x.numpy().shape)\n",
    "    print(\"len dataset : \", tf.data.experimental.cardinality(im_dataset).numpy())\n",
    "    plt.imshow(x)\n",
    "    \n",
    "batch_gen = iter(im_dataset_batch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Making models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "DEPTH = 32\n",
    "LATENT_DEPTH = 512\n",
    "K_SIZE = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sampling(args):\n",
    "    mean, logsigma = args\n",
    "    epsilon = keras.backend.random_normal(shape=keras.backend.shape(mean))\n",
    "    return mean + tf.exp(logsigma / 2) * epsilon\n",
    "\n",
    "def encoder():\n",
    "    input_E = keras.layers.Input(shape=(IM_DIM, IM_DIM, 3))\n",
    "    \n",
    "    X = keras.layers.Conv2D(filters=DEPTH*2, kernel_size=K_SIZE, strides=2, padding='same')(input_E)\n",
    "    X = keras.layers.BatchNormalization()(X)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "\n",
    "    X = keras.layers.Conv2D(filters=DEPTH*4, kernel_size=K_SIZE, strides=2, padding='same')(X)\n",
    "    X = keras.layers.BatchNormalization()(X)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "\n",
    "    X = keras.layers.Conv2D(filters=DEPTH*8, kernel_size=K_SIZE, strides=2, padding='same')(X)\n",
    "    X = keras.layers.BatchNormalization()(X)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "    \n",
    "    X = keras.layers.Flatten()(X)\n",
    "    X = keras.layers.Dense(LATENT_DEPTH)(X)    \n",
    "    X = keras.layers.BatchNormalization()(X)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "    \n",
    "    mean = keras.layers.Dense(LATENT_DEPTH,activation=\"tanh\")(X)\n",
    "    logsigma = keras.layers.Dense(LATENT_DEPTH,activation=\"tanh\")(X)\n",
    "    latent = keras.layers.Lambda(sampling, output_shape=(LATENT_DEPTH,))([mean, logsigma])\n",
    "    \n",
    "    kl_loss = 1 + logsigma - keras.backend.square(mean) - keras.backend.exp(logsigma)\n",
    "    kl_loss = keras.backend.mean(kl_loss, axis=-1)\n",
    "    kl_loss *= -0.5\n",
    "    \n",
    "    return keras.models.Model(input_E, [latent,kl_loss])\n",
    "\n",
    "def generator():\n",
    "    input_G = keras.layers.Input(shape=(LATENT_DEPTH,))\n",
    "\n",
    "    X = keras.layers.Dense(8*8*DEPTH*8)(input_G)\n",
    "    X = keras.layers.BatchNormalization()(X)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "    X = keras.layers.Reshape((8, 8, DEPTH * 8))(X)\n",
    "    \n",
    "    X = keras.layers.Conv2DTranspose(filters=DEPTH*8, kernel_size=K_SIZE, strides=2, padding='same')(X)\n",
    "    X = keras.layers.BatchNormalization()(X)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "\n",
    "    X = keras.layers.Conv2DTranspose(filters=DEPTH*4, kernel_size=K_SIZE, strides=2, padding='same')(X)\n",
    "    X = keras.layers.BatchNormalization()(X)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "    \n",
    "    X = keras.layers.Conv2DTranspose(filters=DEPTH, kernel_size=K_SIZE, strides=2, padding='same')(X)\n",
    "    X = keras.layers.BatchNormalization()(X)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "    \n",
    "    X = keras.layers.Conv2D(filters=3, kernel_size=K_SIZE, padding='same')(X)\n",
    "    X = keras.layers.Activation('sigmoid')(X)\n",
    "\n",
    "    return keras.models.Model(input_G, X)\n",
    "\n",
    "def discriminator():\n",
    "    input_D = keras.layers.Input(shape=(IM_DIM, IM_DIM, 3))\n",
    "    \n",
    "    X = keras.layers.Conv2D(filters=DEPTH, kernel_size=K_SIZE, strides=2, padding='same')(input_D)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "    \n",
    "    X = keras.layers.Conv2D(filters=DEPTH*4, kernel_size=K_SIZE, strides=2, padding='same')(input_D)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "    X = keras.layers.BatchNormalization()(X)\n",
    "\n",
    "    X = keras.layers.Conv2D(filters=DEPTH*8, kernel_size=K_SIZE, strides=2, padding='same')(X)\n",
    "    X = keras.layers.BatchNormalization()(X)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "\n",
    "    X = keras.layers.Conv2D(filters=DEPTH*8, kernel_size=K_SIZE, padding='same')(X)\n",
    "    inner_output = keras.layers.Flatten()(X)\n",
    "    X = keras.layers.BatchNormalization()(X)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "    \n",
    "    X = keras.layers.Flatten()(X)\n",
    "    X = keras.layers.Dense(DEPTH*8)(X)\n",
    "    X = keras.layers.BatchNormalization()(X)\n",
    "    X = keras.layers.LeakyReLU(alpha=0.2)(X)\n",
    "    \n",
    "    output = keras.layers.Dense(1)(X)    \n",
    "    \n",
    "    return keras.models.Model(input_D, [output, inner_output])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "for x in im_dataset_batch.take(1) :\n",
    "    test_images = x\n",
    "test_r = tf.random.normal((batch_size, LATENT_DEPTH))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "E = encoder()\n",
    "G = generator()\n",
    "D = discriminator() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step : 14900  gan_loss 0.171 vae_loss 0.182 fake_dis_loss 0.319 r_dis_loss 0.066 t_dis_loss 0.15 vae_inner_loss 0.021 E_loss 0.034 D_loss 0.171 kl_loss 0.16 normal_loss 0.10708"
     ]
    }
   ],
   "source": [
    "lr=0.0001\n",
    "#lr=0.0001\n",
    "E_opt = keras.optimizers.Adam(lr=lr)\n",
    "G_opt = keras.optimizers.Adam(lr=lr)\n",
    "D_opt = keras.optimizers.Adam(lr=lr)\n",
    "\n",
    "inner_loss_coef = 1\n",
    "normal_coef = 0.1\n",
    "kl_coef = 0.01\n",
    "\n",
    "@tf.function\n",
    "def train_step_vaegan(x):\n",
    "    lattent_r =  tf.random.normal((batch_size, LATENT_DEPTH))\n",
    "    with tf.GradientTape(persistent=True) as tape :\n",
    "        lattent,kl_loss = E(x)\n",
    "        fake = G(lattent)\n",
    "        dis_fake,dis_inner_fake = D(fake)\n",
    "        dis_fake_r,_ = D(G(lattent_r))\n",
    "        dis_true,dis_inner_true = D(x)\n",
    "\n",
    "        vae_inner = dis_inner_fake-dis_inner_true\n",
    "        vae_inner = vae_inner*vae_inner\n",
    "        \n",
    "        mean,var = tf.nn.moments(E(x)[0], axes=0)\n",
    "        var_to_one = var - 1\n",
    "        \n",
    "        normal_loss = tf.reduce_mean(mean*mean) + tf.reduce_mean(var_to_one*var_to_one)\n",
    "        \n",
    "        kl_loss = tf.reduce_mean(kl_loss)\n",
    "        vae_diff_loss = tf.reduce_mean(vae_inner)\n",
    "        f_dis_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(tf.zeros_like(dis_fake), dis_fake))\n",
    "        r_dis_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(tf.zeros_like(dis_fake_r), dis_fake_r))\n",
    "        t_dis_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(tf.ones_like(dis_true), dis_true))\n",
    "        gan_loss = (0.5*t_dis_loss + 0.25*f_dis_loss + 0.25*r_dis_loss)\n",
    "        vae_loss = tf.reduce_mean(tf.abs(x-fake)) \n",
    "        E_loss = vae_diff_loss + kl_coef*kl_loss + normal_coef*normal_loss\n",
    "        G_loss = inner_loss_coef*vae_diff_loss - gan_loss\n",
    "        D_loss = gan_loss\n",
    "    \n",
    "    E_grad = tape.gradient(E_loss,E.trainable_variables)\n",
    "    G_grad = tape.gradient(G_loss,G.trainable_variables)\n",
    "    D_grad = tape.gradient(D_loss,D.trainable_variables)\n",
    "    del tape\n",
    "    E_opt.apply_gradients(zip(E_grad, E.trainable_variables))\n",
    "    G_opt.apply_gradients(zip(G_grad, G.trainable_variables))\n",
    "    D_opt.apply_gradients(zip(D_grad, D.trainable_variables))\n",
    "\n",
    "    return [gan_loss, vae_loss, f_dis_loss, r_dis_loss, t_dis_loss, vae_diff_loss, E_loss, D_loss, kl_loss, normal_loss]\n",
    "\n",
    "step = 0\n",
    "max_step = 10000000\n",
    "log_freq,img_log_freq = 10, 100\n",
    "save_freq,save_number_mult = 1000, 10000\n",
    "\n",
    "metrics_names = [\"gan_loss\", \"vae_loss\", \"fake_dis_loss\", \"r_dis_loss\", \"t_dis_loss\", \"vae_inner_loss\", \"E_loss\", \"D_loss\", \"kl_loss\", \"normal_loss\"]\n",
    "metrics = []\n",
    "for m in metrics_names :\n",
    "    metrics.append(tf.keras.metrics.Mean('m', dtype=tf.float32))\n",
    "\n",
    "current_time = datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "train_log_dir = ('logs/sep_D%dL%d/' % (DEPTH,LATENT_DEPTH)) + current_time \n",
    "train_summary_writer = tf.summary.create_file_writer(train_log_dir)\n",
    "name = ('sep_D%dL%d' % (DEPTH,LATENT_DEPTH))\n",
    "\n",
    "def save_model() :\n",
    "    nb = str(step // save_number_mult)\n",
    "    D.save('saved-models/D_training_' + nb + '.h5')\n",
    "    G.save('saved-models/G_training_' + nb + '.h5')\n",
    "    E.save('saved-models/E_training_' + nb + '.h5')\n",
    "\n",
    "def print_metrics() :\n",
    "    s = \"\"\n",
    "    for name,metric in zip(metrics_names,metrics) :\n",
    "        s+= \" \" + name + \" \" + str(np.around(metric.result().numpy(), 3)) \n",
    "    print(f\"\\rStep : \" + str(step) + \" \" + s, end=\"\", flush=True)\n",
    "    with train_summary_writer.as_default():\n",
    "        for name,metric in zip(metrics_names,metrics) :\n",
    "            tf.summary.scalar(name, metric.result(), step=step)\n",
    "    for metric in metrics : \n",
    "        metric.reset_states()\n",
    "\n",
    "        \n",
    "def log_images() :\n",
    "    with train_summary_writer.as_default():\n",
    "        lattent,_ = E(test_images)\n",
    "        fake = G(lattent)\n",
    "        fake_r = G(test_r)\n",
    "        tf.summary.image(\"reconstructed image\", fake[:8], step=step, max_outputs=8)\n",
    "        tf.summary.image(\"random image\", fake_r[:8], step=step, max_outputs=8)\n",
    "        dis_fake,inner_dis_fake = D(fake)\n",
    "        dis_fake_r,inner_dis_fake_r = D(fake_r)\n",
    "        dis_true,inner_dis_true = D(test_images)\n",
    "        tf.summary.histogram(\"dis fake\", inner_dis_fake, step=step, buckets=20)\n",
    "        tf.summary.histogram(\"dis true\", inner_dis_true, step=step, buckets=20)\n",
    "        tf.summary.histogram(\"dis random\", inner_dis_fake_r, step=step, buckets=20)\n",
    "        tf.summary.histogram(\"dis lattent\", lattent, step=step, buckets=20)\n",
    "        tf.summary.histogram(\"dis normal\", tf.random.normal((batch_size, LATENT_DEPTH)), step=step, buckets=20)        \n",
    "\n",
    "for x in batch_gen :\n",
    "    step += 1\n",
    "    if not step % log_freq :\n",
    "        print_metrics()\n",
    "    if not step % img_log_freq :\n",
    "        log_images()\n",
    "    if not step % save_freq :\n",
    "        save_model()\n",
    "    \n",
    "    results = train_step_vaegan(x)\n",
    "    for metric,result in zip(metrics, results) :\n",
    "        metric(result)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}