{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import pandas as pd\n", "import numpy as np\n", "import os\n", "from tqdm import tqdm" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# Getting a list of files in raw data folder\n", "filenames = os.listdir('./processed_data/')" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [], "source": [ "# Commented out the unused features\n", "header_wanted = [\n", " 'HOURLYVISIBILITY',\n", " 'HOURLYDRYBULBTEMPC',\n", " 'HOURLYWETBULBTEMPC',\n", " 'HOURLYDewPointTempC',\n", " 'HOURLYRelativeHumidity',\n", " 'HOURLYWindSpeed',\n", " #'HOURLYWindGustSpeed',\n", " 'HOURLYStationPressure',\n", " #'HOURLYPressureTendency',\n", " #'HOURLYPressureChange',\n", " #'HOURLYSeaLevelPressure',\n", " #'HOURLYPrecip',\n", " #'HOURLYAltimeterSetting'\n", "]" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "usecols = ['DATE','STATION'] + header_wanted" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████| 372/372 [00:35<00:00, 10.53it/s]\n" ] } ], "source": [ "# Loading all files into a pandas Dataframe and normalizing data to between 0 and 1\n", "tqdm.pandas()\n", "df = pd.concat([pd.read_csv('./processed_data/{}'.format(x), usecols=usecols, low_memory=False) for x in tqdm(filenames)])\n", "df[header_wanted] = (df[header_wanted] - df[header_wanted].min()) / (df[header_wanted].max() - df[header_wanted].min())" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "At this point all the data has been loaded into a single dataframe and any data changes have been made. The next step is to break the data up by WBAN and place in a 2D array at the appropriate grid cell. " ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "stations = pd.read_csv(\"./stations_mask.csv\", usecols = ['STATION_ID', 'LON_SCALED', 'LAT_SCALED'])" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "height = 20\n", "width = 40\n", "depth = 24 * 365 * 2" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "mask = [([0] * width) for i in range(height)]\n", "\n", "wban_loc = dict(zip(stations.STATION_ID,zip(stations.LON_SCALED,stations.LAT_SCALED)))" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "grid = [([pd.DataFrame()] * width) for i in range(height)]" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|████████████████████████████████████████████████████████████████████████████████| 372/372 [03:51<00:00, 1.61it/s]\n" ] } ], "source": [ "# Making a mask while also placing appropriate data frames in their final location\n", "for key, value in tqdm(wban_loc.items()):\n", " mask[value[1]][value[0]] = 1\n", " grid[value[1]][value[0]] = df.loc[df.STATION == key]" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "import matplotlib.pyplot as plt\n", "%matplotlib inline" ] }, { "cell_type": "code", "execution_count": 12, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAADKCAYAAACi9isrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAEpRJREFUeJzt3X+MHOV9x/H3p8bgmkDBGAg/HCCpQ4sQdpKrnYi0MnECBqE4qSCx1R9ORWqIggRqIyWhElAqJPqDkLSOoE5wgZbwo+FH3MSKsRwqQEoMh2sbiAkQx4kvZ9kEA8YlQAzf/rFzZbmb3Vt25mZn7/m8JOtmZ56d+d7c7nfHzz7PfBURmJlZOn6r1wGYmVm1nPjNzBLjxG9mlhgnfjOzxDjxm5klxonfzCwxTvxmZolx4jczS4wTv5lZYg7odQB5Zs6YEifOmjqhx3hqy/TC+3jvaS/3xTGK7mO851e1j/F0cox+UcZrx9LyCv/La/GqOmmrIrdskLQI+BowBfhmRFwzavtBwC3AB4DngE9HxPbx9jswZ1o8vHZW13F14qxj5xbex9rhTX1xjKL7GO/5Ve1jPJ0co1+U8dqxtGyI9eyNPR0l/q67eiRNAb4OnA2cAiyVdMqoZhcAz0fE7wLXAX/f7fHMzKwcRfr45wHPRMS2iHgNuB1YPKrNYuDmbPnbwEJJHX0imZnZxCiS+I8DdjQ9HsrW5baJiP3Ai8ARBY5pZmYFFUn8eVfuo78w6KRNo6G0XNKgpMFnn3u9QFhmZtZOkcQ/BDR/A3s8MNyqjaQDgN8B9uTtLCJWRsRARAwcecSUAmGZmVk7RRL/I8BsSSdJOhBYAqwe1WY1sCxbPg/4Qbjyi5lZTxUdznkO8FUawzlXRcTVkq4CBiNitaRpwL8D76Nxpb8kIraNt99DNSPma2HL7f0ybK+KIXlVDJMsI44qhrZ2ooxhp1WoIk4PGe0/7f7u887aweDmVzoaPFNoAldErAHWjFp3edPyK8D5RY5hZmbl8i0bzMwS48RvZpYYJ34zs8Q48ZuZJcaJ38wsMU78ZmaJceI3M0tMLQuxvPe0l1m7tvVEhbpMPKlisk8VE6PKUIeJUWXUJqiLonH2S50Ge1OVr01f8ZuZJcaJ38wsMU78ZmaJceI3M0tMkZq7syTdL2mrpCckXZLTZoGkFyVtyv5dnrcvMzOrTpFRPfuBv46IjZIOAR6VtC4ifjyq3YMRcW6B45iZWYm6vuKPiJ0RsTFbfgnYytiau2ZmVjOljOOXdCKNYisbcjZ/SNJmGmUZvxART4y3v6e2TG87/rcuY7HrMEa5jLHUk2WugL2pjPdIXd5n/aDf5jQUTvyS3gHcBVwaEXtHbd4InBAR+7JqXfcCs1vsZzmwHGAa04uGZWZmLRQa1SNpKo2kf2tE3D16e0TsjYh92fIaYKqkmXn7ai62PpWDioRlZmZtFBnVI+BGYGtEfKVFm3dm7ZA0Lzvec90e08zMiivS1XM68GfAY5JGOrguA94FEBE3AOcBn5O0H/g1jWLr3Vd3NzOzwrpO/BHxENC2ontErABWdHsMMzMrn2fumpklxonfzCwxTvxmZolRHb9rHZgzLR5eO6vl9ioKblQ12aKKYhiThc/FmybLuajTpKYi6nC+5521g8HNr7T93nWEr/jNzBLjxG9mlhgnfjOzxDjxm5klxonfzCwxTvxmZolx4jczS0wphVjqqIrxwVUUMCnjGHUYY1yXOIv+TepwLqGcOPplLkAdXhd1iGE8T0XnNz4ufMUvabukx7Ji6oM52yXpnyU9I2mLpPcXPaaZmXWvrCv+MyLiVy22nU2j6tZsYD5wffbTzMx6oIo+/sXALdHwI+AwScdUcFwzM8tRRuIP4D5Jj2Z1c0c7DtjR9HgoW/cWkpZLGpQ0+Oxzr5cQlpmZ5Smjq+f0iBiWdBSwTtKTEfFA0/a8mwaNuTNcRKwEVkLjJm0lxGVmZjkKX/FHxHD2czdwDzBvVJMhoPlWm8cDw0WPa2Zm3SmU+CUdLOmQkWXgTODxUc1WA3+eje75IPBiROwsclwzM+teofvxS3o3jat8aHQbfSsirpZ0ETQKrksSjbq7i4CXgb+IiDHDPpuldD/+8VQxXrsuXJvArL1274ENsZ69saej+/EX6uOPiG3AnJz1NzQtB/D5IscxM7Py+JYNZmaJceI3M0uME7+ZWWKc+M3MEuPEb2aWGCd+M7PEOPGbmSWm0ASuiXKoZsR8Lex1GH2hk0lLqUx8qkuxlyqk8jdNSdGJlm9nApev+M3MEuPEb2aWGCd+M7PEOPGbmSWm68Qv6eSswPrIv72SLh3VZoGkF5vaXF48ZDMzK6Lru3NGxE+AuQCSpgC/5M1bNDd7MCLO7fY4ZmZWrrK6ehYCP42In5e0PzMzmyBl1NwFWALc1mLbhyRtplFu8QsR8UReo6xQ+3KAaUxve7DJVJykHwrGVHWuip4Lj10vVxVzBSbL3Iu65JNOFb7il3Qg8HHgP3M2bwROiIg5wL8A97baT0SsjIiBiBiYykFFwzIzsxbK6Oo5G9gYEbtGb4iIvRGxL1teA0yVNLOEY5qZWZfKSPxLadHNI+mdWc1dJM3LjvdcCcc0M7MuFerjlzQd+BhwYdO6/y+0DpwHfE7SfuDXwJKo482BzMwSUrTY+svAEaPWNRdaXwGsKHIMMzMrl2fumpklxonfzCwxTvxmZokpawJX7RSdfFLVhIx+KKhRVbGXKv5mVUyYq2JiUxWviyrOdx1e353otwla4/EVv5lZYpz4zcwS48RvZpYYJ34zs8Q48ZuZJcaJ38wsMU78ZmaJmbTj+Kso6lHG2N46jMceTxnjtavYR7+Mn58sxUesf3V0xS9plaTdkh5vWjdD0jpJT2c/D2/x3GVZm6clLSsrcDMz606nXT03AYtGrfsSsD4iZgPrs8dvIWkGcAUwH5gHXNHqA8LMzKrRUeKPiAeAPaNWLwZuzpZvBj6R89SzgHURsScingfWMfYDxMzMKlTky92jI2InQPbzqJw2xwE7mh4PZevGkLRc0qCkwd/waoGwzMysnYke1aOcdbkVuFxs3cysGkUS/y5JxwBkP3fntBkCZjU9Ph4YLnBMMzMrqEjiXw2MjNJZBnwnp81a4ExJh2df6p6ZrTMzsx7paBy/pNuABcBMSUM0RupcA9wp6QLgF8D5WdsB4KKI+GxE7JH0d8Aj2a6uiojRXxK/bf0yfr4TdRk3PtHK+D36Yc5DXY5h1k5HiT8ilrbYtDCn7SDw2abHq4BVXUVnZmal8y0bzMwS48RvZpYYJ34zs8Q48ZuZJcaJ38wsMU78ZmaJceI3M0vMpC3EUoUyio9UoQ4ThibLpLuqitJUoYpiRVZPvuI3M0uME7+ZWWKc+M3MEuPEb2aWmHETf4tC6/8o6UlJWyTdI+mwFs/dLukxSZskDZYZuJmZdaeTK/6bGFsndx1wakScBjwFfLnN88+IiLkRMdBdiGZmVqZxE39eofWIuC8i9mcPf0SjspaZmfUBReSWwH1rI+lE4LsRcWrOtv8C7oiI/8jZ9jPgeRp1dv81Ila2OcZyYDnANKZ/4MM6p8NfoTt1GYNclzHdE62T8130XNTlb1qFMgr4FN1HGXMaqpDKe2xDrGdv7Mmrcz5GoQlckv4G2A/c2qLJ6RExLOkoYJ2kJ7P/QYyRfSisBDhUM8b/NDIzs650PapH0jLgXOBPosV/GyJiOPu5G7gHmNft8czMrBxdJX5Ji4AvAh+PiJdbtDlY0iEjyzQKrT+e19bMzKrTyXDO24AfAidLGsqKq68ADqHRfbNJ0g1Z22MlrcmeejTwkKTNwMPA9yLi+xPyW5iZWcfG7eNvUWj9xhZth4FzsuVtwJxC0ZmZWek8c9fMLDFO/GZmiXHiNzNLTLKFWMqYADNZlPG7TpZCK3VRxaSjyXK+U5mgVSZf8ZuZJcaJ38wsMU78ZmaJceI3M0uME7+ZWWKc+M3MEuPEb2aWmHHH8UtaReP2y7tHCrFIuhL4S+DZrNllEbEm57mLgK8BU4BvRsQ1JcU94SbT2OCiBTXqMs5/PHUZl16HOSJVFEmp6nxPpvdiXXRbcxfguqyW7twWSX8K8HXgbOAUYKmkU4oEa2ZmxXVVc7dD84BnImJbRLwG3A4s7mI/ZmZWoiJ9/BdL2iJplaTDc7YfB+xoejyUrTMzsx7qNvFfD7wHmAvsBK7NaZNX9LdlLV1JyyUNShr8Da92GZaZmY2nq8QfEbsi4vWIeAP4Bvm1dIeAWU2PjweG2+xzZUQMRMTAVA7qJiwzM+tAtzV3j2l6+Enya+k+AsyWdJKkA4ElwOpujmdmZuXpZDjnbcACYKakIeAKYIGkuTS6brYDF2Ztj6UxbPOciNgv6WJgLY3hnKsi4okJ+S3MzKxjimjZ7d4zh2pGzNfCXodhJapiLkBdxvFXoQ7noqoaDB7H35kNsZ69sSfvu9UxPHPXzCwxTvxmZolx4jczS4wTv5lZYpz4zcwS48RvZpYYJ34zs8Q48ZuZJWbcmbtmZahqsk8/KKNIStHiOp0o43xXEae9fb7iNzNLjBO/mVlinPjNzBLTbbH1O4CTsyaHAS9ExJjOOknbgZeA14H9ETFQUtxmZtalTr7cvQlYAdwysiIiPj2yLOla4MU2zz8jIn7VbYBmZlaucRN/RDwg6cS8bZIEfAr4SLlhmZnZRCnax/+HwK6IeLrF9gDuk/SopOUFj2VmZiUoOo5/KXBbm+2nR8SwpKOAdZKejIgH8hpmHwzLAaYxvWBYNhlVUXykimNUEWcZyjhGGefTY/3L1/UVv6QDgD8G7mjVJiKGs5+7gXvIL8o+0tbF1s3MKlCkq+ejwJMRMZS3UdLBkg4ZWQbOJL8ou5mZVWjcxJ8VW/8hcLKkIUkXZJuWMKqbR9KxktZkD48GHpK0GXgY+F5EfL+80M3MrBudjOpZ2mL9Z3LWDQPnZMvbgDkF4zMzs5J55q6ZWWKc+M3MEuPEb2aWGCd+M7PEKCJ6HcMYh2pGzNfCXodhZjXgYi6d2RDr2Rt71ElbX/GbmSXGid/MLDFO/GZmiXHiNzNLjBO/mVlinPjNzBLjxG9mlphajuOX9Czw86ZVM4F+qNvbD3H2Q4zgOMvmOMtVxzhPiIgjO2lYy8Q/mqTBiBjodRzj6Yc4+yFGcJxlc5zl6pc4W3FXj5lZYpz4zcwS0y+Jf2WvA+hQP8TZDzGC4yyb4yxXv8SZqy/6+M3MrDz9csVvZmYlqXXil7RI0k8kPSPpS72OpxVJ2yU9JmmTpMFexzNC0ipJuyU93rRuhqR1kp7Ofh7eyxizmPLivFLSL7NzuknSOb2MMYtplqT7JW2V9ISkS7L1tTmnbWKs1fmUNE3Sw5I2Z3H+bbb+JEkbsnN5h6QDaxrnTZJ+1nQ+++re0LXt6pE0BXgK+BgwBDwCLI2IH/c0sByStgMDEVGrcb2S/gjYB9wSEadm6/4B2BMR12QfpodHxBdrGOeVwL6I+KdextZM0jHAMRGxUdIhwKPAJ4DPUJNz2ibGT1Gj8ylJwMERsU/SVOAh4BLgr4C7I+J2STcAmyPi+hrGeRHw3Yj4dq9iK6LOV/zzgGciYltEvAbcDizucUx9JSIeAPaMWr0YuDlbvplGUuipFnHWTkTsjIiN2fJLwFbgOGp0TtvEWCvRsC97ODX7F8BHgJFk2vPXZ5s4+1qdE/9xwI6mx0PU8AWcCeA+SY9KWt7rYMZxdETshEaSAI7qcTztXCxpS9YV1PMuqWaSTgTeB2ygpud0VIxQs/MpaYqkTcBuYB3wU+CFiNifNanFe350nBExcj6vzs7ndZIO6mGIb1udE39eCbG6ftKeHhHvB84GPp91XVgx1wPvAeYCO4FrexvOmyS9A7gLuDQi9vY6njw5MdbufEbE6xExFziexv/wfz+vWbVR5QQwKk5JpwJfBn4P+ANgBtDT7tK3q86JfwiY1fT4eGC4R7G0FRHD2c/dwD00XsR1tSvrBx7pD97d43hyRcSu7A33BvANanJOs37eu4BbI+LubHWtzmlejHU9nwAR8QLw38AHgcMkHZBtqtV7vinORVmXWkTEq8C/UaPz2Yk6J/5HgNnZt/wHAkuA1T2OaQxJB2dfoiHpYOBM4PH2z+qp1cCybHkZ8J0extLSSCLNfJIanNPsi74bga0R8ZWmTbU5p61irNv5lHSkpMOy5d8GPkrj+4j7gfOyZj1/fbaI88mmD3rR+B6i56/Pt6O2o3oAsiFnXwWmAKsi4uoehzSGpHfTuMoHOAD4Vl3ilHQbsIDGnQR3AVcA9wJ3Au8CfgGcHxE9/WK1RZwLaHRLBLAduHCkH71XJH0YeBB4DHgjW30ZjT70WpzTNjEupUbnU9JpNL68nULjAvTOiLgqez/dTqP75H+AP82uqusW5w+AI2l0SW8CLmr6Erj2ap34zcysfHXu6jEzswngxG9mlhgnfjOzxDjxm5klxonfzCwxTvxmZolx4jczS4wTv5lZYv4PkSb1hQw8Pa4AAAAASUVORK5CYII=", "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "plt.imshow(mask)" ] }, { "cell_type": "code", "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "# Loops through every record and slices off a single record for each pixel to make a single frame and then exports\n", "# an array of frames\n", "def create_frames(data,height, width, depth):\n", " days = []\n", " frames = []\n", " for i in tqdm(range(depth)):\n", " frame = np.zeros((height,width,7))\n", " frame[:,:,:] = np.nan\n", " for y in range(height):\n", " for x in range(width):\n", " if(not data[y][x].empty):\n", " frame[y][x] = data[y][x].iloc[[i],1:8].values.flatten()\n", " if((i+1)%24 != 0):\n", " frames.append(frame)\n", " else:\n", " frames.append(frame)\n", " days.append(frames)\n", " frames = []\n", " return np.array(days)" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "# Returns an array of neighbors to be used in average_grid\n", "def get_neighbors(x,y,g):\n", " neighbors = []\n", " for i in [y-1,y,y+1]:\n", " for j in [x-1,x,x+1]:\n", " if(i >= 0 and j >= 0):\n", " if(i != y or j != x ):\n", " try:\n", " neighbors.append(g[i][j])\n", " except:\n", " pass\n", " return np.array(neighbors)" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "# Merges the average frame with the true data frame to create an averaged frame\n", "def merge_frames(average, mask, data, height, width):\n", " for i in range(height):\n", " for j in range(width):\n", " if(mask[i][j] != 1):\n", " data[i][j] = average[i][j]" ] }, { "cell_type": "code", "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "# Takes a frame and then computes and average based on neighbor pixels without causing data drift\n", "def average_grid(mask,data, height, width):\n", " temp_frame = [([0] * width) for i in range(height)]\n", " for i in range(height):\n", " for j in range(width):\n", " if(mask[i][j] != 1):\n", " neighbors = get_neighbors(j,i,data)\n", " temp_frame[i][j] = np.nanmean(neighbors, axis=0)\n", " merge_frames(temp_frame, mask, data, height, width)" ] }, { "cell_type": "code", "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "# Used to break frames into training, validation, and testing sets and store them in hickle files along with the\n", "# sources for each set\n", "def store_sequence(frames):\n", " import hickle as hkl\n", " train_frames = []\n", " train_sources = []\n", " validation_frames = []\n", " validation_sources = []\n", " test_frames = []\n", " test_sources = []\n", " \n", " for day in range(len(frames)):\n", " for hour in range(len(frames[day])):\n", " if(day/len(frames) < .6):\n", " train_sources.append(\"{}\".format(day))\n", " train_frames.append(frames[day][hour])\n", " elif(day/len(frames) >= .6 and day/len(frames) < .8):\n", " validation_sources.append(\"{}\".format(day))\n", " validation_frames.append(frames[day][hour])\n", " else:\n", " test_sources.append(\"{}\".format(day))\n", " test_frames.append(frames[day][hour])\n", " \n", " print(\"There are {} frames and {} sources in training set\".format(len(train_frames), len(train_sources)))\n", " print(\"There are {} frames and {} sources in validation set\".format(len(validation_frames), len(validation_sources)))\n", " print(\"There are {} frames and {} sources in testing set\".format(len(test_frames), len(test_sources)))\n", " \n", " hkl.dump(np.array(train_frames), '../data/x_train.hkl')\n", " hkl.dump(train_sources, '../data/sources_train.hkl')\n", " hkl.dump(np.array(validation_frames), '../data/x_val.hkl')\n", " hkl.dump(validation_sources, '../data/sources_val.hkl')\n", " hkl.dump(np.array(test_frames), '../data/x_test.hkl')\n", " hkl.dump(test_sources, '../data/sources_test.hkl')\n", " " ] }, { "cell_type": "code", "execution_count": 18, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "100%|██████████████████████████████████████████████████████████████████████████| 17520/17520 [1:12:35<00:00, 4.02it/s]\n" ] } ], "source": [ "frames = create_frames(grid, height, width,depth)" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "ename": "NameError", "evalue": "name 'plt' is not defined", "output_type": "error", "traceback": [ "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[1;31mNameError\u001b[0m Traceback (most recent call last)", "\u001b[1;32m\u001b[0m in \u001b[0;36m\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mfig\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mplt\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfigure\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfigsize\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m15\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnorm\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2\u001b[0m \u001b[0mcolumns\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m3\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[0mrows\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m4\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m+\u001b[0m\u001b[0mrows\u001b[0m \u001b[1;33m+\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[0mfig\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_subplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrows\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", "\u001b[1;31mNameError\u001b[0m: name 'plt' is not defined" ] } ], "source": [ "# Plotting the newly made frame\n", "fig=plt.figure(figsize=(15,10))\n", "columns = 3\n", "rows = 4\n", "for i in range(1,columns+rows +1):\n", " fig.add_subplot(rows,columns,i)\n", " plt.imshow(frames[0,0,:,:,i-1])\n", " plt.ylabel(header_wanted[i-1], fontsize=10)\n" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ " 0%| | 0/730 [00:00" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "# Plotting a newly averaged frame\n", "fig=plt.figure(figsize=(15,10))\n", "columns = 3\n", "rows = 4\n", "for i in range(1,columns+rows +1):\n", " fig.add_subplot(rows,columns,i)\n", " plt.imshow(frames[0,0,:,:,i-1])\n", " plt.ylabel(header_wanted[i-1], fontsize=10)" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "frames.shape" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "There are 10512 frames and 10512 sources in training set\n", "There are 3504 frames and 3504 sources in validation set\n", "There are 3504 frames and 3504 sources in testing set\n" ] } ], "source": [ "store_sequence(frames)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "At this point the data has been processed and made into discrete frames and it is time to run it through the PredNet architecture for training." ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.4" } }, "nbformat": 4, "nbformat_minor": 2 }