diff --git a/1_to_1_multi_layer.ipynb b/1_to_1_multi_layer.ipynb index 57f4160..c850ffc 100644 --- a/1_to_1_multi_layer.ipynb +++ b/1_to_1_multi_layer.ipynb @@ -75,25 +75,25 @@ "source": [ "### Reshape original array into the shape (particlenumber, timesteps, input = coordinates)###\n", "\n", - "def reshapor(arr_orig):\n", - " timesteps = int(arr_orig.shape[1]/3)\n", + "def reshapor(arr_orig, num_inputs=3):\n", + " timesteps = int(arr_orig.shape[1]/num_inputs)\n", " number_examples = int(arr_orig.shape[0])\n", - " arr = np.zeros((number_examples, timesteps, 3))\n", + " arr = np.zeros((number_examples, timesteps, num_inputs))\n", " \n", " for i in range(number_examples):\n", " for t in range(timesteps):\n", - " arr[i,t,0:3] = arr_orig[i,3*t:3*t+3]\n", + " arr[i,t,:] = arr_orig[i,num_inputs*t:num_inputs*t+num_inputs]\n", " \n", " return arr\n", "\n", - "def reshapor_inv(array_shaped):\n", + "def reshapor_inv(array_shaped, num_inputs=3):\n", " timesteps = int(array_shaped.shape[1])\n", " num_examples = int(array_shaped.shape[0])\n", - " arr = np.zeros((num_examples, timesteps*3))\n", + " arr = np.zeros((num_examples, timesteps*num_inputs))\n", " \n", " for i in range(num_examples):\n", " for t in range(timesteps):\n", - " arr[i,3*t:3*t+3] = array_shaped[i,t,:]\n", + " arr[i,num_inputs*t:num_inputs*t+num_inputs] = array_shaped[i,t,:]\n", " \n", " return arr" ] @@ -137,7 +137,7 @@ "metadata": {}, "outputs": [], "source": [ - "train_set, test_set = create_random_sets(tset, 0.99)\n", + "train_set, test_set = create_random_sets(tset, 0.9)\n", "\n", "#print(test_set.shape, train_set.shape, reshapor(tset).shape)\n", "#print(test_set[0,:,:])" @@ -150,6 +150,13 @@ "outputs": [], "source": [ "#Normalize the data advanced version with scikit learn\n", + "def correct_array_steps(arr, steps= 8, num_inputs= 3): #steps > array_steps\n", + " if arr.shape[1] != steps:\n", + " _ = np.zeros((arr.shape[0], steps, num_inputs))\n", + " _[:,:arr.shape[1],:] += arr\n", + " arr = _\n", + " return arr\n", + "\n", "\n", "#set the transormation based on training set\n", "def set_min_max_scaler(arr, feature_range= (-1,1)):\n", @@ -164,49 +171,15 @@ "\n", "\n", "#transform data\n", - "def min_max_scaler(arr, min_max_scalor= min_max_scalor):\n", - " \n", - " if len(arr.shape) == 3:\n", - " if arr.shape[1] == 8:\n", - " arr = reshapor(min_max_scalor.transform(reshapor_inv(arr)))\n", - " else: \n", - " arr_ = np.zeros((arr.shape[0],24))\n", - " arr = reshapor_inv(arr)\n", - " arr_[:,:arr.shape[1]] += arr\n", - " arr = min_max_scalor.transform(arr_)[:,:arr.shape[1]]\n", - " arr = reshapor(arr)\n", - " \n", - " else:\n", - " if arr.shape[1] == 24:\n", - " arr = min_max_scalor.transform(arr)\n", - " else:\n", - " arr_ = np.zeros((arr.shape[0],24))\n", - " arr_[:,:arr.shape[1]] += arr\n", - " arr = min_max_scalor.transform(arr_)[:,:arr.shape[1]]\n", - " \n", + "def min_max_scaler(arr, min_max_scalor= min_max_scalor, num_inputs= 3):\n", + " arr = correct_array_steps(arr)\n", + " arr = reshapor(min_max_scalor.transform(reshapor_inv(arr)))\n", " return arr\n", " \n", "#inverse transformation\n", - "def min_max_scaler_inv(arr, min_max_scalor= min_max_scalor):\n", - " \n", - " if len(arr.shape) == 3:\n", - " if arr.shape[1] == 8:\n", - " arr = reshapor(min_max_scalor.inverse_transform(reshapor_inv(arr)))\n", - " else: \n", - " arr_ = np.zeros((arr.shape[0],24))\n", - " arr = reshapor_inv(arr)\n", - " arr_[:,:arr.shape[1]] += arr\n", - " arr = min_max_scalor.inverse_transform(arr_)[:,:arr.shape[1]]\n", - " arr = reshapor(arr)\n", - " \n", - " else:\n", - " if arr.shape[1] == 24:\n", - " arr = min_max_scalor.inverse_transform(arr)\n", - " else:\n", - " arr_ = np.zeros((arr.shape[0],24))\n", - " arr_[:,:arr.shape[1]] += arr\n", - " arr = min_max_scalor.nverse_transform(arr_)[:,:arr.shape[1]]\n", - " \n", + "def min_max_scaler_inv(arr, min_max_scalor= min_max_scalor, num_inputs= 3):\n", + " arr = correct_array_steps(arr)\n", + " arr = reshapor(min_max_scalor.inverse_transform(reshapor_inv(arr)))\n", " return arr" ] }, @@ -222,57 +195,23 @@ "def set_std_scaler(arr):\n", " std_scalor = preprocessing.StandardScaler()\n", " if len(arr.shape) == 3:\n", - " arr = reshapor(std_scalor.fit(reshapor_inv(arr))) \n", + " arr = reshapor(std_scalor.fit_transform(reshapor_inv(arr))) \n", " else:\n", - " arr = std_scalor.fit(arr)\n", + " arr = std_scalor.fit_transform(arr)\n", " return std_scalor\n", "\n", "std_scalor = set_std_scaler(train_set)\n", "\n", "#transform data\n", - "def std_scaler(arr, std_scalor= std_scalor):\n", - " \n", - " if len(arr.shape) == 3:\n", - " if arr.shape[1] == 8:\n", - " arr = reshapor(std_scalor.transform(reshapor_inv(arr)))\n", - " else: \n", - " arr_ = np.zeros((arr.shape[0],24))\n", - " arr = reshapor_inv(arr)\n", - " arr_[:,:arr.shape[1]] += arr\n", - " arr = std_scalor.transform(arr_)[:,:arr.shape[1]]\n", - " arr = reshapor(arr)\n", - " \n", - " else:\n", - " if arr.shape[1] == 24:\n", - " arr = std_scalor.transform(arr)\n", - " else:\n", - " arr_ = np.zeros((arr.shape[0],24))\n", - " arr_[:,:arr.shape[1]] += arr\n", - " arr = std_scalor.transform(arr_)[:,:arr.shape[1]]\n", - " \n", + "def std_scaler(arr, std_scalor= std_scalor, num_inputs=3):\n", + " arr = correct_array_steps(arr)\n", + " arr = reshapor(std_scalor.transform(reshapor_inv(arr)))\n", " return arr\n", " \n", "#inverse transformation\n", - "def std_scaler_inv(arr, std_scalor= std_scalor):\n", - " \n", - " if len(arr.shape) == 3:\n", - " if arr.shape[1] == 8:\n", - " arr = reshapor(std_scalor.inverse_transform(reshapor_inv(arr)))\n", - " else: \n", - " arr_ = np.zeros((arr.shape[0],24))\n", - " arr = reshapor_inv(arr)\n", - " arr_[:,:arr.shape[1]] += arr\n", - " arr = std_scalor.inverse_transform(arr_)[:,:arr.shape[1]]\n", - " arr = reshapor(arr)\n", - " \n", - " else:\n", - " if arr.shape[1] == 24:\n", - " arr = std_scalor.inverse_transform(arr)\n", - " else:\n", - " arr_ = np.zeros((arr.shape[0],24))\n", - " arr_[:,:arr.shape[1]] += arr\n", - " arr = std_scalor.inverse_transform(arr_)[:,:arr.shape[1]]\n", - " \n", + "def std_scaler_inv(arr, std_scalor= std_scalor, num_inputs=3):\n", + " arr = correct_array_steps(arr)\n", + " arr = reshapor(std_scalor.inverse_transform(reshapor_inv(arr)))\n", " return arr\n", "\n" ] @@ -300,9 +239,14 @@ "#Scale data either with MinMax scaler or with Standard scaler\n", "#Return scalor if fit = True and and scaled array otherwise\n", "\n", - "def scaler(arr, std_scalor= std_scalor, min_max_scalor= min_max_scalor, scalerfunc= \"std\"):\n", + "def scaler(arr, std_scalor= std_scalor, min_max_scalor= min_max_scalor, scalerfunc= \"minmax\", scalor = False):\n", " \n", - " if scalerfunc == \"std\":\n", + " if scalor != False:\n", + " arr = correct_array_steps(arr)\n", + " arr = reshapor(scalor.transform(reshapor_inv(arr)))\n", + " return arr\n", + " \n", + " elif scalerfunc == \"std\":\n", " arr = std_scaler(arr, std_scalor= std_scalor)\n", " return arr\n", " \n", @@ -313,14 +257,19 @@ " else:\n", " raise ValueError(\"Uknown scaler chosen: {}\".format(scalerfunc))\n", "\n", - "def scaler_inv(arr, std_scalor= std_scalor, min_max_scalor= min_max_scalor, scalerfunc= \"std\"):\n", + "def scaler_inv(arr, std_scalor= std_scalor, min_max_scalor= min_max_scalor, scalerfunc= \"std\", scalor = False, num_inputs= 3):\n", "\n", - " if scalerfunc == \"std\":\n", + " if scalor != False:\n", + " arr = correct_array_steps(arr)\n", + " arr = reshapor(scalor.inverse_transform(reshapor_inv(arr)))\n", + " return arr\n", + " \n", + " elif scalerfunc == \"std\":\n", " arr = std_scaler_inv(arr, std_scalor= std_scalor)\n", " return arr\n", " \n", " elif scalerfunc == \"minmax\":\n", - " arr = min_max_scaler_inv(arr, min_max_scalor= std_scalor)\n", + " arr = min_max_scaler_inv(arr, min_max_scalor= min_max_scalor)\n", " return arr\n", " \n", " else:\n", @@ -362,7 +311,7 @@ " p = np.random.permutation(a.shape[0])\n", " return a[p,:,:], b[p,:,:]\n", "\n", - "def random_mini_batches(inputt, target, minibatch_size = 500):\n", + "def random_mini_batches(inputt, target, minibatch_size = 100):\n", " \n", " num_examples = inputt.shape[0]\n", " \n", @@ -463,6 +412,7 @@ " self._ = cell_type #later used to create folder name\n", " self.__ = activation #later used to create folder name\n", " self.loss_list = []\n", + " self.loss_validation = []\n", " self.scalor = scalor\n", " \n", " #### The input is of shape (num_examples, time_steps, ninputs)\n", @@ -555,15 +505,16 @@ "\n", " \n", " \n", - " def fit(self, minibatches, epochs, print_step, checkpoint = 5, patience = 200):\n", + " def fit(self, minibatches, epochs, print_step, validation_input, validation_output, checkpoint = 5, patience = 20, patience_trigger= 2./10**6):\n", " patience_cnt = 0\n", " start = len(self.loss_list)\n", " epoche_save = start\n", " \n", - " folder = \"./rnn_model_\" + str(self._)+ \"_\" + self.__ + \"_\" + str(self.ncells).replace(\" \",\"\") + \"c\" + \"_checkpoint/rnn_basic\"\n", + " folder = \"./rnn_model_\" + str(self._)+ \"_\" + self.__ + \"_\" + str(self.ncells).replace(\" \",\"\") + \"c/rnn_basic\"\n", " \n", " for iep in range(start, start + epochs):\n", " loss = 0\n", + " loss_val = 0\n", " \n", " batches = len(minibatches)\n", " #Here I iterate over the batches\n", @@ -575,49 +526,54 @@ " \n", " \n", " loss += self.sess.run(self.cost, feed_dict={self.X:train, self.Y:target})\n", + " loss_val += rnn.sess.run(self.cost, feed_dict={self.X:validation_input, rnn.Y:validation_output})\n", " \n", " #Normalize loss over number of batches and scale it back before normaliziation\n", " loss /= batches\n", + " loss_val /= batches\n", " self.loss_list.append(loss)\n", + " self.loss_validation.append(loss_val)\n", " \n", " #print(loss)\n", " \n", " #Here I create the checkpoint if the perfomance is better\n", - " if iep > 1 and iep%checkpoint == 0 and self.loss_list[iep] < self.loss_list[epoche_save]:\n", + " if iep > 1 and iep%checkpoint == 0 and self.loss_validation[iep] < self.loss_validation[epoche_save]:\n", " #print(\"Checkpoint created at epoch: \", iep)\n", " self.save(folder)\n", " epoche_save = iep\n", " \n", " #early stopping with patience\n", - " if iep > 1 and abs(self.loss_list[iep]-self.loss_list[iep-1]) < 1.5/10**7:\n", + " if iep > 1 and abs(self.loss_validation[iep]-self.loss_validation[iep-1]) < patience_trigger:\n", " patience_cnt += 1\n", " #print(\"Patience now at: \", patience_cnt, \" of \", patience)\n", " \n", " if patience_cnt + 1 > patience:\n", - " print(\"\\n\", \"Early stopping at epoch \", iep, \", difference: \", abs(self.loss_list[iep]-self.loss_list[iep-1]))\n", - " print(\"Cost: \",loss)\n", + " print(\"\\n\", \"Early stopping at epoch \", iep, \", difference: \", abs(self.loss_validation[iep]-self.loss_validation[iep-1]))\n", + " print(\"Cost: \", loss*10**6, \"e-6\")\n", + " print(\"Cost on valdiation_set: \",loss_val*10**6, \"e-6\")\n", " break\n", " \n", " #Note that the loss here is multiplied with 1000 for easier reading\n", " if iep%print_step==0:\n", " print(\"Epoch number \",iep)\n", " print(\"Cost: \",loss*10**6, \"e-6\")\n", + " print(\"Cost on validation_set: \",loss_val*10**6, \"e-6\")\n", " print(\"Patience: \",patience_cnt, \"/\", patience)\n", " print(\"Last checkpoint at: Epoch \", epoche_save, \"\\n\")\n", " \n", " #Set model back to the last checkpoint if performance was better\n", - " if self.loss_list[epoche_save] < self.loss_list[iep]:\n", + " if self.loss_validation[epoche_save] < self.loss_validation[iep]:\n", " self.load(folder)\n", " print(\"\\n\")\n", " print(\"State of last checkpoint checkpoint at epoch \", epoche_save, \" restored\")\n", " print(\"Performance at last checkpoint is \" ,(self.loss_list[iep] - self.loss_list[epoche_save])/self.loss_list[iep]*100, \"% better\" )\n", + "\n", " \n", - " folder = \"./rnn_model_\" + str(self._)+ \"_\" + self.__ + \"_\" + str(self.ncells).replace(\" \",\"\") + \"c/rnn_basic\"\n", - " self.save(folder)\n", " print(\"\\n\")\n", " print(\"Model saved in at: \", folder)\n", " \n", " \n", + " \n", " \n", " def predict(self, x):\n", " return self.sess.run(self.output, feed_dict={self.X:x})\n", @@ -648,7 +604,8 @@ " \"cell_type\": rnn._, #cell_type\n", " \"activation\": rnn.__, #Activation\n", " \"loss_list\": rnn.loss_list,\n", - " \"scalor\": rnn.scalor}\n", + " \"scalor\": rnn.scalor,\n", + " \"loss_validation\": rnn.loss_validation}\n", " \n", " if train == True:\n", " pkl_dic[\"minibatches\"] = minibatches\n", @@ -695,8 +652,8 @@ " minibatches = pkl_dic[\"minibatches\"]\n", " if \"test_input\" in pkl_dic:\n", " test = True\n", - " test_input = [\"test_input\"]\n", - " test_target = [\"test_target\"]\n", + " test_input = pkl_dic[\"test_input\"]\n", + " test_target = pkl_dic[\"test_target\"]\n", " \n", " #loads and initializes a new model with the exact same properties\n", " \n", @@ -710,6 +667,8 @@ " \n", " rnn.loss_list = pkl_dic[\"loss_list\"]\n", " \n", + " rnn.loss_validation = pkl_dic[\"loss_validation\"]\n", + " \n", " print(\"Model succesfully loaded\")\n", " \n", " if batch and test:\n", @@ -732,8 +691,7 @@ " print(\"Only Model restored, no trainig or test data found in {}\".format(pkl_name))\n", " print(\"Returned data is empty!\")\n", " return rnn, data\n", - " \n", - " \n", + "\n", "#returns the folder name used by full_save and full_load for a given architecture\n", "def get_rnn_folder(ncells, cell_type, activation):\n", " folder = \"./rnn_model_\" + cell_type + \"_\" + activation + \"_\" + str(ncells).replace(\" \",\"\") + \"c/rnn_basic\"\n", @@ -752,7 +710,9 @@ "ninputs = 3\n", "\n", "#ncells as int or list of int\n", - "ncells = [50, 40, 30, 20, 10]\n", + "ncells = [100, 100, 100]\n", + "activation = \"leaky_relu\"\n", + "cell_type = \"lstm\"\n", "\n", "num_output = 3" ] @@ -806,20 +766,318 @@ "output_type": "stream", "text": [ "Epoch number 0\n", - "Cost: 376274.61996484315 e-6\n", - "Patience: 0 / 200\n", + "Cost: 316163.4969156842 e-6\n", + "Cost on validation_set: 317156.1284120692 e-6\n", + "Patience: 0 / 20\n", "Last checkpoint at: Epoch 0 \n", "\n", + "Epoch number 10\n", + "Cost: 17620.41081349517 e-6\n", + "Cost on validation_set: 17808.353961553683 e-6\n", + "Patience: 0 / 20\n", + "Last checkpoint at: Epoch 10 \n", + "\n", + "Epoch number 20\n", + "Cost: 12158.986289314058 e-6\n", + "Cost on validation_set: 12332.019157880959 e-6\n", + "Patience: 0 / 20\n", + "Last checkpoint at: Epoch 20 \n", + "\n", + "Epoch number 30\n", + "Cost: 8429.908161159865 e-6\n", + "Cost on validation_set: 8557.763253880103 e-6\n", + "Patience: 0 / 20\n", + "Last checkpoint at: Epoch 30 \n", + "\n", + "Epoch number 40\n", + "Cost: 6765.2886581802095 e-6\n", + "Cost on validation_set: 6839.355951998123 e-6\n", + "Patience: 0 / 20\n", + "Last checkpoint at: Epoch 40 \n", + "\n", + "Epoch number 50\n", + "Cost: 5792.310645500588 e-6\n", + "Cost on validation_set: 5863.479143658349 e-6\n", + "Patience: 0 / 20\n", + "Last checkpoint at: Epoch 50 \n", + "\n", + "Epoch number 60\n", + "Cost: 5376.994696467422 e-6\n", + "Cost on validation_set: 5446.046904855689 e-6\n", + "Patience: 0 / 20\n", + "Last checkpoint at: Epoch 60 \n", + "\n", + "Epoch number 70\n", + "Cost: 4835.011047679324 e-6\n", + "Cost on validation_set: 4899.287358099638 e-6\n", + "Patience: 0 / 20\n", + "Last checkpoint at: Epoch 70 \n", + "\n", + "Epoch number 80\n", + "Cost: 4453.0257474371165 e-6\n", + "Cost on validation_set: 4506.959038418393 e-6\n", + "Patience: 0 / 20\n", + "Last checkpoint at: Epoch 80 \n", + "\n", + "Epoch number 90\n", + "Cost: 4041.3462069578645 e-6\n", + "Cost on validation_set: 4103.328707779563 e-6\n", + "Patience: 0 / 20\n", + "Last checkpoint at: Epoch 90 \n", + "\n", + "Epoch number 100\n", + "Cost: 3746.72157949833 e-6\n", + "Cost on validation_set: 3810.4774051367544 e-6\n", + "Patience: 0 / 20\n", + "Last checkpoint at: Epoch 100 \n", + "\n", + "Epoch number 110\n", + "Cost: 3510.6311173199915 e-6\n", + "Cost on validation_set: 3572.610001144714 e-6\n", + "Patience: 1 / 20\n", + "Last checkpoint at: Epoch 110 \n", + "\n", + "Epoch number 120\n", + "Cost: 3328.6676720477817 e-6\n", + "Cost on validation_set: 3388.9460420712485 e-6\n", + "Patience: 1 / 20\n", + "Last checkpoint at: Epoch 120 \n", + "\n", + "Epoch number 130\n", + "Cost: 3178.6257574378055 e-6\n", + "Cost on validation_set: 3237.9631613662773 e-6\n", + "Patience: 1 / 20\n", + "Last checkpoint at: Epoch 130 \n", + "\n", + "Epoch number 140\n", + "Cost: 3045.4906938207705 e-6\n", + "Cost on validation_set: 3103.9184121804874 e-6\n", + "Patience: 1 / 20\n", + "Last checkpoint at: Epoch 140 \n", + "\n", + "Epoch number 150\n", + "Cost: 3014.904371094565 e-6\n", + "Cost on validation_set: 3073.1504057469065 e-6\n", + "Patience: 1 / 20\n", + "Last checkpoint at: Epoch 150 \n", + "\n", + "Epoch number 160\n", + "Cost: 2844.7661992855546 e-6\n", + "Cost on validation_set: 2901.79907257647 e-6\n", + "Patience: 1 / 20\n", + "Last checkpoint at: Epoch 160 \n", + "\n", + "Epoch number 170\n", + "Cost: 2818.9633171572245 e-6\n", + "Cost on validation_set: 2878.737646739843 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 165 \n", + "\n", + "Epoch number 180\n", + "Cost: 2705.8817240474527 e-6\n", + "Cost on validation_set: 2767.457837931985 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 180 \n", + "\n", + "Epoch number 190\n", + "Cost: 2802.5307259414085 e-6\n", + "Cost on validation_set: 2868.40574902504 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 185 \n", + "\n", + "Epoch number 200\n", + "Cost: 2473.8708554312243 e-6\n", + "Cost on validation_set: 2533.3558035971123 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 200 \n", + "\n", + "Epoch number 210\n", + "Cost: 2456.7929551351904 e-6\n", + "Cost on validation_set: 2513.4514421657764 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 210 \n", + "\n", + "Epoch number 220\n", + "Cost: 2420.2582480515853 e-6\n", + "Cost on validation_set: 2477.84735375019 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 220 \n", + "\n", + "Epoch number 230\n", + "Cost: 2906.6367897876476 e-6\n", + "Cost on validation_set: 2968.231827930309 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 225 \n", + "\n", + "Epoch number 240\n", + "Cost: 2734.680800850308 e-6\n", + "Cost on validation_set: 2792.3258075620547 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 225 \n", + "\n", + "Epoch number 250\n", + "Cost: 2242.8401929954457 e-6\n", + "Cost on validation_set: 2296.759985214056 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 250 \n", + "\n", + "Epoch number 260\n", + "Cost: 2807.354240483323 e-6\n", + "Cost on validation_set: 2874.5259948854527 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 250 \n", + "\n", + "Epoch number 270\n", + "Cost: 2101.4384015701535 e-6\n", + "Cost on validation_set: 2161.6989031953867 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 270 \n", + "\n", + "Epoch number 280\n", + "Cost: 2128.7833589540664 e-6\n", + "Cost on validation_set: 2190.027813708713 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 275 \n", + "\n", + "Epoch number 290\n", + "Cost: 2030.9363811298513 e-6\n", + "Cost on validation_set: 2090.106961902144 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 290 \n", + "\n", + "Epoch number 300\n", + "Cost: 2049.279973170785 e-6\n", + "Cost on validation_set: 2111.154213157851 e-6\n", + "Patience: 2 / 20\n", + "Last checkpoint at: Epoch 290 \n", + "\n", + "Epoch number 310\n", + "Cost: 1975.3764973693462 e-6\n", + "Cost on validation_set: 2037.5676658870868 e-6\n", + "Patience: 3 / 20\n", + "Last checkpoint at: Epoch 310 \n", + "\n", + "Epoch number 320\n", + "Cost: 2357.8102402605637 e-6\n", + "Cost on validation_set: 2425.418484436218 e-6\n", + "Patience: 3 / 20\n", + "Last checkpoint at: Epoch 315 \n", + "\n", + "Epoch number 330\n", + "Cost: 1948.3688953440897 e-6\n", + "Cost on validation_set: 2011.0500397074013 e-6\n", + "Patience: 3 / 20\n", + "Last checkpoint at: Epoch 330 \n", + "\n", + "Epoch number 340\n", + "Cost: 1970.0811163327376 e-6\n", + "Cost on validation_set: 2031.9556784924378 e-6\n", + "Patience: 4 / 20\n", + "Last checkpoint at: Epoch 330 \n", + "\n", + "Epoch number 350\n", + "Cost: 1940.5714647714483 e-6\n", + "Cost on validation_set: 2004.524148719082 e-6\n", + "Patience: 5 / 20\n", + "Last checkpoint at: Epoch 350 \n", + "\n", + "Epoch number 360\n", + "Cost: 1968.3953304233592 e-6\n", + "Cost on validation_set: 2038.7437665661755 e-6\n", + "Patience: 5 / 20\n", + "Last checkpoint at: Epoch 355 \n", + "\n", + "Epoch number 370\n", + "Cost: 1976.2183642448033 e-6\n", + "Cost on validation_set: 2046.2597617446338 e-6\n", + "Patience: 5 / 20\n", + "Last checkpoint at: Epoch 355 \n", + "\n", + "Epoch number 380\n", + "Cost: 2104.1538024884326 e-6\n", + "Cost on validation_set: 2173.166846596571 e-6\n", + "Patience: 5 / 20\n", + "Last checkpoint at: Epoch 355 \n", + "\n", + "Epoch number 390\n", + "Cost: 1913.8040573357841 e-6\n", + "Cost on validation_set: 1984.506214665639 e-6\n", + "Patience: 6 / 20\n", + "Last checkpoint at: Epoch 385 \n", + "\n", + "Epoch number 400\n", + "Cost: 1854.3496478868778 e-6\n", + "Cost on validation_set: 1922.2521764594455 e-6\n", + "Patience: 6 / 20\n", + "Last checkpoint at: Epoch 400 \n", + "\n", + "Epoch number 410\n", + "Cost: 1861.1944058563472 e-6\n", + "Cost on validation_set: 1930.6607739350131 e-6\n", + "Patience: 6 / 20\n", + "Last checkpoint at: Epoch 400 \n", + "\n", + "Epoch number 420\n", + "Cost: 1866.6057852854908 e-6\n", + "Cost on validation_set: 1936.6092286830724 e-6\n", + "Patience: 6 / 20\n", + "Last checkpoint at: Epoch 400 \n", + "\n", + "Epoch number 430\n", + "Cost: 1881.5245793395957 e-6\n", + "Cost on validation_set: 1952.4438295859932 e-6\n", + "Patience: 6 / 20\n", + "Last checkpoint at: Epoch 400 \n", + "\n", + "Epoch number 440\n", + "Cost: 1911.4178085569727 e-6\n", + "Cost on validation_set: 1983.390877369878 e-6\n", + "Patience: 9 / 20\n", + "Last checkpoint at: Epoch 400 \n", + "\n", + "Epoch number 450\n", + "Cost: 1808.151681257715 e-6\n", + "Cost on validation_set: 1876.5825160973986 e-6\n", + "Patience: 9 / 20\n", + "Last checkpoint at: Epoch 450 \n", + "\n", + "Epoch number 460\n", + "Cost: 1807.536144719221 e-6\n", + "Cost on validation_set: 1876.1289024422335 e-6\n", + "Patience: 9 / 20\n", + "Last checkpoint at: Epoch 460 \n", + "\n", + "Epoch number 470\n", + "Cost: 1835.432499213967 e-6\n", + "Cost on validation_set: 1907.6892071901711 e-6\n", + "Patience: 9 / 20\n", + "Last checkpoint at: Epoch 460 \n", + "\n", + "Epoch number 480\n", + "Cost: 1824.4561668875258 e-6\n", + "Cost on validation_set: 1897.5317781401236 e-6\n", + "Patience: 13 / 20\n", + "Last checkpoint at: Epoch 460 \n", "\n", "\n", - "Model saved in at: ./rnn_model_lstm_leaky_relu_[50,40,30,20,10]c/rnn_basic\n", - "Model saved at: ./rnn_model_lstm_leaky_relu_[50,40,30,20,10]c/rnn_basic\n", - "Remaining data saved as: rnn_model_lstm_leaky_relu_[50,40,30,20,10]c.pkl\n" + " Early stopping at epoch 489 , difference: 5.850276021763991e-07\n", + "Cost: 0.0018202092466052882\n", + "Cost on valdiation_set: 0.0018940789727927294\n", + "INFO:tensorflow:Restoring parameters from ./rnn_model_lstm_leaky_relu_[100,100,100]c/rnn_basic\n", + "\n", + "\n", + "State of last checkpoint checkpoint at epoch 460 restored\n", + "Performance at last checkpoint is 0.6962442317938289 % better\n", + "\n", + "\n", + "Model saved in at: ./rnn_model_lstm_leaky_relu_[100,100,100]c/rnn_basic\n", + "Model saved at: ./rnn_model_lstm_leaky_relu_[100,100,100]c/rnn_basic\n", + "Remaining data saved as: rnn_model_lstm_leaky_relu_[100,100,100]c.pkl\n" ] } ], "source": [ - "rnn.fit(minibatches, epochs = 5, print_step=5)\n", + "rnn.fit(minibatches, epochs = 5000, print_step=10, validation_input = test_input, validation_output= test_target)\n", "full_save(rnn)" ] }, @@ -832,7 +1090,7 @@ "outputs": [ { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VGXexvHvLwkhhCoQegklFlCkhN4VEFcFO6AiKKI0EVF23XXd1911fdeyKCpFQHFlUUQUXxZURKRKDdJ7hyglgIB0As/7R4Y1soFJJDNnJnN/rivXzpw5k7k963jntOcx5xwiIiKXEuV1ABERCX0qCxER8UtlISIifqksRETEL5WFiIj4pbIQERG/VBYiIuKXykJERPxSWYiIiF8xXgfILSVLlnSJiYlexxARCStLly7d75xL8LdenimLxMREUlJSvI4hIhJWzGxHdtbTYSgREfFLZSEiIn6pLERExC+VhYiI+KWyEBERv1QWIiLil8pCRET8iviycM7x4ufr2Jp21OsoIiIhK+LLYtv+Y4xfvJP2Q+byxoxNnE4/53UkEZGQE/FlUTWhEF8/1ZJ2NUozePpGfvPGXJZsP+h1LBGRkBLxZQFQqnAcb91XlzEP1efE6bPcM2IBv/90FYdPnPE6mohISFBZZNL6qlJMH9iCns2r8NGSnbQZPJspK3/AOed1NBERT6ksLhAfG8Ozt9Rgcr9mlCkSR78PltHjnymk/njc62giIp5RWVzEteWLMqlPE567tQYLtx6g7eA5jJ67lfSzOgEuIpFHZXEJMdFR9GhWhekDW9KkWglemLqO24d9y+rvD3sdTUQkqFQW2VC+WAFGd0tm2P112XvkFB3emscLU9Zy7FS619FERIJCZZFNZsZvrivL1wNb0qVBJUbP20a71+bwzfq9XkcTEQk4lUUOFS2Qj7/dcR0TezUmPjaah99Loe+479h35KTX0UREAkZl8SslJxZnav/mPN3uSqav28uNg2czbtEOzp3TZbYikveoLC5DbEwU/W5I4ssnmnNtuaI8O2k19769gI17f/I6mohIrlJZ5IKqCYX4oGdDXrm7FpvTjnLLG3P5x1cbOHnmrNfRRERyhcoil5gZ9yRXZMbAltxWqxxvfrOZm4fMZf6W/V5HExG5bCqLXFaiUH4Gd6rN2B4NOHvOcd+oRQz6eAU/HjvtdTQRkV9NZREgzZMSmDagBb1bVWPSsu+5cfBsJi1L1ThTIhKWVBYBVCA2mt+1v5op/ZtRuUQ8T360ggffXcyOA8e8jiYikiMqiyC4ukwRJvZqwl871mTZzkO0e20Ow2dt4YzGmRKRMKGyCJLoKKNr40S+HtiS1leV4qUv13Pbm/NYtvNHr6OJiPilsgiyMkXjGNG1HiO71uPQ8TPcOXw+//N/q/nppCZaEpHQpbLwSLuaZfj6qZZ0a5zI+wt30HbwHKat2eN1LBGRLAW0LMysvZltMLPNZvZMFq/3MrNVZrbczOaZWQ3f8kQzO+FbvtzMRgQyp1cK5Y/h+Q41mdSnKVcUjOWxsUt59P0Udh8+4XU0EZFfsEBdymlm0cBGoC2QCiwBujjn1mZap4hz7ojvcQegj3OuvZklAlOcc9dm9/OSk5NdSkpKLv4TBNeZs+d4Z942Xv96IzFRUTzd7kq6Nk4kOsq8jiYieZiZLXXOJftbL5B7Fg2Azc65rc6508B4oGPmFc4XhU9BIGJvQsgXHUWvltX4akBL6lQqxvP/Xsudw+ezbvcR/28WEQmwQJZFeWBXpuepvmW/YGZ9zWwL8DLQP9NLVcxsmZnNNrPmWX2AmT1qZilmlpKWlpab2T1TqUQ87z/cgCGda5N68Di3vjmPv3+xnhOnNc6UiHgnkGWR1fGT/9pzcM4Ndc5VA34H/NG3eDdQyTlXBxgIfGBmRbJ470jnXLJzLjkhISEXo3vLzOhYuzwznmrJXXXLM2L2Ftq9Pps5G/NGIYpI+AlkWaQCFTM9rwD8cIn1xwO3AzjnTjnnDvgeLwW2AFcGKGfIKhYfy8t3X8+HPRuRLyqKB99dzIDxy9h/9JTX0UQkwgSyLJYASWZWxcxigc7A5MwrmFlSpqe3AJt8yxN8J8gxs6pAErA1gFlDWuNqJfj8ieb0vzGJqat202bwbCak7NI4UyISNAErC+dcOtAPmAasAyY459aY2V98Vz4B9DOzNWa2nIzDTd18y1sAK81sBTAR6OWcOxiorOEgLl80A9teyef9m5NUqhC/nbiSLqMWsjXtqNfRRCQCBOzS2WAL90tnc+LcOcdHKbt48fN1nEo/R7/W1enVshqxMbrHUkRyJhQunZUAiYoyujSoxIynWtKuRmkGT9/Ib96Yy5LtEb3zJSIBpLIIY6UKx/HWfXUZ070+J06f5Z4RC/j9p6s4fELjTIlI7lJZ5AGtry7F9IEt6Nm8Ch8t2UmbwbOZsvIHnQAXkVyjssgj4mNjePaWGkzu14wyReLo98EyevwzhdQfj3sdTUTyAJVFHnNt+aJM6tOEP95yDQu3HqDt4DmMnruVdE20JCKXQWWRB8VER/FI86p89WQLGlcrwQtT13H7sG9Z/f1hr6OJSJhSWeRhFa6I551uyQy9ry57j5yiw1vzeGHKWo6dSvc6moiEGZVFHmdm3FKrLF8PbEnnBpUYPW8b7V6bwzfr93odTUTCiMoiQhQtkI8X77iOj3s1Jj42moffS6HvuO/Yd+Sk19FEJAyoLCJM/cTiTO3fnKfaXsn0dXu5cfBsxi3awblzusxWRC5OZRGBYmOiePzGJL58ojk1yxXh2UmrufftBWzc+5PX0UQkRKksIljVhEJ82LMRr9xdi81pR7nljbn846sNnDyjiZZE5JdUFhHOzLgnuSIzBrbk1lrlePObzdw8ZC7zt+z3OpqIhBCVhQBQolB+XutUm7E9GnD2nOO+UYsY9PEKfjx22utoIhICVBbyC82TEpg2oAW9W1Xj02Xfc+Pg2UxalqpxpkQinMpC/kuB2Gh+1/5qpjzejErF43nyoxU8+O5idhw45nU0EfGIykIu6pqyRfikdxP+3KEmy3Yeot1rcxg+awtnNM6USMRRWcglRUcZ3ZokMn1gC1pdlcBLX67ntjfnsW73Ea+jiUgQqSwkW8oWLcDbXZN5u2s9Dh47zd3D5zN7Y5rXsUQkSFQWkiM31SzD5H7NqFSiIA+/t4Txi3d6HUlEgkBlITlWpmgcH/dqTLPqJXnm01W8Mm29rpYSyeNUFvKrFMofw+huyXRpUJGhM7cw4KPlnErXnd8ieVWM1wEkfOWLjuLFO66jYvF4Xv5yA7sPn2Rk13oUi4/1OpqI5DLtWchlMTP6tKrOkM61Wb7zEHcNn8+ug5r3WySvUVlIruhYuzxjezRg/9HT3DHsW1bsOuR1JBHJRSoLyTUNq5bgk95NKBAbTaeRC/hqzR6vI4lILlFZSK6qXqoQn/ZuylVlivDYv5by3rfbvI4kIrlAZSG5LqFwfsb3bESba0rz/L/X8tcpazUTn0iYU1lIQBSIjWbEA/Xo3iSRd+Zto8+47zSpkkgYU1lIwERHGc93qMmfbq3BtLV76DJqIQeOnvI6loj8CioLCbiHm1Vh+P31WPvDEe4cPp+taUe9jiQiOaSykKBof20Zxj/aiKMn07lz+HxSth/0OpKI5IDKQoKmTqUr+LRPE4rHx3Lf6EVMWfmD15FEJJtUFhJUlUsU5JPeTbi+QlH6fbCMEbO3aBBCkTCgspCgu6JgLGN7NOTWWmX5+xfr+eNnq0nX7HsiIU0DCYon4vJF80bnOlS4Ip4Rs7ew+/BJ3uxSh4L59a+kSCjSnoV4JirKeObmq/nbHdcya8M+Oo1cwL4jJ72OJSJZCGhZmFl7M9tgZpvN7JksXu9lZqvMbLmZzTOzGple+73vfRvM7KZA5hRv3d+wMu90q8/WtGPcMWw+G/f+5HUkEblAwMrCzKKBocDNQA2gS+Yy8PnAOXedc6428DIw2PfeGkBnoCbQHhjm+32SR7W+uhQTHmvMmbPnuGv4fOZv3u91JBHJJJB7Fg2Azc65rc6508B4oGPmFZxzRzI9LQicvyymIzDeOXfKObcN2Oz7fZKHXVu+KJP6NqVs0Ti6jVnMJ0tTvY4kIj6BLIvywK5Mz1N9y37BzPqa2RYy9iz65+S9kveUL1aAib2b0KBKcZ76eAWvf71Rl9aKhIBAloVlsey/vvXOuaHOuWrA74A/5uS9ZvaomaWYWUpaWtplhZXQUSQuH2O6N+CuuhV4/etNDJq4ktPpurRWxEuBLItUoGKm5xWAS92yOx64PSfvdc6NdM4lO+eSExISLjOuhJLYmChevacWA9okMXFpKg+9t5gjJ894HUskYgWyLJYASWZWxcxiyThhPTnzCmaWlOnpLcAm3+PJQGczy29mVYAkYHEAs0oIMjMGtLmSV++5nkVbD3L38Pl8f+iE17FEIlLAysI5lw70A6YB64AJzrk1ZvYXM+vgW62fma0xs+XAQKCb771rgAnAWuBLoK9zTpMhRKi761Xgnw83YPfhk9wx9FtWf3/Y60giEcfyysnD5ORkl5KS4nUMCaCNe3/ioTFL+PH4aYbeV5fWV5fyOpJI2DOzpc65ZH/r6Q5uCRtXli7MpD5NqJpQkEfeT2Hcoh1eRxKJGCoLCSulisTx0aONaZFUkmcnrebvX6zX/N4iQaCykLBTMH8Mox5M5v6GlRgxewtPfLRc83uLBJiG+JSwFBMdxQu3X0vF4vH8/Yv17Dl8gpFdk7miYKzX0UTyJO1ZSNgyM3q1rMZb99VhReph7ho+n50HjnsdSyRPUllI2Lu1VjnGPdKQg8dPc8ewb1m280evI4nkOSoLyRPqJxbn095NKJg/hi6jFvLl6j1eRxLJU1QWkmdUTSjEpD5NuKZsEXqPW8o787Z5HUkkz1BZSJ5SolB+PuzZiJtqlOGvU9by/OQ1nNWltSKXTWUheU5cvmiG3l+XHs2q8N787fT+11JOnNaltSKXQ2UheVJ0lPHcrTV4/rYaTF+3l86jFrL/6CmvY4mELZWF5Gndm1bh7QfqsWHPEe4Y9i1b0o56HUkkLKksJM9rV7MM4x9tzInTZ7lz2HwWbzvodSSRsKOykIhQu2IxJvVpSolCsTwwehGTV1xqHi4RuVC2ysLMxmZnmUgoq1g8nk97N6F2pWL0/3AZw2Zt1vzeItmU3T2LmpmfmFk0UC/344gEVrH4WMb2aEDH2uV4+csN/GHSatLPan5vEX8uOZCgmf0e+ANQwMyOnF8MnAZGBjibSEDkj4nmtXtrU+GKAgyduYUfDp1g6P11KZRf42qKXMwl9yycc//rnCsMvOKcK+L7KeycK+Gc+32QMorkuqgoY9BNV/O/d17HvM37uXfEAvYeOel1LJGQld3DUFPMrCCAmT1gZoPNrHIAc4kERZcGlXinWzI7Dhzj9qHfsn7PEf9vEolA2S2L4cBxM7se+C2wA3g/YKlEgqjVVaWY0Ksx55zjnuELmLdpv9eRREJOdssi3WVcNtIRGOKcGwIUDlwskeCqWa4ok/o0pfwVBeg+ZjETUnZ5HUkkpGS3LH7ynezuCkz1XQ2VL3CxRIKvXLECfNyrMY2rleC3E1cy+KsNurRWxCe7ZdEJOAU87JzbA5QHXglYKhGPFI7Lx7vd63NvcgXe+GYzT01Ywel0XVorkq2y8BXEOKComd0KnHTO6ZyF5En5oqN46a5aPNX2Sj5d9j3d3l3M4RNnvI4l4qns3sF9L7AYuAe4F1hkZncHMpiIl8yMx29M4rVO15Oy4yB3D59P6o+a31siV3YPQz0L1HfOdXPOPQg0AJ4LXCyR0HBHnQq8/3BD9h45yR3D5rMq9bDXkUQ8kd2yiHLO7cv0/EAO3isS1hpXK8EnvZsQGx3FvW8vYMa6vV5HEgm67P4H/0szm2Zm3c2sOzAV+DxwsURCS1Lpwkzq24TqpQrR8/0Uxi7c4XUkkaC6ZFmYWXUza+qcGwS8DdQCrgcWoLGhJMKUKhzHR4814oarS/HcZ6t58fN1nNP83hIh/O1ZvA78BOCc+9Q5N9A59yQZexWvBzqcSKiJj43h7a7JPNi4MiPnbOXxD5dx8ozm95a8z98wm4nOuZUXLnTOpZhZYkASiYS46Cjjzx1qUvGKeP72+Tr2HDnJqAeTKV4w1utoIgHjb88i7hKvFcjNICLhxMzo2aIqw+6vy6rvD3PX8Pls33/M61giAeOvLJaYWc8LF5pZD2BpYCKJhI/fXFeWD3s25NDx09w5fD5Ld/zodSSRgLBLjX1jZqWBSWRMdnS+HJKBWOAO353dISE5OdmlpKR4HUMi1Lb9x3hozGJ2Hz7J651qc/N1Zb2OJJItZrbUOZfsbz1/kx/tdc41Af4MbPf9/Nk51ziUikLEa1VKFuTTPk2pWa4IfT74jtFzt2oQQslTsjWPpHNuJjAzwFlEwlrxgrF80LMRAycs54Wp69h18Dh/uq0m0VHmdTSRy6a7sEVyUVy+aN7qUpdHW1Tlnwt28NjYpRw/ne51LJHLFtCyMLP2ZrbBzDab2TNZvD7QzNaa2Uozm5F5qlYzO2tmy30/kwOZUyQ3RUUZf/jNNfy1Y02+Wb+XziMXsk/ze0uYC1hZ+CZIGgrcDNQAuphZjQtWWwYkO+dqAROBlzO9dsI5V9v30yFQOUUCpWvjREZ2TWbT3qPc8I/ZDJ+1RTfwSdgK5J5FA2Czc26rc+40MJ6MaVn/wzk30zl3ftznhUCFAOYRCbo2NUoztX8zGlUtzktfrqfN4NlMXblbJ78l7ASyLMoDmScyTvUtu5gewBeZnseZWYqZLTSz2wMRUCQYqiYUYnS3+ox7pCGF8sfQ94PvuGfEAlbsOuR1NJFsC2RZZHUJSJZ/TpnZA2Tcv5F5qtZKvmt/7wNeN7NqWbzvUV+hpKSlpeVGZpGAaVq9JFP7N+fvd17H9gPH6Tj0W578aDm7D5/wOpqIX4Esi1SgYqbnFYAfLlzJzNqQMblSB+fcqfPLnXM/+P53KzALqHPhe51zI51zyc655ISEhNxNLxIA0VFG5waVmDWoFX1bV2Pqqt20fnUWg7/awLFTumpKQlcgy2IJkGRmVcwsFugM/OKqJjOrQ8bQ5x0yT65kZleYWX7f45JAU2BtALOKBFWh/DEMuulqvnmqJW1rlOGNbzbT+tVZfJyyS8OeS0gKWFk459KBfsA0YB0wwTm3xsz+Ymbnr256BSgEfHzBJbLXAClmtoKMmwH/7pxTWUieU+GKeN7sUodPejehXLECDJq4ktvemsfCrQe8jibyC5ccGyqcaGwoCXfOOSav+IGXvljPD4dPclPN0vz+5mtILFnQ62iSh+XK2FAiEjxmRsfa5fnm6VYMuukq5m3aT9vXZvO3qWs5fOKM1/EkwqksREJMXL5o+rauzsxBrbizTgVGz9tGq1dm8v6C7aSfPed1PIlQKguREFWqcBwv3V2LqY8355qyRfjT/62h/ZC5zFy/Tzf1SdCpLERCXI1yRRj3SENGPZjM2XOOh95bwoPvLmbDnp+8jiYRRGUhEgbMjLY1SjNtQAueu7UGK3Yd4uYhc3h20ir2Hz3l/xeIXCaVhUgYiY2JokezKswe1JoHGyfy0ZJdtH5lFiNmb+FUugYplMBRWYiEoSsKxvJ8h5pMe7IFDasW5+9fZAxS+PkqDVIogaGyEAlj1XyDFP6rR0MKxsbQZ9x33Pu2BimU3KeyEMkDmiX9PEjhtv0ZgxQO1CCFkotUFiJ5ROZBCvu0qsaU84MUTt+oqV3lsqksRPKYQvlj+G37q5kxsCVtrinNGzM20frVWUxcmqpBCuVXU1mI5FEVi8fz1n11+aR3E8oWLcDTH6+gw9B5LNIghfIrqCxE8rh6la/g095NGNK5NgePnqbTyIX0GruUHQeOeR1NwojKQiQCREX9PEjh0+2uZM6mNNoM1iCFkn0qC5EIEpcvmn43JDHr6Z8HKWz96izGLtAghXJpKguRCFSqSMYghVMeb8ZVpQvz3PlBCjfs8/9miUgqC5EIVrNcUT7o2ZCRXeuRfvYcD43JGKRw414NUii/pLIQiXBmRruaZfjqyZY8d2sNlu/8kfavZwxSeECDFIqPykJEgP8epHD8kl20emUWb2uQQkFlISIX+M8ghQNa0KBKcf73i/W0HTyHLzRIYURTWYhIlqqXKsQ73TMGKYyPjab3uO/o9PZCVqZqkMJIpLIQkUs6P0jhi3dcx9b9R+nw1rcMnKBBCiONykJE/IqOMu5rWImZT7eid6tqTFmZMUjhaxqkMGKoLEQk2wrH5eN3mQYpHOIbpPATDVKY56ksRCTHfh6ksDFlihbgqY9X0HHotyzedtDraBIgKgsR+dXqVS7OJN8ghQeOnuLetxfQ+19L2XnguNfRJJfFeB1ARMLb+UEK29Uow+i5Wxk+ewsz1u2je9NE+t1QnSJx+byOKLlAexYikisKxEbz+I1JzHy6FR1rl2PU3K20emUWYxfu0CCFeYDKQkRyVekicbxyz/X8u18zrixdiOc+W83NQ+YyS4MUhjWVhYgExLXli/Jhz0a83bUeZ86eo/uYJXR7dzGbNEhhWFJZiEjAmBk3+QYp/OMt17Bs54+0HzKX5z5brUEKw4zKQkQCLjYmikeaV2X2oNZ0bVSZDxbvpNWrsxg5R4MUhguVhYgETeZBCusnFufFzzVIYbhQWYhI0FUvVYh3u9dnbI8GFMjnG6Rw5EJWpR72OppchMpCRDzTPCmBqf2bZQxSmHaU296ax8AJy9lz+KTX0eQClld2/ZKTk11KSorXMUTkV/rp5BmGzdrCO/O2YUD3pon0blmNYvGxXkfL08xsqXMu2e96KgsRCSW7Dh7n9a838emyVArlj6FXy2o81DSR+FgNOBEI2S2LgB6GMrP2ZrbBzDab2TNZvD7QzNaa2Uozm2FmlTO91s3MNvl+ugUyp4iEjorF4/nHvdfz5RMtaFilBK9M20BL353gZ3QnuGcCtmdhZtHARqAtkAosAbo459ZmWqc1sMg5d9zMegOtnHOdzKw4kAIkAw5YCtRzzv14sc/TnoVI3rR0x0Fe+mIDi7cfpHKJeAa2vZLbapUjKsq8jpYnhMKeRQNgs3Nuq3PuNDAe6Jh5BefcTOfc+eEpFwIVfI9vAqY75w76CmI60D6AWUUkRNWrXJyPHmvEmIfqEx8bwxPjl3PLm/OYuWGfLrcNokCWRXlgV6bnqb5lF9MD+OJXvldE8jAzo/VVpZj6eDOGdK7NsVPpPDRmCZ1GLmTpDs2hEQyBLIus9hGz/DPAzB4g45DTKzl5r5k9amYpZpaSlpb2q4OKSHg4Pxz61wNb8tfbr2Xb/mPcNXwBj/wzhQ17NOZUIAWyLFKBipmeVwB+uHAlM2sDPAt0cM6dysl7nXMjnXPJzrnkhISEXAsuIqEtNiaKro0qM3tQKwbddBWLth2g/ZA5DJywnF0HNfFSIATyBHcMGSe4bwS+J+ME933OuTWZ1qkDTATaO+c2ZVpenIyT2nV9i74j4wT3Rfc3dYJbJHIdOn6a4bO38N632znnHPc3rEy/G6pTslB+r6OFvJC4z8LMfgO8DkQD7zrn/mZmfwFSnHOTzexr4Dpgt+8tO51zHXzvfRj4g2/535xzYy71WSoLEdlz+CRDZmxiQsou8vsGL+zZvAqFNVvfRYVEWQSTykJEztuadpR/TN/I1JW7uSI+H31bV+eBRpWJyxftdbSQo7IQkYi3KvUwL09bz9xN+ylXNI4Bba/kzjrliYnWsHjnhcJ9FiIinrquQlHG9mjIB480JKFIHL+duJL2Q+by5eo9ukcjh1QWIpLnNaleks/6NGHEA/VwztHrX0u5fdh85m/Z73W0sKGyEJGIYGa0v7YM0wa04OW7a5F25CT3jVpE13cWsfp7zaPhj85ZiEhEOnnmLP9auIOhMzfz4/Ez3FqrLE+1u4oqJQt6HS2odIJbRCQbjpw8w+g5Wxk9bxun0s/RqX5FnrgxidJF4ryOFhQqCxGRHEj76RRDZ25m3KIdRJnxUNMq9G5ZjaLxefseDZWFiMivsOvgcV6bvpFJy7+PiMmXVBYiIpdh/Z4jvDptA1+v20dC4fz0vzGJzvUrki+P3aOh+yxERC7D1WWKMLpbfSb2akxiiXie+2w1bQbPZvKKHzh3Lm/8kZ0TKgsRkUtITizOhMcaM6Z7fQrki6b/h8u49c15zIqwyZdUFiIifpgZra8uxef9mzOkc22Onkqn+5gldB65kKU7Ljrbc56ishARyaZfTL7UsSZb0o5x1/D59Hw/hY178/bkSzrBLSLyKx0/nc6Yb7czYtYWjp5O5846FRjQJomKxeO9jpZtuhpKRCRIfjx2mhGzt/De/O04B/c3qkTf1uEx+ZLKQkQkyHYfPsEbMzYxISWVON/kS4+E+ORLKgsREY9sSTvK4K82MnXVbooXjKVv6+rc37BSSE6+pLIQEfHYytRDvDJtA3M37ad8sQIMaJPEnXUrEB1lXkf7D92UJyLisVoVijG2R0PGPdKQkoViGTRxJe1fn8O0NeE3+ZLKQkQkwJpWL8lnfZsy4oG6nHWOx8Yu5Y5h81mw5YDX0bJNZSEiEgQZky+V5asBLXj5rlrsPXKSLqMW8uC7i8Ni8iWdsxAR8cDJM2cZu2AHQ2dt5pCHky/pBLeISBg4cvIMo+ZsZfTcbZw+G/zJl1QWIiJhJO2nU7z1zSY+WLyT6Cije5PgTL6kshARCUM7Dxznta838tny7ymcP4ZerarxUJMqFIgNzD0aKgsRkTC2bnfG5Esz1u+jlG/ypU4BmHxJ91mIiISxa8oW4Z3u9fm4V2MqFY/nj5+tpq2Hky+pLEREQlj9xOJ83Ksx73ZPJs7DyZdUFiIiIc7MuOHq0nzevzmvd6rNT6fOBH3yJZWFiEiYiIoybq9TnhkDW/GXTJMv9R33XcD3MmIC+ttFRCTXxcZE8WDjRO6qW4Ex327jxJmzmAV2cEKVhYhImCqYP4Z+NyQF5bN0GEpERPxSWYiIiF8qCxER8UtlISIifqksRETEL5WFiIj4pbIQERG/VBYiIuJXnhmi3MzSgB2X8StKAvvbbz14AAAF40lEQVRzKU5uUq6cUa6cUa6cyYu5KjvnEvytlGfK4nKZWUp2xnQPNuXKGeXKGeXKmUjOpcNQIiLil8pCRET8Uln8bKTXAS5CuXJGuXJGuXImYnPpnIWIiPilPQsREfErosrCzNqb2QYz22xmz2Txen4z+8j3+iIzSwyRXN3NLM3Mlvt+HglSrnfNbJ+Zrb7I62Zmb/hyrzSzuiGSq5WZHc60vf4UpFwVzWymma0zszVm9kQW6wR9m2UzV9C3mZnFmdliM1vhy/XnLNYJ+ncym7k8+U76PjvazJaZ2ZQsXgvc9nLORcQPEA1sAaoCscAKoMYF6/QBRvgedwY+CpFc3YG3PNhmLYC6wOqLvP4b4AvAgEbAohDJ1QqY4sH2KgvU9T0uDGzM4v/LoG+zbOYK+jbzbYNCvsf5gEVAowvW8eI7mZ1cnnwnfZ89EPggq/+/Arm9ImnPogGw2Tm31Tl3GhgPdLxgnY7AP32PJwI3WqDnKsxeLk845+YABy+xSkfgfZdhIVDMzMqGQC5POOd2O+e+8z3+CVgHlL9gtaBvs2zmCjrfNjjqe5rP93PhSdSgfyezmcsTZlYBuAUYfZFVAra9IqksygO7Mj1P5b+/MP9ZxzmXDhwGSoRALoC7fIctJppZxQBnyq7sZvdCY99hhC/MrGawP9y3+1+HjL9KM/N0m10iF3iwzXyHVJYD+4DpzrmLbq8gfiezkwu8+U6+DvwWOHeR1wO2vSKpLLJq1wv/WsjOOrktO5/5byDROVcL+Jqf/3LwmhfbKzu+I2MIg+uBN4HPgvnhZlYI+AQY4Jw7cuHLWbwlKNvMTy5Ptplz7qxzrjZQAWhgZtdesIon2ysbuYL+nTSzW4F9zrmll1oti2W5sr0iqSxSgcztXwH44WLrmFkMUJTAH+7wm8s5d8A5d8r3dBRQL8CZsis72zTonHNHzh9GcM59DuQzs5LB+Gwzy0fGf5DHOec+zWIVT7aZv1xebjPfZx4CZgHtL3jJi++k31wefSebAh3MbDsZh6tvMLN/XbBOwLZXJJXFEiDJzKqYWSwZJ38mX7DOZKCb7/HdwDfOd6bIy1wXHNPuQMYx51AwGXjQd4VPI+Cwc26316HMrMz547Rm1oCMf88PBOFzDXgHWOecG3yR1YK+zbKTy4ttZmYJZlbM97gA0AZYf8FqQf9OZieXF99J59zvnXMVnHOJZPx34hvn3AMXrBaw7RWTG78kHDjn0s2sHzCNjCuQ3nXOrTGzvwApzrnJZHyhxprZZjLauHOI5OpvZh2AdF+u7oHOBWBmH5JxlUxJM0sF/oeMk30450YAn5Nxdc9m4DjwUIjkuhvobWbpwAmgcxBKHzL+8usKrPId7wb4A1ApUzYvtll2cnmxzcoC/zSzaDLKaYJzborX38ls5vLkO5mVYG0v3cEtIiJ+RdJhKBER+ZVUFiIi4pfKQkRE/FJZiIiIXyoLERHxS2UhkgNmdjbTSKPLLYtRgi/jdyfaRUbSFfFaxNxnIZJLTviGgRCJKNqzEMkFZrbdzF7yzYOw2Myq+5ZXNrMZvgHnZphZJd/y0mY2yTdw3woza+L7VdFmNsoy5lH4yncHsYjnVBYiOVPggsNQnTK9dsQ51wB4i4zRQfE9ft834Nw44A3f8jeA2b6B++oCa3zLk4ChzrmawCHgrgD/84hki+7gFskBMzvqnCuUxfLtwA3Oua2+Qfv2OOdKmNl+oKxz7oxv+W7nXEkzSwMqZBqM7vzw4dOdc0m+578D8jnnXgj8P5nIpWnPQiT3uIs8vtg6WTmV6fFZdF5RQoTKQiT3dMr0vwt8j+fz82Bu9wPzfI9nAL3hPxPtFAlWSJFfQ3+1iORMgUwjtwJ86Zw7f/lsfjNbRMYfYV18y/oD75rZICCNn0eZfQIYaWY9yNiD6A14Pry7yMXonIVILvCds0h2zu33OotIIOgwlIiI+KU9CxER8Ut7FiIi4pfKQkRE/FJZiIiIXyoLERHxS2UhIiJ+qSxERMSv/we7vQ+aZEsu8gAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xt8VPWd//HX58wlCQSQS/ACIlCxilyiDYjgj0qtitt6+VmtF7bq1l+1tRZXf1Wxbm2XrbVuL1Yfdddq6/a3Xau2tbqsYr3grbVWCUptuahAQVNQbpKES5K5fH5/zEkYkkkmQCYTkvfz8ZhHZr5zzpnvGYZ5z/f7Ped7zN0RERHpSFDsCoiISM+nsBARkbwUFiIikpfCQkRE8lJYiIhIXgoLERHJS2EhIiJ5KSxERCQvhYWIiOQVLXYFusqwYcN89OjRxa6GiMgBZcmSJZvdvSLfcr0mLEaPHk11dXWxqyEickAxs3WdWU7dUCIikpfCQkRE8lJYiIhIXr1mzEJEdkskEtTU1NDQ0FDsqkgPUVpaysiRI4nFYvu0vsJCpBeqqalhwIABjB49GjMrdnWkyNydLVu2UFNTw5gxY/ZpG+qGEumFGhoaGDp0qIJCADAzhg4dul8tTYWFSC+loJBs+/t5UFg0bofnboUanaMhItKePh8WH9bVwUv/yht/XFTsqoj0KuXl5cWuQodeeOEF/vCHP+z1etXV1cydOzfvctOnT9+Xau23b3/72wXZbp8PCywCQFMiUeSKiEh36igskslku+tVVVVx11135d3+vgRRV1BYFEgQCQ8IS6eLWxGRPmDdunWccsopTJo0iVNOOYV3330XgF/96ldMmDCByZMnM3PmTACWLVvG1KlTqaysZNKkSbzzzjtttvfggw8yceJEJkyYwI033thSXl5ezs0338zkyZOZNm0aH3zwwR7rrV27lnvuuYc77riDyspKfve733HZZZdx3XXXMWvWLG688UZee+01pk+fznHHHcf06dN56623gEzIfPrTnwbgm9/8Jp///Oc5+eSTGTt27B4h0tyyeuGFFzj55JM577zzOProo5kzZw7uDsDChQs5+uijOemkk5g7d27LdrO19z7813/9V0v5lVdeSSqVYt68eezatYvKykrmzJmzb/9I7ejzh84GkTAv06niVkSkQP75f5axfH1dl25z/GED+caZx+71eldffTWXXHIJl156Kffffz9z587lscceY/78+Tz11FOMGDGCbdu2AXDPPfdwzTXXMGfOHJqamkil9vw/un79em688UaWLFnC4MGDOe2003jsscc455xz2LFjB9OmTePWW2/lhhtu4L777uOf/umfWtYdPXo0X/ziFykvL+erX/0qAD/96U95++23efbZZ4lEItTV1fHSSy8RjUZ59tln+drXvsYjjzzSZp9WrlzJ888/T319PR/96Ef50pe+1OZchjfeeINly5Zx2GGHMWPGDF5++WWqqqq48soreemllxgzZgwXXXRRzvcs1/uwYsUKHn74YV5++WVisRhXXXUVDzzwAN/5znf40Y9+xNKlS/f63yafPt+yiIQtC3eFhUihvfLKK1x88cUAfO5zn+P3v/89ADNmzOCyyy7jvvvuawmFE088kW9/+9vcfvvtrFu3jrKysj22tXjxYk4++WQqKiqIRqPMmTOHl156CYB4PN7yK/1jH/sYa9eu7VT9zj//fCKRTNd0bW0t559/PhMmTODaa69l2bJlOdf51Kc+RUlJCcOGDWP48OFtWjEAU6dOZeTIkQRBQGVlJWvXrmXlypWMHTu25byH9sIi1/uwaNEilixZwpQpU6isrGTRokWsWbOmU/u4r/p8yyISbe6GUlhI77QvLYDu0nw45z333MOrr77KE088QWVlJUuXLuXiiy/mhBNO4IknnuD000/nJz/5CZ/4xCda1m3uysklFou1bDsSiXQ4BpGtf//+Lfe//vWvM2vWLB599FHWrl3LySefnHOdkpKSlvvtvVauZTqqf7Zc74O7c+mll3Lbbbd1ahtdQS2LQGEh0l2mT5/OQw89BMADDzzASSedBMDq1as54YQTmD9/PsOGDeO9995jzZo1jB07lrlz53LWWWfx5ptv7rGtE044gRdffJHNmzeTSqV48MEH+fjHP97pugwYMID6+vp2n6+trWXEiBEA/OxnP9vLPc3v6KOPZs2aNS2tnocffjjncrneh1NOOYVf//rXbNy4EYCtW7eybl1mpvFYLEaiAAfsFDQszGy2mb1lZqvMbF6O579oZn82s6Vm9nszG5/13E3hem+Z2emFqmNzk5N05355iEjn7Ny5k5EjR7bcfvCDH3DXXXfxH//xH0yaNImf//zn3HnnnQBcf/31LQPVM2fOZPLkyTz88MNMmDCByspKVq5cySWXXLLH9g899FBuu+02Zs2axeTJkzn++OM5++yzO12/M888k0cffbRlgLu1G264gZtuuokZM2a0GS/pCmVlZfzbv/0bs2fP5qSTTuLggw9m0KBBbZbL9T6MHz+eb33rW5x22mlMmjSJU089lQ0bNgBwxRVXMGnSpC4f4LbONoX2esNmEeBt4FSgBlgMXOTuy7OWGejudeH9s4Cr3H12GBoPAlOBw4BngaO8g4GFqqoq39eLHyW/MZjqkZcw7Qt37tP6Ij3NihUrOOaYY4pdDclj+/btlJeX4+58+ctfZty4cVx77bUFe71cnwszW+LuVfnWLWTLYiqwyt3XuHsT8BCwR+w3B0WoP9CcXGcDD7l7o7v/FVgVbq8g0gTgOnRWRLrXfffdR2VlJcceeyy1tbVceeWVxa5Suwo5wD0CeC/rcQ1wQuuFzOzLwHVAHGgevRoB/LHVuiMKU01IEYCOhhKRbnbttdcWtCXRlQrZssg1a1WbPi93v9vdPwLcCDQfCN2pdc3sCjOrNrPqTZs27XNF0wSYBrhFRNpVyLCoAQ7PejwSWN/B8g8B5+zNuu5+r7tXuXtVRUXFPlc0pW4oEZEOFTIsFgPjzGyMmcWBC4EF2QuY2bish58Cms/nXwBcaGYlZjYGGAe8VqiKpi3A1A0lItKugo1ZuHvSzK4GngIiwP3uvszM5gPV7r4AuNrMPgkkgA+BS8N1l5nZL4HlQBL4ckdHQu2vNIHOsxAR6UBBz7Nw94XufpS7f8Tdbw3LbgmDAne/xt2PdfdKd5/l7suy1r01XO+j7v5kIeuZ1gC3SJfrrVOUQ2Yiwl/84hddUo8f/vCH7Ny5s0u2VUh9/gxuyIxZmMYsRPoUhcXeUVgQHg2lloVIwfXkKco3bdrEZz7zGaZMmcKUKVN4+eWXAXjxxReprKyksrKS4447jvr6eubNm8fvfvc7KisrueOOO/bY9oYNG5g5cyaVlZVMmDCh5ezwp59+mhNPPJHjjz+e888/n+3bt3PXXXexfv16Zs2axaxZs7rujS6Agp3B3d325wzuv/3zUWwYOImqa3/dxbUSKY49ztR9ch68/+eufYFDJsIZ3+lwkfLycrZv375H2Zlnnsl5553XMkX5ggULeOyxx5g4cSK//e1vW6YoP+igg/jKV77CtGnT9piaO3vm2fXr1zNt2rQ9piifO3cu55xzDmbGggULOPPMM7nhhhsYOHDgHlOUQ+ZaFNlTlF988cVcddVVnHTSSbz77rucfvrprFixgjPPPJN58+YxY8YMtm/fTmlpKb///e/53ve+x+OPP95mv7///e/T0NDAzTffTCqVYufOnTQ2NnLuuefy5JNP0r9/f26//XYaGxu55ZZbGD16NNXV1QwbNmxf/zU6bX/O4O7zs84CuLqhRLrFK6+8wm9+8xsgM0X5DTfcAOyeovyzn/0s5557LpCZmvvWW2+lpqaGc889l3Hjxu2xrewpyoGWKcrPOeecNlOUP/PMM3nr9uyzz7J8ectsRNTV1VFfX8+MGTO47rrrmDNnDueeey4jR47scDtTpkzh85//PIlEgnPOOYfKykpefPFFli9fzowZMwBoamrixBNP7Mxb1mMoLNDRUNLL5WkBFFNPmqI8nU7zyiuvtLluxrx58/jUpz7FwoULmTZtGs8++2yH25k5cyYvvfQSTzzxBJ/73Oe4/vrrGTx4MKeeeioPPvhg3nr0VBqzIDzPArUsRAqtJ09Rftppp/GjH/2o5XHz1eZWr17NxIkTufHGG6mqqmLlypUdTm++bt06hg8fzhe+8AUuv/xyXn/9daZNm8bLL7/MqlWrgMyMvG+//XbOevRUCgsgTUQD3CJd7ECbovyuu+6iurqaSZMmMX78eO655x4gc7RS8+B7WVkZZ5xxBpMmTSIajTJ58uQ2A9wvvPBCy2D4I488wjXXXENFRQU/+9nPuOiii5g0aRLTpk1j5cqVQGZK8TPOOEMD3N1lfwa4V//LcdSXHEzlDb/t4lqJFIemKJdceuoU5QeMFBECtSxERNqlsABcc0OJiHRIYUHzRIIa4Jbepbd0MUvX2N/Pg8ICcA1wSy9TWlrKli1bFBgCZIJiy5YtlJaW7vM2dJ4FmZZFRIfOSi8ycuRIampq2J+LgknvUlpamveEwo4oLGg+gzv/STsiB4pYLMaYMWOKXQ3pRdQNBbjpaCgRkY4oLAiPhlI3lIhIuxQWNLcsFBYiIu1RWNDcslA3lIhIexQWqGUhIpKPwgKNWYiI5KOwQC0LEZF8FBYAFhBozEJEpF0KC8KWhbqhRETaVdCwMLPZZvaWma0ys3k5nr/OzJab2ZtmtsjMjsh6LmVmS8PbgkLWU91QIiIdK9h0H2YWAe4GTgVqgMVmtsDdl2ct9gZQ5e47zexLwL8CF4TP7XL3ykLVL5tbhIi6oURE2lXIlsVUYJW7r3H3JuAhYI9rHrr78+6+M3z4R2DfZ7naD+qGEhHpWCHDYgTwXtbjmrCsPZcDT2Y9LjWzajP7o5mdk2sFM7siXKZ6v2bXDAKFhYhIBwo566zlKMs5ub6Z/T1QBXw8q3iUu683s7HAc2b2Z3dfvcfG3O8F7oXMNbj3vaZqWYiIdKSQLYsa4PCsxyOB9a0XMrNPAjcDZ7l7Y3O5u68P/64BXgCOK1RF3QIiGuAWEWlXIcNiMTDOzMaYWRy4ENjjqCYzOw74MZmg2JhVPtjMSsL7w4AZQPbAeNdSy0JEpEMF64Zy96SZXQ08BUSA+919mZnNB6rdfQHwXaAc+JWZAbzr7mcBxwA/NrM0mUD7TqujqLpWENGV8kREOlDQK+W5+0JgYauyW7Luf7Kd9f4ATCxk3fZ4PbUsREQ6pDO4AbMIUVNYiIi0R2EBEEQA8LROzBMRyUVhARBk3oZUMlnkioiI9EwKC2hpWaTSCgsRkVwUFgCWCYt0Ut1QIiK5KCxALQsRkTwUFtDSskil1LIQEclFYQFY89FQGuAWEclJYQEQZM5NTKUTRa6IiEjPpLCA3edZaIBbRCQnhQW7u6E0wC0ikpvCgsx0HwBpDXCLiOSksIDd3VAKCxGRnBQWgEUyb0Na3VAiIjkpLAAsPBpKLQsRkZwUFkAQaZ51Vi0LEZFcFBZkHQ2lQ2dFRHJSWJB1BrdaFiIiOSks2B0WaV38SEQkJ4UFYOF0H2pZiIjkprAgqxtKR0OJiOSksGD30VA6g1tEJLeChoWZzTazt8xslZnNy/H8dWa23MzeNLNFZnZE1nOXmtk74e3SgtYz7IbSSXkiIrkVLCwsM+HS3cAZwHjgIjMb32qxN4Aqd58E/Br413DdIcA3gBOAqcA3zGxwweoadkORThfqJUREDmiFbFlMBVa5+xp3bwIeAs7OXsDdn3f3neHDPwIjw/unA8+4+1Z3/xB4BphdqIq2dEOpZSEiklMhw2IE8F7W45qwrD2XA0/u47r7pbkbipTCQkQkl2gBt205yjzngmZ/D1QBH9+bdc3sCuAKgFGjRu1bLcme7kMD3CIiuRSyZVEDHJ71eCSwvvVCZvZJ4GbgLHdv3Jt13f1ed69y96qKiop9rujuAW6FhYhILoUMi8XAODMbY2Zx4EJgQfYCZnYc8GMyQbEx66mngNPMbHA4sH1aWFYQzS0LFBYiIjkVrBvK3ZNmdjWZL/kIcL+7LzOz+UC1uy8AvguUA78yM4B33f0sd99qZv9CJnAA5rv71kLVNYjoDG4RkY4UcswCd18ILGxVdkvW/U92sO79wP2Fq91uuycSVMtCRCQXncENRFpaFgoLEZFcFBZkd0MpLEREclFYoAFuEZF8FBZAEKhlISLSEYUFYJHM2+CusBARyUVhAUSap/tQy0JEJCeFBRCJqhtKRKQjCgvAwqOhTGEhIpKTwgKI6KQ8EZEOKSzY3Q2FBrhFRHJSWABBJJa5o5aFiEhOnQoLM/t5Z8oOVM3TfSgsRERy62zL4tjsB+H1tT/W9dUpjiAw0m7qhhIRaUeHYWFmN5lZPTDJzOrCWz2wEfjvbqlhNzAzUgTg6WJXRUSkR+owLNz9NncfAHzX3QeGtwHuPtTdb+qmOnaLNIG6oURE2tHZbqjHzaw/ZK6XbWY/MLMjClivbpdpWSgsRERy6WxY/Duw08wmAzcA64D/LFitikAtCxGR9nU2LJLu7sDZwJ3uficwoHDV6n5pCzC1LEREcursZVXrzewm4HPA/wqPhooVrlrdTwPcIiLt62zL4gKgEfi8u78PjAC+W7BaFUEatSxERNrTqbAIA+IBYJCZfRpocPfeN2ahsBARyamzZ3B/FngNOB/4LPCqmZ1XyIp1tzQBllY3lIhILp3throZmOLul7r7JcBU4Ov5VjKz2Wb2lpmtMrN5OZ6faWavm1mydfiYWcrMloa3BZ2s5z5Ty0JEpH2dHeAO3H1j1uMt5D/7OwLcDZwK1ACLzWyBuy/PWuxd4DLgqzk2scvdKztZv/2WORpKLQsRkVw6Gxa/NbOngAfDxxcAC/OsMxVY5e5rAMzsITKH3raEhbuvDZ8r+rd0mogGuEVE2tFhWJjZkcDB7n69mZ0LnAQY8AqZAe+OjADey3pcA5ywF3UrNbNqIAl8x90f24t195rOsxARaV++lsUPga8BuPtvgN8AmFlV+NyZHaxrOcp8L+o2yt3Xm9lY4Dkz+7O7r97jBcyuAK4AGDVq1F5suq0UUYWFiEg78g1wj3b3N1sXuns1MDrPujXA4VmPRwLrO1sxd18f/l0DvAAcl2OZe929yt2rKioqOrvpnNKmbigRkfbkC4vSDp4ry7PuYmCcmY0xszhwIdCpo5rMbLCZlYT3hwEzyBrrKIS0RQjSyUK+hIjIAStfWCw2sy+0LjSzy4ElHa3o7kngauApYAXwS3dfZmbzzeyscDtTzKyGzPkbPzazZeHqxwDVZvYn4HkyYxaFDQsNcIuItCvfmMU/Ao+a2Rx2h0MVEAf+d76Nu/tCWh015e63ZN1fTKZ7qvV6fwAm5tt+V0pbhEBhISKSU4dh4e4fANPNbBYwISx+wt2fK3jNulnaIgQoLEREcunUeRbu/jyZ7qBeyy1CJN1Y7GqIiPRInZ3uo9dTN5SISPsUFqG0RRUWIiLtUFiEXGEhItIuhUXIAw1wi4i0R2ERSluUiFoWIiI5KSyaqWUhItIuhUXILaKWhYhIOxQWIQ+iRNSyEBHJSWER8iBChKJfg0lEpEdSWDQztSxERNqjsAh5oKOhRETao7BoFkTUshARaYfCoplFNWYhItIOhUWzIErMUuB7c5lwEZG+QWHRLMjM1p5KqStKRKQ1hUWzSASAZLKpyBUREel5FBbNmlsWyUSRKyIi0vMoLEIWhkVSYSEi0obColkYFumEwkJEpDWFRcgiYcsipbAQEWlNYdEsEgMgnUwWuSIiIj1PQcPCzGab2VtmtsrM5uV4fqaZvW5mSTM7r9Vzl5rZO+Ht0kLWE3aPWaTUshARaaNgYWFmEeBu4AxgPHCRmY1vtdi7wGXAL1qtOwT4BnACMBX4hpkNLlRdYXdYpDXALSLSRiFbFlOBVe6+xt2bgIeAs7MXcPe17v4mtJln43TgGXff6u4fAs8AswtY15Yxi1RK3VAiIq0VMixGAO9lPa4Jy7psXTO7wsyqzax606ZN+1xR2B0WalmIiLRVyLCwHGWdnXipU+u6+73uXuXuVRUVFXtVuTYvGGQGuFMa4BYRaaOQYVEDHJ71eCSwvhvW3SdBc8tCA9wiIm0UMiwWA+PMbIyZxYELgQWdXPcp4DQzGxwObJ8WlhXM7jELhYWISGsFCwt3TwJXk/mSXwH80t2Xmdl8MzsLwMymmFkNcD7wYzNbFq67FfgXMoGzGJgflhVMEJ5n4RrgFhFpI1rIjbv7QmBhq7Jbsu4vJtPFlGvd+4H7C1m/bC3dUBqzEBFpQ2dwh4Jo85iFwkJEpDWFRajl0NmUrmchItKawiIUjZUCkE40FrkmIiI9j8IiFC0pAyCd2FXkmoiI9DwKi1As3g9Qy0JEJBeFRShWkumG8kRDkWsiItLzKCxC8eZuqKRaFiIirSksQvGyTFigloWISBsKi1A8HoaFWhYiIm0oLELRWIyERyClsBARaU1hkSVBVC0LEZEcFBZZGi2OqWUhItKGwiJLghiBwkJEpA2FRZaExTDNDSUi0obCIkvC4gRphYWISGsKiyxJixNRN5SISBsKiyxJixNJKyxERFpTWGRJBnEC1zW4RURaU1hkSQVxohqzEBFpQ2GRJR3EibrCQkSkNYVFllSkhJhaFiIibSgssqSDODE0ZiEi0lpBw8LMZpvZW2a2yszm5Xi+xMweDp9/1cxGh+WjzWyXmS0Nb/cUsp7N0tFSSlxHQ4mItBYt1IbNLALcDZwK1ACLzWyBuy/PWuxy4EN3P9LMLgRuBy4In1vt7pWFql8u6dIhDPR60skkQbRgb42IyAGnkC2LqcAqd1/j7k3AQ8DZrZY5G/h/4f1fA6eYmRWwTh2y8uFEzKnftrFYVRAR6ZEKGRYjgPeyHteEZTmXcfckUAsMDZ8bY2ZvmNmLZva/CljPFtGBFQDUbt7QHS8nInLAKGRfS64WgndymQ3AKHffYmYfAx4zs2PdvW6Plc2uAK4AGDVq1H5XuHTQIQDs2KqwEBHJVsiWRQ1weNbjkcD69pYxsygwCNjq7o3uvgXA3ZcAq4GjWr+Au9/r7lXuXlVRUbHfFe4/JBMWDds+2O9tiYj0JoUMi8XAODMbY2Zx4EJgQatlFgCXhvfPA55zdzezinCAHDMbC4wD1hSwrgAMHHoYAIk6hYWISLaCdUO5e9LMrgaeAiLA/e6+zMzmA9XuvgD4KfBzM1sFbCUTKAAzgflmlgRSwBfdfWuh6trsoKHDSbmR3r6p0C8lInJAKejxoe6+EFjYquyWrPsNwPk51nsEeKSQdcslGo2y2QYR2bW5u19aRKRH0xncrdQFg4g1bCl2NUREehSFRSs7ooMpayp4j5eIyAFFYdFKQ8kQypMfFrsaIiI9isKilWTpUAZ5bbGrISLSoygsWvF+FZSzi6ZdO4pdFRGRHkNh0UowIHNy37bNfytyTUREeg6FRSslQzInnX+4YW1R6yEi0pMoLFoZPCIzq8iO91cVuSYiIj2HwqKVQ0aNI+1GYnPBZxcRETlgKCxaKS3rxwc2lGjtumJXRUSkx1BY5LAlPoKBO9YWuxoiIj2GwiKH2qGVjEmsomGHzrcQEQGFRU79PnoyUUuzpvrpYldFRKRHUFjkMK7qVLb4IOIvfx9PJYpdHRGRolNY5FBePoC/TLyRI5tWsOzH/4Anm4pdJRGRolJYtGPmZ67i6YrLmLDxf1j9vU/wwbKXil0lEZGiUVi0w8w49aofsuiYbzFs1xoO/tWZrPjBp3n/z8+De7GrJyLSrRQWHTAzTrngKzRc/SbPHPJ/OLT2dQ555BzW3jaFFY/fSWpXXbGrKCLSLcx7ya/kqqoqr66uLuhrfLBlC28+fg+j//oQ43iXHZSy+pC/49BPfJGKo04o6GuLiBSCmS1x96q8yyks9l4imaL65adJvHo/U3a8QJk18df4UewcfwGHTz2bgYeN65Z6iIjsL4VFN6nZsIGVT93HmLW/5CO8B8D6yGG8P/QEIodOZPDYj3HoUccTKxvY7XUTEclHYdHNEskUK5a9wealTzLwby9yVOMyBtpOANJu7LRSwFhfMobtAz6C9xsGkTgWjUO0hCBWykFjjqO0fDDxsnJKyvoTBBFKBw7FIrGi7ZeI9G49IizMbDZwJxABfuLu32n1fAnwn8DHgC3ABe6+NnzuJuByIAXMdfenOnqtYodFa02JFO+ufYvN7yyhaf2bBA21kGpkaN1yDkmuZwA7iVo673YyQVNGI3HcjPrgIHYEAwjM8UgJHimBaCnJIE4QK4VoKRbLhE80XkYQLyOIlRKJl2LRUixWShDb/TcSPp+9fDReSjReSiQSwcy64d0SkWLpbFhEC1iBCHA3cCpQAyw2swXuvjxrscuBD939SDO7ELgduMDMxgMXAscChwHPmtlR7p4qVH27WjwW4chx4zly3Pg2z6XTTlMqzc5EgkRTI8mmRrbXbmHT6tdJN+0i3bgTT+zE0ynYsZmgqZ4g1Yilm+jX8AFxkriDNdUTS28hmm4ibk3EPUEJTZSQoNT2/8zzJo/QRJxGYjQRo8nipIiQtgAnCP9GWv66BeEtAln33QKwCB5kyrEIBBHcIgQ46UgJWEAQBCSjZVg6RUntGhqGTcRwYlvfpmnwOCyIQLIRSgfhu7Zi5RWkmxoIPIWVDiIoKcOiJSS3vgslAzAMTzYSKR+aeb1UAk8lAYgNGJY5mi0Swxu2Q9lBBP0Ooum91wksYOC46TTUbSKIlZJ0I2JO1CAo6UeyYQeRWAkAO977CyXDxzBg+BhSac/sRyTGzq3vQTpN47YNxPofxKDDxrH1r29y8DHTSCUSbH7nNYaMqaR+cw0Hj53Mzu3b2PiX5zls8mnsqN3IoOFHYAYfrF5KxZiJbN/6PuWDh1O36W8MHnEkqaYG0u7Ub67hiGOnE41GWb30JYaPPobaTes5ZOwEtn/4AbUbaxg+ejx1W95n2GGj2VG3jVSykZ21Wzj0yEp21W9h26b1jDiykl3bt7J1wzoOPXIy27e+z0EHj2brhtWUDhhCQ/1WBgw9jLpNNQw+ZDR1m//l0tIbAAAKOUlEQVTGoIOPILGrnm0b32PY4UeTTuxiR91Wyoccyo4PP2DAsJFYEJBq2E5TYwORWAnxfgNINmQuWRwt7b/nB849vKV333A8ncLTadLpFOm0E+s3ADyNRUvyf4jdIfsHT7IJovH9/r/RFxWsZWFmJwLfdPfTw8c3Abj7bVnLPBUu84qZRYH3gQpgXvay2cu193o9rWVRDOm005hMsyuRYkdDgp0NO2nYtZNEwy4STbsg0UA60QjJBjzZAImGzJdvKvybbCRINWCpRizZSJAKb+lGglQTkVQjeBLzNKRTmKcy/2k9hWX9xdMYzWXpzJc5aQJPE4TlAZkbQKk3YjjglNKUCRCMEst8sTd5hLhlfickPELMUiQ96FTLTAon+98g+34qDNdGYqSI0I+GlvIGKyHuCQwnaVEipDCcwJ3AOvddlHYjjZGwGFGSpIiQJErz2gZESRLxFDFLsZNSAAJSlJKglvJwOQ9vQHi/uSQdnlUQIZX1bBB+SttvbXv4SR7ADnZRRpIIEVIEpEkRwfCWbTtGnAQOJImStChkfuLgYf06ep1s7/f7KBOv/22n3r/Wit6yAEZAOOKbUQO0Pr60ZRl3T5pZLTA0LP9jq3VHFK6qvUMQGGXxCGXxCEP6x4H+edfpadJpJ5l2YgZb62qJl5bRLx7nw/o6Ssr6EQ0ibPpwM4MGD2XTti306z8At4DtdVtp2rWTpoYdDDnkCBKNu0gmEpT2K2f7tk14OkUkGieIxkglm9ixbTP9Bg4hmWigtN8gEju20bRjK1Y6kGSiiYbaLZQNPphU0y4CnGT4/zaxfSsl5UNINjWAGRaJ0bjtfcAJggA8jacSmRZYMkGkpJxUQy14mqB0EKkdmzCLYP2GkKrbQBAtIZ1owKIxPFaO139AUDoATyfwVIpIST9SO2uxIEI62YBF4pBOgkWIlA0gUlJO4/srAcPipXiiEYv3wxvqsNJBmRZeYz3E+0GiASs7KPM4lYBUE8T6YQbeUA/x/plf8431UDYYmrbjsXIssQMicUjsgHg5NNZBfADWWIvH+kMQhYZaPFaGBVGsqR6P9cca64A0HisP/3FTWGoXBDGcANKJlhZmpsXZ+maZr87weQvCL8jtH4BFId0EQTzzAyaV2KMBkSKKRWI0eUBJKtOScQsy44RN28PWhmWSpfl0s7Cs+UcQOG7RTHnY4jF2/0CxrC/szCKOe+ZrPB0pwZK7Ml/8QRQnwNJJdodSZgseKQEyP76CdPa0QrsjrLVcIZIaNCrv/639VciwyBW/rfeyvWU6sy5mdgVwBcCoUYV/s6TwgsCIh18KQwYPbikffNBBLfcrhh+c+VtxcEtZedlhrba0e/mhQ4bkeCUd3iyyNwp5BncNcHjW45HA+vaWCbuhBgFbO7ku7n6vu1e5e1VFRUUXVl1ERLIVMiwWA+PMbIyZxckMWC9otcwC4NLw/nnAc54ZRFkAXGhmJWY2hszPwNcKWFcREelAwbqhwjGIq4GnyBw6e7+7LzOz+UC1uy8Afgr83MxWkWlRXBiuu8zMfgksB5LAlw+kI6FERHobnZQnItKHdfZoKM06KyIieSksREQkL4WFiIjkpbAQEZG8es0At5ltAtbtxyaGAZu7qDoHEu1339JX9xv67r7n2+8j3D3viWq9Jiz2l5lVd+aIgN5G+9239NX9hr6771213+qGEhGRvBQWIiKSl8Jit3uLXYEi0X73LX11v6Hv7nuX7LfGLEREJC+1LEREJK8+HxZmNtvM3jKzVWY2r9j16Wpmdr+ZbTSzv2SVDTGzZ8zsnfDv4LDczOyu8L1408yOL17N952ZHW5mz5vZCjNbZmbXhOW9er8BzKzUzF4zsz+F+/7PYfkYM3s13PeHw5mgCWd2fjjc91fNbHQx67+/zCxiZm+Y2ePh416/32a21sz+bGZLzaw6LOvyz3qfDous64SfAYwHLgqv/92b/AyY3apsHrDI3ccBi8LHkHkfxoW3K4B/76Y6drUk8H/d/RhgGvDl8N+1t+83QCPwCXefDFQCs81sGpnr298R7vuHwOXh8pcDH7r7kcAd4XIHsmuAFVmP+8p+z3L3yqxDZLv+s958KcC+eANOBJ7KenwTcFOx61WA/RwN/CXr8VvAoeH9Q4G3wvs/Bi7KtdyBfAP+Gzi1D+53P+B1Mpcz3gxEw/KWzz2ZSwicGN6PhstZseu+j/s7Mvxi/ATwOJkrbvaF/V4LDGtV1uWf9T7dsiD3dcL7wrW+D3b3DQDh3+Fhea97P8LuheOAV+kj+x12xSwFNgLPAKuBbe6eDBfJ3r+WfQ+frwWGdm+Nu8wPgRug5ULZQ+kb++3A02a2JLzUNBTgs17Ia3AfCDp1re8+pFe9H2ZWDjwC/KO715nl2r3MojnKDtj99syFwirN7CDgUeCYXIuFf3vFvpvZp4GN7r7EzE5uLs6xaK/a79AMd19vZsOBZ8xsZQfL7vN+9/WWRaeu9d0LfWBmhwKEfzeG5b3m/TCzGJmgeMDdfxMW9/r9zubu24AXyIzbHBRe5x723L+WfQ+fH0TmqpUHmhnAWWa2FniITFfUD+n9+427rw//biTz42AqBfis9/Ww6Mx1wnuj7GufX0qmT7+5/JLwiIlpQG1zU/ZAYpkmxE+BFe7+g6ynevV+A5hZRdiiwMzKgE+SGfB9nsx17qHtvje/J+cBz3nYmX0gcfeb3H2ku48m8//4OXefQy/fbzPrb2YDmu8DpwF/oRCf9WIPzhT7Bvwd8DaZft2bi12fAuzfg8AGIEHmV8XlZPpmFwHvhH+HhMsamaPDVgN/BqqKXf993OeTyDSt3wSWhre/6+37He7LJOCNcN//AtwSlo8FXgNWAb8CSsLy0vDxqvD5scXehy54D04GHu8L+x3u35/C27Lm77BCfNZ1BreIiOTV17uhRESkExQWIiKSl8JCRETyUliIiEheCgsREclLYSGyF8wsFc7u2XzrspmKzWy0Zc0OLNKT9PXpPkT21i53ryx2JUS6m1oWIl0gvKbA7eG1JF4zsyPD8iPMbFF47YBFZjYqLD/YzB4NrzvxJzObHm4qYmb3hdeieDo8C1uk6BQWInunrFU31AVZz9W5+1TgR2TmJSK8/5/uPgl4ALgrLL8LeNEz1504nszZt5C5zsDd7n4ssA34TIH3R6RTdAa3yF4ws+3uXp6jfC2Ziw6tCScxfN/dh5rZZjLXC0iE5RvcfZiZbQJGuntj1jZGA8945oI1mNmNQMzdv1X4PRPpmFoWIl3H27nf3jK5NGbdT6FxRekhFBYiXeeCrL+vhPf/QGYWVIA5wO/D+4uAL0HLxYoGdlclRfaFfrWI7J2y8Cp0zX7r7s2Hz5aY2atkfoRdFJbNBe43s+uBTcA/hOXXAPea2eVkWhBfIjM7sEiPpDELkS4QjllUufvmYtdFpBDUDSUiInmpZSEiInmpZSEiInkpLEREJC+FhYiI5KWwEBGRvBQWIiKSl8JCRETy+v+6HLRK//x40QAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -843,13 +1101,15 @@ ], "source": [ "#Plot the loss\n", - "def plot_loss_list(loss_list= rnn.loss_list):\n", - " plt.plot(rnn.loss_list)\n", + "def plot_loss_list(loss_list = rnn.loss_list, loss_validation = rnn.loss_validation):\n", + " plt.plot(rnn.loss_list, label='Loss on training set')\n", + " plt.plot(rnn.loss_validation, label='Loss on test set')\n", + " plt.legend()\n", " plt.xlabel(\"Epoch\")\n", " plt.ylabel(\"Cost\")\n", " plt.show()\n", "\n", - "plot_loss_list()" + "plot_loss_list(rnn.loss_list)" ] }, { @@ -865,24 +1125,45 @@ "metadata": {}, "outputs": [], "source": [ - "#folder = get_rnn_folder(ncells = ncells, cell_type = \"lstm\", activation = \"leaky_relu\")\n", - "#rnn, data = full_load(folder)" + "folder = get_rnn_folder(ncells = ncells, cell_type = cell_type, activation = activation)\n", + "#rnn, data = full_load(folder)\n", + "#minibatches, test_input, test_target = data" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(4690, 7, 3)" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "test_input.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, "outputs": [], "source": [ - "def rnn_test(rnn, test_input= test_input, test_target= test_target):\n", + "def rnn_test(rnn, test_input= test_input, test_target= test_target, scalor= rnn.scalor):\n", " \n", " #Here I predict based on my test set\n", " test_pred = rnn.predict(test_input)\n", " \n", " #Here i subtract a prediction (random particle) from the target to get an idea of the predictions\n", " #scaler_inv(test_input, scalerfunc = func)[0,:,:]\n", - " diff = scaler_inv(test_pred, scalerfunc = func)-scaler_inv(test_target, scalerfunc = func )\n", + " diff = scaler_inv(test_pred, scalerfunc = func, scalor= scalor)-scaler_inv(test_target, scalerfunc = func, scalor= scalor)\n", " print(diff[random.randint(0,test_pred.shape[0]),:,:])\n", " \n", " #Here I evaluate my model on the test set based on mean_squared_error\n", @@ -901,21 +1182,22 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[[-12.40089102 11.3887205 14.85893158]\n", - " [ -2.58961379 15.07054156 13.61432775]\n", - " [ -0.97645625 9.64583194 6.30191887]\n", - " [ -0.18267044 -0.89442267 -5.31442916]\n", - " [ 1.98853966 9.28189136 -5.76572485]\n", - " [ -0.40098165 2.10118956 -3.39426367]\n", - " [ 2.50788286 -4.49019351 2.29174324]]\n", - "Loss on test set: 0.14005564\n" + "[[ 0.22876946 -1.05757492 0.05743555]\n", + " [ 0.4227805 -1.49878024 -0.45964226]\n", + " [ 1.57940168 -0.0964699 -0.29063407]\n", + " [ 8.23204061 -7.21131909 0.2346999 ]\n", + " [ 0.63359657 -1.23937142 0.23865187]\n", + " [ 3.5459507 -2.39105333 0.56137755]\n", + " [ 0.0832766 -0.32882895 0.49288811]\n", + " [ 0. 0. 0. ]]\n", + "Loss on test set: 0.0018604315\n" ] } ], @@ -924,10 +1206,8 @@ ] }, { - "cell_type": "code", - "execution_count": null, + "cell_type": "raw", "metadata": {}, - "outputs": [], "source": [] }, {