diff --git a/1_to_1_multi_layer.ipynb b/1_to_1_multi_layer.ipynb index 0d8528f..839daa0 100644 --- a/1_to_1_multi_layer.ipynb +++ b/1_to_1_multi_layer.ipynb @@ -9,7 +9,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "c:\\users\\sascha\\anaconda3\\envs\\rnn-tf-ker\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", + "c:\\users\\sa_li\\anaconda3\\envs\\rnn-tf-ker\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n", " from ._conv import register_converters as _register_converters\n" ] } @@ -208,14 +208,14 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ "#Create random minibatches of train and test set with input and target array\n", "\n", "\n", - "minibatches = random_mini_batches(train_set[:,:-1,:], train_set[:,1:,:])\n", + "minibatches = random_mini_batches(train_set[:,:-1,:], train_set[:,1:,:], minibatch_size = 1000)\n", "#_train, _target = minibatches[0]\n", "test_input, test_target = test_set[:,:-1,:], test_set[:,1:,:]\n", "#print(train[0,:,:], target[0,:,:])" @@ -223,7 +223,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -259,7 +259,7 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -273,7 +273,6 @@ " self.ninputs = ninputs\n", " self.ncells = ncells\n", " self.num_output = num_output\n", - " \n", " self._ = cell_type\n", " \n", " #### The input is of shape (num_examples, time_steps, ninputs)\n", @@ -333,18 +332,31 @@ " self.init = tf.global_variables_initializer()\n", " self.saver = tf.train.Saver()\n", " self.sess.run(self.init)\n", + " \n", + " \n", + " def save(self, filename=\"./rnn_model/rnn_basic\"):\n", + " self.saver.save(self.sess, filename)\n", + " \n", + " \n", + " def load(self, filename=\"./rnn_model/rnn_basic\"):\n", + " self.saver.restore(self.sess, filename)\n", " \n", " \n", " \n", - " def fit(self, minibatches, epochs, print_step):\n", + " def fit(self, minibatches, epochs, print_step, checkpoint = 10, patience = 80):\n", " self.loss_list = []\n", + " patience_cnt = 0\n", + " epoche_save = 0\n", + " \n", + " folder = \"./rnn_model_\" + str(self._) + str(self.ncells).replace(\" \",\"\") + \"c\" \"_checkpoint/rnn_basic\"\n", + " \n", " for iep in range(epochs):\n", " loss = 0\n", " \n", " #Here I iterate through the batches\n", " for batch in range(len(minibatches)):\n", " #### Here I train the RNNcell\n", - " #### The X is the time serie, the Z is shifted by 1 time step\n", + " #### The X is the time series, the Y is shifted by 1 time step\n", " train, target = minibatches[batch]\n", " self.sess.run(self.train, feed_dict={self.X:train, self.Y:target})\n", " \n", @@ -353,27 +365,35 @@ " \n", " self.loss_list.append(loss)\n", " \n", - " print(loss)\n", + " #print(loss)\n", " \n", + " #Here I create the checkpoint if the perfomance is better\n", + " if iep > 1 and iep%checkpoint == 0 and self.loss_list[iep] < self.loss_list[epoche_save]:\n", + " #print(\"Checkpoint created at epoch: \", iep)\n", + " self.save(folder)\n", + " epoche_save = iep\n", " \n", - " #early stopping\n", - " if iep > 100 and abs(self.loss_list[iep]-self.loss_list[iep-100]) < 0.5:\n", - " print(\"Early stopping at epoch \", iep, \", difference: \", abs(self.loss_list[iep]-self.loss_list[iep-100]))\n", - " break\n", + " #early stopping with patience\n", + " if iep > 1 and abs(self.loss_list[iep]-self.loss_list[iep-1]) < 0.05:\n", + " patience_cnt += 1\n", + " #print(\"Patience now at: \", patience_cnt, \" of \", patience)\n", + " if patience_cnt + 1 > patience:\n", + " print(\"Early stopping at epoch \", iep, \", difference: \", abs(self.loss_list[iep]-self.loss_list[iep-1]))\n", + " \n", + " #Set model back to the last checkpoint if performance was better\n", + " if self.loss_list[epoche_save] < self.loss_list[iep]:\n", + " self.load(folder)\n", + " print(\"Last checkpoint at epoch \", epoche_save, \" loaded\")\n", + " print(\"Performance at last checkpoint is \" ,self.loss_list[iep] - self.loss_list[epoche_save], \" better\" )\n", + " break\n", " \n", " if iep%print_step==0:\n", " print(\"Epoch number \",iep)\n", " print(\"Cost: \",loss)\n", - " \n", - " \n", - " \n", - " def save(self, filename=\"./rnn_model/rnn_basic\"):\n", - " self.saver.save(self.sess, filename)\n", + " print(\"Patience: \",patience_cnt, \"/\", patience)\n", + " print(\"Last checkpoint at: Epoch \", epoche_save, \"\\n\")\n", " \n", " \n", - " def load(self, filename=\"./rnn_model/rnn_basic\"):\n", - " self.saver.restore(self.sess, filename)\n", - " \n", " \n", " def predict(self, x):\n", " return self.sess.run(self.output, feed_dict={self.X:x})\n", @@ -390,7 +410,7 @@ }, { "cell_type": "code", - "execution_count": 83, + "execution_count": 27, "metadata": {}, "outputs": [], "source": [ @@ -400,14 +420,14 @@ "ninputs = 3\n", "\n", "#ncells as int or list of int\n", - "ncells = [30, 30, 30]\n", + "ncells = [50, 40, 30, 20, 10]\n", "\n", "num_output = 3" ] }, { "cell_type": "code", - "execution_count": 84, + "execution_count": 28, "metadata": {}, "outputs": [], "source": [ @@ -418,7 +438,7 @@ }, { "cell_type": "code", - "execution_count": 85, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -427,41 +447,35 @@ }, { "cell_type": "code", - "execution_count": 86, + "execution_count": null, "metadata": { - "scrolled": false + "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "106114.12908935547\n", "Epoch number 0\n", - "Cost: 106114.12908935547\n", - "37096.763900756836\n", - "15050.049377441406\n", - "10696.98137664795\n", - "8839.393844604492\n", - "7496.866744995117\n", - "6476.7746925354\n", - "5627.203907012939\n", - "4952.669666290283\n", - "4406.38809967041\n" + "Cost: 65161.071533203125\n", + "Patience: 0 / 80\n", + "Last checkpoint at: Epoch 0 \n", + "\n" ] } ], "source": [ - "rnn.fit(minibatches, epochs=10, print_step=500)" + "rnn.fit(minibatches, epochs=5000, print_step=500)" ] }, { "cell_type": "code", - "execution_count": 87, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "folder = \"./trained_models/rnn_model_\" + str(rnn._) + \"_\" + str(len(rnn.ncells)) + \"l_\" + str(ncells) + \"c/rnn_basic\"\n", + "#save in a folder that describes the model\n", + "folder = \"./rnn_model_\" + str(rnn._) + \"_\" + str(len(rnn.ncells)) + \"l_\" + str(rnn.ncells).replace(\" \",\"\") + \"c/rnn_basic\"\n", "rnn.save(folder)" ] }, @@ -521,24 +535,26 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "print(1%10)" + ] }, { "cell_type": "code", - "execution_count": 82, + "execution_count": null, "metadata": { "scrolled": true }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "1\n" - ] - } - ], - "source": [] + "outputs": [], + "source": [ + "z = 0\n", + "for _ in range(len(rnn.loss_list)):\n", + " if abs(rnn.loss_list[_]-rnn.loss_list[_-1]) < 0.5:\n", + " z += 1\n", + " print(_)\n", + "\n", + "print(z)" + ] }, { "cell_type": "code",