Early stopping with patience added
Checkpoints during training added to go back to best state if perfomance decreases
1 parent 618a744 commit f874eb88c9e4eb31afb5f3a31940748b3978fe6b
@Sascha Liechti Sascha Liechti authored on 16 Apr 2018
Showing 1 changed file
View
502
1_to_1_multi_layer.ipynb
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\users\\sascha\\anaconda3\\envs\\rnn-tf-ker\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
"c:\\users\\sa_li\\anaconda3\\envs\\rnn-tf-ker\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
" from ._conv import register_converters as _register_converters\n"
]
}
],
]
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
"#Create random minibatches of train and test set with input and target array\n",
"\n",
"\n",
"minibatches = random_mini_batches(train_set[:,:-1,:], train_set[:,1:,:])\n",
"minibatches = random_mini_batches(train_set[:,:-1,:], train_set[:,1:,:], minibatch_size = 1000)\n",
"#_train, _target = minibatches[0]\n",
"test_input, test_target = test_set[:,:-1,:], test_set[:,1:,:]\n",
"#print(train[0,:,:], target[0,:,:])"
]
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 25,
"metadata": {},
"outputs": [],
"source": [
"#minibatches = random_mini_batches(inputt_train, target_train)\n",
"source": []
},
{
"cell_type": "code",
"execution_count": 67,
"execution_count": 26,
"metadata": {},
"outputs": [],
"source": [
"class RNNPlacePrediction():\n",
" self.future_steps = future_steps\n",
" self.ninputs = ninputs\n",
" self.ncells = ncells\n",
" self.num_output = num_output\n",
" \n",
" self._ = cell_type\n",
" \n",
" #### The input is of shape (num_examples, time_steps, ninputs)\n",
" #### ninputs is the dimentionality (number of features) of the time series (here coordinates)\n",
" #### Variable initializer\n",
" self.init = tf.global_variables_initializer()\n",
" self.saver = tf.train.Saver()\n",
" self.sess.run(self.init)\n",
" \n",
" \n",
" \n",
" def fit(self, minibatches, epochs, print_step):\n",
" \n",
" \n",
" def save(self, filename=\"./rnn_model/rnn_basic\"):\n",
" self.saver.save(self.sess, filename)\n",
" \n",
" \n",
" def load(self, filename=\"./rnn_model/rnn_basic\"):\n",
" self.saver.restore(self.sess, filename)\n",
" \n",
" \n",
" \n",
" def fit(self, minibatches, epochs, print_step, checkpoint = 10, patience = 80):\n",
" self.loss_list = []\n",
" patience_cnt = 0\n",
" epoche_save = 0\n",
" \n",
" folder = \"./rnn_model_\" + str(self._) + str(self.ncells).replace(\" \",\"\") + \"c\" \"_checkpoint/rnn_basic\"\n",
" \n",
" for iep in range(epochs):\n",
" loss = 0\n",
" \n",
" #Here I iterate through the batches\n",
" for batch in range(len(minibatches)):\n",
" #### Here I train the RNNcell\n",
" #### The X is the time serie, the Z is shifted by 1 time step\n",
" #### The X is the time series, the Y is shifted by 1 time step\n",
" train, target = minibatches[batch]\n",
" self.sess.run(self.train, feed_dict={self.X:train, self.Y:target})\n",
" \n",
" \n",
" loss += self.sess.run(self.cost, feed_dict={self.X:train, self.Y:target})\n",
" \n",
" self.loss_list.append(loss)\n",
" \n",
" print(loss)\n",
" \n",
" \n",
" #early stopping\n",
" if iep > 100 and abs(self.loss_list[iep]-self.loss_list[iep-100]) < 0.5:\n",
" print(\"Early stopping at epoch \", iep, \", difference: \", abs(self.loss_list[iep]-self.loss_list[iep-100]))\n",
" break\n",
" #print(loss)\n",
" \n",
" #Here I create the checkpoint if the perfomance is better\n",
" if iep > 1 and iep%checkpoint == 0 and self.loss_list[iep] < self.loss_list[epoche_save]:\n",
" #print(\"Checkpoint created at epoch: \", iep)\n",
" self.save(folder)\n",
" epoche_save = iep\n",
" \n",
" #early stopping with patience\n",
" if iep > 1 and abs(self.loss_list[iep]-self.loss_list[iep-1]) < 0.05:\n",
" patience_cnt += 1\n",
" #print(\"Patience now at: \", patience_cnt, \" of \", patience)\n",
" if patience_cnt + 1 > patience:\n",
" print(\"Early stopping at epoch \", iep, \", difference: \", abs(self.loss_list[iep]-self.loss_list[iep-1]))\n",
" \n",
" #Set model back to the last checkpoint if performance was better\n",
" if self.loss_list[epoche_save] < self.loss_list[iep]:\n",
" self.load(folder)\n",
" print(\"Last checkpoint at epoch \", epoche_save, \" loaded\")\n",
" print(\"Performance at last checkpoint is \" ,self.loss_list[iep] - self.loss_list[epoche_save], \" better\" )\n",
" break\n",
" \n",
" if iep%print_step==0:\n",
" print(\"Epoch number \",iep)\n",
" print(\"Cost: \",loss)\n",
" \n",
" \n",
" \n",
" def save(self, filename=\"./rnn_model/rnn_basic\"):\n",
" self.saver.save(self.sess, filename)\n",
" \n",
" \n",
" def load(self, filename=\"./rnn_model/rnn_basic\"):\n",
" self.saver.restore(self.sess, filename)\n",
" \n",
" print(\"Patience: \",patience_cnt, \"/\", patience)\n",
" print(\"Last checkpoint at: Epoch \", epoche_save, \"\\n\")\n",
" \n",
" \n",
" \n",
" def predict(self, x):\n",
" return self.sess.run(self.output, feed_dict={self.X:x})\n",
" \n",
"source": []
},
{
"cell_type": "code",
"execution_count": 83,
"execution_count": 27,
"metadata": {},
"outputs": [],
"source": [
"timesteps = 7\n",
"\n",
"ninputs = 3\n",
"\n",
"#ncells as int or list of int\n",
"ncells = [30, 30, 30]\n",
"ncells = [50, 40, 30, 20, 10]\n",
"\n",
"num_output = 3"
]
},
{
"cell_type": "code",
"execution_count": 84,
"execution_count": 28,
"metadata": {},
"outputs": [],
"source": [
"tf.reset_default_graph()\n",
]
},
{
"cell_type": "code",
"execution_count": 85,
"execution_count": 29,
"metadata": {},
"outputs": [],
"source": [
"rnn.set_cost_and_functions()"
]
},
{
"cell_type": "code",
"execution_count": 86,
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"106114.12908935547\n",
"Epoch number 0\n",
"Cost: 106114.12908935547\n",
"37096.763900756836\n",
"15050.049377441406\n",
"10696.98137664795\n",
"8839.393844604492\n",
"7496.866744995117\n",
"6476.7746925354\n",
"5627.203907012939\n",
"4952.669666290283\n",
"4406.38809967041\n"
]
}
],
"source": [
"rnn.fit(minibatches, epochs=10, print_step=500)"
]
},
{
"cell_type": "code",
"execution_count": 87,
"metadata": {},
"outputs": [],
"source": [
"folder = \"./trained_models/rnn_model_\" + str(rnn._) + \"_\" + str(len(rnn.ncells)) + \"l_\" + str(ncells) + \"c/rnn_basic\"\n",
"rnn.save(folder)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"###rnn.load(folder)###"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"###test_input.shape###"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#Here I predict based on my test set\n",
"\n",
"#test_pred = rnn.predict(test_input)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#Here i subtract a prediction (random particle) from the target to get an idea of the predictions\n",
"\n",
"#print(test_pred[5,:,:]-test_target[5,:,:])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#Here I evaluate my model on the test set based on mean_squared_error\n",
"\n",
"#rnn.sess.run(rnn.cost, feed_dict={rnn.X:test_input, rnn.Y:test_target})"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 82,
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"1\n"
"Epoch number 0\n",
"Cost: 65161.071533203125\n",
"Patience: 0 / 80\n",
"Last checkpoint at: Epoch 0 \n",
"\n"
]
}
],
"source": []
"source": [
"rnn.fit(minibatches, epochs=5000, print_step=500)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#save in a folder that describes the model\n",
"folder = \"./rnn_model_\" + str(rnn._) + \"_\" + str(len(rnn.ncells)) + \"l_\" + str(rnn.ncells).replace(\" \",\"\") + \"c/rnn_basic\"\n",
"rnn.save(folder)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"###rnn.load(folder)###"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"###test_input.shape###"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#Here I predict based on my test set\n",
"\n",
"#test_pred = rnn.predict(test_input)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#Here i subtract a prediction (random particle) from the target to get an idea of the predictions\n",
"\n",
"#print(test_pred[5,:,:]-test_target[5,:,:])"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"#Here I evaluate my model on the test set based on mean_squared_error\n",
"\n",
"#rnn.sess.run(rnn.cost, feed_dict={rnn.X:test_input, rnn.Y:test_target})"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(1%10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": [
"z = 0\n",
"for _ in range(len(rnn.loss_list)):\n",
" if abs(rnn.loss_list[_]-rnn.loss_list[_-1]) < 0.5:\n",
" z += 1\n",
" print(_)\n",
"\n",
"print(z)"
]
},
{
"cell_type": "code",
"execution_count": null,