{ "name" : "stderr" , "output_type" : "stream" , "text" : [ "c:\\users\\sascha\\anaconda3\\envs\\rnn-tf-ker\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n" , "c:\\users\\sa_li\\anaconda3\\envs\\rnn-tf-ker\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n" , " from ._conv import register_converters as _register_converters\n" ] } ], ] }, { "cell_type" : "code" , "execution_count" : 10 , "execution_count" : 24 , "metadata" : {}, "outputs" : [], "source" : [ "#Create random minibatches of train and test set with input and target array\n" , "\n" , "\n" , "minibatches = random_mini_batches(train_set[:,:-1,:], train_set[:,1:,:])\n" , "minibatches = random_mini_batches(train_set[:,:-1,:], train_set[:,1:,:], minibatch_size = 1000)\n" , "#_train, _target = minibatches[0]\n" , "test_input, test_target = test_set[:,:-1,:], test_set[:,1:,:]\n" , "#print(train[0,:,:], target[0,:,:])" ] }, { "cell_type" : "code" , "execution_count" : 11 , "execution_count" : 25 , "metadata" : {}, "outputs" : [], "source" : [ "#minibatches = random_mini_batches(inputt_train, target_train)\n" , "source" : [] }, { "cell_type" : "code" , "execution_count" : 67 , "execution_count" : 26 , "metadata" : {}, "outputs" : [], "source" : [ "class RNNPlacePrediction():\n" , " self.future_steps = future_steps\n" , " self.ninputs = ninputs\n" , " self.ncells = ncells\n" , " self.num_output = num_output\n" , " \n" , " self._ = cell_type\n" , " \n" , " #### The input is of shape (num_examples, time_steps, ninputs)\n" , " #### ninputs is the dimentionality (number of features) of the time series (here coordinates)\n" , " #### Variable initializer\n" , " self.init = tf.global_variables_initializer()\n" , " self.saver = tf.train.Saver()\n" , " self.sess.run(self.init)\n" , " \n" , " \n" , " \n" , " def fit(self, minibatches, epochs, print_step):\n" , " \n" , " \n" , " def save(self, filename=\"./rnn_model/rnn_basic\"):\n" , " self.saver.save(self.sess, filename)\n" , " \n" , " \n" , " def load(self, filename=\"./rnn_model/rnn_basic\"):\n" , " self.saver.restore(self.sess, filename)\n" , " \n" , " \n" , " \n" , " def fit(self, minibatches, epochs, print_step, checkpoint = 10, patience = 80):\n" , " self.loss_list = []\n" , " patience_cnt = 0\n" , " epoche_save = 0\n" , " \n" , " folder = \"./rnn_model_\" + str(self._) + str(self.ncells).replace(\" \",\"\") + \"c\" \"_checkpoint/rnn_basic\"\n" , " \n" , " for iep in range(epochs):\n" , " loss = 0\n" , " \n" , " #Here I iterate through the batches\n" , " for batch in range(len(minibatches)):\n" , " #### Here I train the RNNcell\n" , " #### The X is the time serie, the Z is shifted by 1 time step\n" , " #### The X is the time series, the Y is shifted by 1 time step\n" , " train, target = minibatches[batch]\n" , " self.sess.run(self.train, feed_dict={self.X:train, self.Y:target})\n" , " \n" , " \n" , " loss += self.sess.run(self.cost, feed_dict={self.X:train, self.Y:target})\n" , " \n" , " self.loss_list.append(loss)\n" , " \n" , " print(loss)\n" , " \n" , " \n" , " #early stopping\n" , " if iep > 100 and abs(self.loss_list[iep]-self.loss_list[iep-100]) < 0.5:\n" , " print(\"Early stopping at epoch \", iep, \", difference: \", abs(self.loss_list[iep]-self.loss_list[iep-100]))\n" , " break\n" , " #print(loss)\n" , " \n" , " #Here I create the checkpoint if the perfomance is better\n" , " if iep > 1 and iep%checkpoint == 0 and self.loss_list[iep] < self.loss_list[epoche_save]:\n" , " #print(\"Checkpoint created at epoch: \", iep)\n" , " self.save(folder)\n" , " epoche_save = iep\n" , " \n" , " #early stopping with patience\n" , " if iep > 1 and abs(self.loss_list[iep]-self.loss_list[iep-1]) < 0.05:\n" , " patience_cnt += 1\n" , " #print(\"Patience now at: \", patience_cnt, \" of \", patience)\n" , " if patience_cnt + 1 > patience:\n" , " print(\"Early stopping at epoch \", iep, \", difference: \", abs(self.loss_list[iep]-self.loss_list[iep-1]))\n" , " \n" , " #Set model back to the last checkpoint if performance was better\n" , " if self.loss_list[epoche_save] < self.loss_list[iep]:\n" , " self.load(folder)\n" , " print(\"Last checkpoint at epoch \", epoche_save, \" loaded\")\n" , " print(\"Performance at last checkpoint is \" ,self.loss_list[iep] - self.loss_list[epoche_save], \" better\" )\n" , " break\n" , " \n" , " if iep%print_step==0:\n" , " print(\"Epoch number \",iep)\n" , " print(\"Cost: \",loss)\n" , " \n" , " \n" , " \n" , " def save(self, filename=\"./rnn_model/rnn_basic\"):\n" , " self.saver.save(self.sess, filename)\n" , " \n" , " \n" , " def load(self, filename=\"./rnn_model/rnn_basic\"):\n" , " self.saver.restore(self.sess, filename)\n" , " \n" , " print(\"Patience: \",patience_cnt, \"/\", patience)\n" , " print(\"Last checkpoint at: Epoch \", epoche_save, \"\\n\")\n" , " \n" , " \n" , " \n" , " def predict(self, x):\n" , " return self.sess.run(self.output, feed_dict={self.X:x})\n" , " \n" , "source" : [] }, { "cell_type" : "code" , "execution_count" : 83 , "execution_count" : 27 , "metadata" : {}, "outputs" : [], "source" : [ "timesteps = 7\n" , "\n" , "ninputs = 3\n" , "\n" , "#ncells as int or list of int\n" , "ncells = [30, 30, 30]\n" , "ncells = [50, 40, 30, 20, 10]\n" , "\n" , "num_output = 3" ] }, { "cell_type" : "code" , "execution_count" : 84 , "execution_count" : 28 , "metadata" : {}, "outputs" : [], "source" : [ "tf.reset_default_graph()\n" , ] }, { "cell_type" : "code" , "execution_count" : 85 , "execution_count" : 29 , "metadata" : {}, "outputs" : [], "source" : [ "rnn.set_cost_and_functions()" ] }, { "cell_type" : "code" , "execution_count" : 86 , "metadata" : { "scrolled" : false }, "outputs" : [ { "name" : "stdout" , "output_type" : "stream" , "text" : [ "106114.12908935547\n" , "Epoch number 0\n" , "Cost: 106114.12908935547\n" , "37096.763900756836\n" , "15050.049377441406\n" , "10696.98137664795\n" , "8839.393844604492\n" , "7496.866744995117\n" , "6476.7746925354\n" , "5627.203907012939\n" , "4952.669666290283\n" , "4406.38809967041\n" ] } ], "source" : [ "rnn.fit(minibatches, epochs=10, print_step=500)" ] }, { "cell_type" : "code" , "execution_count" : 87 , "metadata" : {}, "outputs" : [], "source" : [ "folder = \"./trained_models/rnn_model_\" + str(rnn._) + \"_\" + str(len(rnn.ncells)) + \"l_\" + str(ncells) + \"c/rnn_basic\"\n" , "rnn.save(folder)" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [ "###rnn.load(folder)###" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [ "###test_input.shape###" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [ "#Here I predict based on my test set\n" , "\n" , "#test_pred = rnn.predict(test_input)" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [ "#Here i subtract a prediction (random particle) from the target to get an idea of the predictions\n" , "\n" , "#print(test_pred[5,:,:]-test_target[5,:,:])" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [ "#Here I evaluate my model on the test set based on mean_squared_error\n" , "\n" , "#rnn.sess.run(rnn.cost, feed_dict={rnn.X:test_input, rnn.Y:test_target})" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [] }, { "cell_type" : "code" , "execution_count" : 82 , "execution_count" : null , "metadata" : { "scrolled" : true }, "outputs" : [ { "name" : "stdout" , "output_type" : "stream" , "text" : [ "1\n" "Epoch number 0\n" , "Cost: 65161.071533203125\n" , "Patience: 0 / 80\n" , "Last checkpoint at: Epoch 0 \n" , "\n" ] } ], "source" : [] "source" : [ "rnn.fit(minibatches, epochs=5000, print_step=500)" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [ "#save in a folder that describes the model\n" , "folder = \"./rnn_model_\" + str(rnn._) + \"_\" + str(len(rnn.ncells)) + \"l_\" + str(rnn.ncells).replace(\" \",\"\") + \"c/rnn_basic\"\n" , "rnn.save(folder)" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [ "###rnn.load(folder)###" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [ "###test_input.shape###" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [ "#Here I predict based on my test set\n" , "\n" , "#test_pred = rnn.predict(test_input)" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [ "#Here i subtract a prediction (random particle) from the target to get an idea of the predictions\n" , "\n" , "#print(test_pred[5,:,:]-test_target[5,:,:])" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [ "#Here I evaluate my model on the test set based on mean_squared_error\n" , "\n" , "#rnn.sess.run(rnn.cost, feed_dict={rnn.X:test_input, rnn.Y:test_target})" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : {}, "outputs" : [], "source" : [ "print(1%10)" ] }, { "cell_type" : "code" , "execution_count" : null , "metadata" : { "scrolled" : true }, "outputs" : [], "source" : [ "z = 0\n" , "for _ in range(len(rnn.loss_list)):\n" , " if abs(rnn.loss_list[_]-rnn.loss_list[_-1]) < 0.5:\n" , " z += 1\n" , " print(_)\n" , "\n" , "print(z)" ] }, { "cell_type" : "code" , "execution_count" : null ,