{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"c:\\users\\sa_li\\anaconda3\\envs\\rnn-tf-ker\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
" from ._conv import register_converters as _register_converters\n"
]
}
],
"source": [
"import pandas as pd\n",
"import numpy as np\n",
"import matplotlib as mpl\n",
"import random\n",
"import math\n",
"import numpy as np\n",
"import matplotlib.pyplot as plt\n",
"import tensorflow as tf\n",
"from tensorflow.python.framework import ops"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"#import data as array\n",
"# 8 hits with x,y,z\n",
"\n",
"testset = pd.read_pickle('matched_8hittracks.pkl')"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"#Check testset with arbitrary particle\n",
"\n",
"tset = np.array(testset)\n",
"tset = tset.astype('float32')\n",
"#print(tset.shape)\n",
"#for i in range(8):\n",
" #print(tset[1,3*i:(3*i+3)])\n",
"#print(tset[0,:])"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"### Reshape original array into the shape (particlenumber, timesteps, input = coordinates)###\n",
"\n",
"def reshapor(arr_orig):\n",
" timesteps = int(arr_orig.shape[1]/3)\n",
" number_examples = int(arr_orig.shape[0])\n",
" arr = np.zeros((number_examples, timesteps, 3))\n",
" \n",
" for i in range(number_examples):\n",
" for t in range(timesteps):\n",
" arr[i,t,0:3] = arr_orig[i,3*t:3*t+3]\n",
" \n",
" return arr"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [],
"source": [
"### create the training set and the test set###\n",
"\n",
"def create_random_sets(dataset, train_to_total_ratio):\n",
" #shuffle the dataset\n",
" num_examples = dataset.shape[0]\n",
" p = np.random.permutation(num_examples)\n",
" dataset = dataset[p,:]\n",
" \n",
" #evaluate siye of training and test set and initialize them\n",
" train_set_size = np.int(num_examples*train_to_total_ratio)\n",
" test_set_size = num_examples - train_set_size\n",
" \n",
" train_set = np.zeros((train_set_size, dataset.shape[1]))\n",
" test_set = np.zeros((test_set_size, dataset.shape[1]))\n",
" \n",
"\n",
" #fill train and test sets\n",
" for i in range(num_examples):\n",
" if train_set_size > i:\n",
" train_set[i,:] += dataset[i,:]\n",
" else:\n",
" test_set[i - train_set_size,:] += dataset[i,:]\n",
" \n",
" \n",
" train_set = reshapor(train_set)\n",
" test_set = reshapor(test_set)\n",
" \n",
" return train_set, test_set\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"train_set, test_set = create_random_sets(tset, 0.99)\n",
"#print(test_set.shape, train_set.shape, reshapor(tset).shape)\n",
"#print(test_set[0,:,:])"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
"### create target array of shape (num_examples, 4 timesteps, 3 = n_inputs), inputt array of shape (num_examples, 4 timesteps, 12 = n_inputs)###\n",
"\n",
"def target_and_input(data_set):\n",
" \n",
" num_ex = data_set.shape[0]\n",
" inputt = np.zeros((num_ex, 4, 12))\n",
" target = np.zeros((num_ex, 4, 3))\n",
" \n",
" \n",
" for i in range(4):\n",
" target[:,i,:] = data_set[:,4+i,:]\n",
" for f in range(4):\n",
" inputt[:,i,3*f:3*f+3] = data_set[:,i+f,:]\n",
" \n",
" \n",
" \n",
" \n",
" return inputt, target\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"inputt_train, target_train = target_and_input(train_set)\n",
"inputt_test, target_test = target_and_input(test_set)\n",
"#print(inputt_train[0,:,:])\n",
"#print(target_train[0,:,:])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"###create random mini_batches###\n",
"\n",
"\n",
"def unison_shuffled_copies(a, b):\n",
" assert a.shape[0] == b.shape[0]\n",
" p = np.random.permutation(a.shape[0])\n",
" return a[p,:,:], b[p,:,:]\n",
"\n",
"def random_mini_batches(inputt, target, minibatch_size = 500):\n",
" \n",
" num_examples = inputt.shape[0]\n",
" \n",
" \n",
" #Number of complete batches\n",
" \n",
" number_of_batches = int(num_examples/minibatch_size)\n",
" minibatches = []\n",
" \n",
" #shuffle particles\n",
" _i, _t = unison_shuffled_copies(inputt, target)\n",
" #print(_t.shape)\n",
" \n",
" \n",
" for i in range(number_of_batches):\n",
" \n",
" minibatch_train = _i[minibatch_size*i:minibatch_size*(i+1), :, :]\n",
" \n",
" minibatch_true = _t[minibatch_size*i:minibatch_size*(i+1), :, :]\n",
" \n",
" minibatches.append((minibatch_train, minibatch_true))\n",
" \n",
" \n",
" minibatches.append((_i[number_of_batches*minibatch_size:, :, :], _t[number_of_batches*minibatch_size:, :, :]))\n",
" \n",
" \n",
" return minibatches\n",
" "
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [],
"source": [
"#Create random minibatches of train and test set with input and target array\n",
"\n",
"\n",
"minibatches = random_mini_batches(train_set[:,:-1,:], train_set[:,1:,:], minibatch_size = 1000)\n",
"#_train, _target = minibatches[0]\n",
"test_input, test_target = test_set[:,:-1,:], test_set[:,1:,:]\n",
"#print(train[0,:,:], target[0,:,:])"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
"#minibatches = random_mini_batches(inputt_train, target_train)\n",
"\n",
"\n",
"#_inputt, _target = minibatches[int(inputt_train.shape[0]/500)]\n",
"\n",
"#print(len(minibatches))\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [],
"source": [
"class RNNPlacePrediction():\n",
" \n",
" \n",
" def __init__(self, time_steps, future_steps, ninputs, ncells, num_output, cell_type=\"basic_rnn\"):\n",
" \n",
" self.nsteps = time_steps\n",
" self.future_steps = future_steps\n",
" self.ninputs = ninputs\n",
" self.ncells = ncells\n",
" self.num_output = num_output\n",
" self._ = cell_type\n",
" \n",
" #### The input is of shape (num_examples, time_steps, ninputs)\n",
" #### ninputs is the dimentionality (number of features) of the time series (here coordinates)\n",
" self.X = tf.placeholder(dtype=tf.float32, shape=(None, time_steps, ninputs))\n",
" self.Y = tf.placeholder(dtype=tf.float32, shape=(None, time_steps, ninputs))\n",
" \n",
" \n",
" if cell_type==\"basic_rnn\":\n",
" self.cell_type = tf.contrib.rnn.BasicRNNCell\n",
" \n",
" elif cell_type==\"lstm\":\n",
" self.cell_type = tf.contrib.rnn.BasicLSTMCell\n",
" \n",
" elif cell_type==\"GRU\":\n",
" self.cell_type = tf.contrib.rnn.GRUCell\n",
" \n",
" else: # JONAS\n",
" raise ValueError(\"Wrong rnn cell type: {}\".format(cell_type))\n",
" \n",
" \n",
" #Check Input of ncells\n",
" \n",
" if (type(self.ncells) == int):\n",
" self.ncells = [self.ncells]\n",
" \n",
" if (type(self.ncells) != list):\n",
" raise ValueError(\"Wrong type of Input for ncells\")\n",
" \n",
" for _ in range(len(self.ncells)):\n",
" if type(self.ncells[_]) != int:\n",
" raise ValueError(\"Wrong type of Input for ncells\")\n",
" \n",
" \n",
" self.cell = tf.contrib.rnn.MultiRNNCell([self.cell_type(num_units=self.ncells[layer], activation=tf.nn.relu)\n",
" for layer in range(len(self.ncells))])\n",
" \n",
" \n",
" #### I now define the output\n",
" self.RNNCell = tf.contrib.rnn.OutputProjectionWrapper(self.cell, output_size= num_output)\n",
" \n",
" \n",
" \n",
" \n",
" \n",
" self.sess = tf.Session()\n",
" \n",
" def set_cost_and_functions(self, LR=0.001):\n",
" #### I define here the function that unrolls the RNN cell\n",
" self.output, self.state = tf.nn.dynamic_rnn(self.RNNCell, self.X, dtype=tf.float32)\n",
" #### I define the cost function as the mean_squared_error (distance of predicted point to target)\n",
" self.cost = tf.reduce_mean(tf.losses.mean_squared_error(self.Y, self.output)) \n",
" \n",
" #### the rest proceed as usual\n",
" self.train = tf.train.AdamOptimizer(LR).minimize(self.cost)\n",
" #### Variable initializer\n",
" self.init = tf.global_variables_initializer()\n",
" self.saver = tf.train.Saver()\n",
" self.sess.run(self.init)\n",
" \n",
" \n",
" def save(self, filename=\"./rnn_model/rnn_basic\"):\n",
" self.saver.save(self.sess, filename)\n",
" \n",
" \n",
" def load(self, filename=\"./rnn_model/rnn_basic\"):\n",
" self.saver.restore(self.sess, filename)\n",
" \n",
" \n",
" \n",
" def fit(self, minibatches, epochs, print_step, checkpoint = 10, patience = 15):\n",
" self.loss_list = []\n",
" patience_cnt = 0\n",
" epoche_save = 0\n",
" \n",
" folder = \"./rnn_model_\" + str(self._) + str(self.ncells).replace(\" \",\"\") + \"c\" \"_checkpoint/rnn_basic\"\n",
" \n",
" for iep in range(epochs):\n",
" loss = 0\n",
" \n",
" batches = len(minibatches)\n",
" #Here I iterate over the batches\n",
" for batch in range(batches):\n",
" #### Here I train the RNNcell\n",
" #### The X is the time series, the Y is shifted by 1 time step\n",
" train, target = minibatches[batch]\n",
" self.sess.run(self.train, feed_dict={self.X:train, self.Y:target})\n",
" \n",
" \n",
" loss += self.sess.run(self.cost, feed_dict={self.X:train, self.Y:target})\n",
" \n",
" #Normalize loss over number of batches\n",
" loss /= batches\n",
" self.loss_list.append(loss)\n",
" \n",
" #print(loss)\n",
" \n",
" #Here I create the checkpoint if the perfomance is better\n",
" if iep > 1 and iep%checkpoint == 0 and self.loss_list[iep] < self.loss_list[epoche_save]:\n",
" #print(\"Checkpoint created at epoch: \", iep)\n",
" self.save(folder)\n",
" epoche_save = iep\n",
" \n",
" #early stopping with patience\n",
" if iep > 1 and abs(self.loss_list[iep]-self.loss_list[iep-1]) < 0.005:\n",
" patience_cnt += 1\n",
" #print(\"Patience now at: \", patience_cnt, \" of \", patience)\n",
" \n",
" if patience_cnt + 1 > patience:\n",
" print(\"Early stopping at epoch \", iep, \", difference: \", abs(self.loss_list[iep]-self.loss_list[iep-1]))\n",
" print(\"Cost: \",loss)\n",
" break\n",
" \n",
" if iep%print_step==0:\n",
" print(\"Epoch number \",iep)\n",
" print(\"Cost: \",loss)\n",
" print(\"Patience: \",patience_cnt, \"/\", patience)\n",
" print(\"Last checkpoint at: Epoch \", epoche_save, \"\\n\")\n",
" \n",
" #Set model back to the last checkpoint if performance was better\n",
" if self.loss_list[epoche_save] < self.loss_list[iep]:\n",
" self.load(folder)\n",
" print(\"Last checkpoint at epoch \", epoche_save, \" loaded\")\n",
" print(\"Performance at last checkpoint is \" ,self.loss_list[iep] - self.loss_list[epoche_save], \" better\" )\n",
" \n",
" \n",
" \n",
" def predict(self, x):\n",
" return self.sess.run(self.output, feed_dict={self.X:x})\n",
" \n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
"timesteps = 7\n",
"future_steps = 1\n",
"\n",
"ninputs = 3\n",
"\n",
"#ncells as int or list of int\n",
"ncells = [50, 40, 30, 20, 10]\n",
"\n",
"num_output = 3"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"WARNING:tensorflow:From c:\\users\\sa_li\\anaconda3\\envs\\rnn-tf-ker\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\base.py:198: retry (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Use the retry module or similar alternatives.\n"
]
}
],
"source": [
"tf.reset_default_graph()\n",
"rnn = RNNPlacePrediction(time_steps=timesteps, future_steps=future_steps, ninputs=ninputs, \n",
" ncells=ncells, num_output=num_output, cell_type=\"lstm\")"
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [],
"source": [
"rnn.set_cost_and_functions()"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Epoch number 0\n",
"Cost: 3.893127314587857\n",
"Patience: 0 / 5\n",
"Last checkpoint at: Epoch 0 \n",
"\n",
"Early stopping at epoch 10 , difference: 0.002693013941988287\n",
"Cost: 3.91306800537921\n",
"INFO:tensorflow:Restoring parameters from ./rnn_model_lstm[50,40,30,20,10]c_checkpoint/rnn_basic\n",
"Last checkpoint at epoch 0 loaded\n",
"Performance at last checkpoint is 0.019940690791353077 better\n"
]
}
],
"source": [
"rnn.fit(minibatches, epochs=22, print_step=10)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"plt.plot(rnn.loss_list)\n",
"plt.xlabel(\"Epoch\")\n",
"plt.ylabel(\"Cost\")\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [],
"source": [
"#save in a folder that describes the model\n",
"folder = \"./rnn_model_\" + str(rnn._) + \"_\" + str(len(rnn.ncells)) + \"l_\" + str(rnn.ncells).replace(\" \",\"\") + \"c/rnn_basic\"\n",
"rnn.save(folder)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [],
"source": [
"#folder = \"./trained_models/rnn_model_\" + str(rnn._) + \"_\" + str(len(rnn.ncells)) + \"l_\" + str(rnn.ncells).replace(\" \",\"\") + \"c/rnn_basic\"\n",
"#rnn.load(folder)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [],
"source": [
"###test_input.shape###"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [],
"source": [
"#Here I predict based on my test set\n",
"\n",
"test_pred = rnn.predict(test_input)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
"#Here i subtract a prediction (random particle) from the target to get an idea of the predictions\n",
"\n",
"#print(test_pred[5,:,:]-test_target[5,:,:])"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"4.3867254"
]
},
"execution_count": 25,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"#Here I evaluate my model on the test set based on mean_squared_error\n",
"\n",
"rnn.sess.run(rnn.cost, feed_dict={rnn.X:test_input, rnn.Y:test_target})"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"scrolled": true
},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.5"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
