diff --git a/Report/00_main.log b/Report/00_main.log index 0c34dd5..7d5e1c1 100644 --- a/Report/00_main.log +++ b/Report/00_main.log @@ -1,4 +1,4 @@ -This is pdfTeX, Version 3.14159265-2.6-1.40.19 (MiKTeX 2.9.6730 64-bit) (preloaded format=pdflatex 2018.7.26) 1 AUG 2018 16:06 +This is pdfTeX, Version 3.14159265-2.6-1.40.19 (MiKTeX 2.9.6730 64-bit) (preloaded format=pdflatex 2018.7.26) 1 AUG 2018 20:31 entering extended mode **./00_main.tex (00_main.tex @@ -1680,12 +1680,12 @@ [] - + File: img/beta_decay_feynman.png Graphic file (type png) Package pdftex.def Info: img/beta_decay_feynman.png used on input line 85. (pdftex.def) Requested size: 140.39958pt x 140.40762pt. - + File: img/muon-decay-feynman.png Graphic file (type png) Package pdftex.def Info: img/muon-decay-feynman.png used on input line 90. @@ -1721,7 +1721,7 @@ [] [8] - + File: img/neutrino_oscillation.png Graphic file (type png) Package pdftex.def Info: img/neutrino_oscillation.png used on input line 144. @@ -1737,17 +1737,17 @@ [] [9 <./img/neutrino_oscillation.png>] - + File: img/LFV-neutrino_osc.png Graphic file (type png) Package pdftex.def Info: img/LFV-neutrino_osc.png used on input line 168. (pdftex.def) Requested size: 140.39958pt x 70.74054pt. - + File: img/LFV-SUSY.png Graphic file (type png) Package pdftex.def Info: img/LFV-SUSY.png used on input line 173. (pdftex.def) Requested size: 140.39958pt x 71.59482pt. - + File: img/LFV-tree_lvl.png Graphic file (type png) Package pdftex.def Info: img/LFV-tree_lvl.png used on input line 178. @@ -1801,27 +1801,27 @@ \openout2 = `03_experimental_setup.aux'. (03_experimental_setup.tex - + File: img/setup-Ia.png Graphic file (type png) Package pdftex.def Info: img/setup-Ia.png used on input line 23. (pdftex.def) Requested size: 312.00119pt x 145.84636pt. - + File: img/tracks-phase_I.png Graphic file (type png) Package pdftex.def Info: img/tracks-phase_I.png used on input line 28. (pdftex.def) Requested size: 140.39958pt x 143.79482pt. - + File: img/tracks-phase_II.png Graphic file (type png) Package pdftex.def Info: img/tracks-phase_II.png used on input line 33. (pdftex.def) Requested size: 140.39958pt x 145.23878pt. - + File: img/setup-Ib.png Graphic file (type png) Package pdftex.def Info: img/setup-Ib.png used on input line 38. (pdftex.def) Requested size: 390.0pt x 123.95313pt. - + File: img/setup-II.png Graphic file (type png) Package pdftex.def Info: img/setup-II.png used on input line 43. @@ -1845,13 +1845,13 @@ [] [17] - + File: img/tracks_in_det_xy.png Graphic file (type png) Package pdftex.def Info: img/tracks_in_det_xy.png used on input line 70. (pdftex.def) Requested size: 312.00119pt x 361.16603pt. [18] - + File: img/tracks_in_det_z.png Graphic file (type png) Package pdftex.def Info: img/tracks_in_det_z.png used on input line 78. @@ -1874,12 +1874,12 @@ ] - + File: img/neural_network.png Graphic file (type png) Package pdftex.def Info: img/neural_network.png used on input line 29. (pdftex.def) Requested size: 312.00119pt x 169.33112pt. - + File: img/neuron.png Graphic file (type png) Package pdftex.def Info: img/neuron.png used on input line 34. @@ -1890,17 +1890,17 @@ [] [22 <./img/neural_network.png> <./img/neuron.png>] - + File: img/selu.png Graphic file (type png) Package pdftex.def Info: img/selu.png used on input line 59. (pdftex.def) Requested size: 175.49881pt x 120.31479pt. - + File: img/relu.png Graphic file (type png) Package pdftex.def Info: img/relu.png used on input line 64. (pdftex.def) Requested size: 175.49881pt x 120.00516pt. - + File: img/tanh.png Graphic file (type png) Package pdftex.def Info: img/tanh.png used on input line 69. @@ -1914,13 +1914,13 @@ LaTeX Font Info: Font shape `OMS/cmr/m/n' in size <12> not available (Font) Font shape `OMS/cmsy/m/n' tried instead on input line 99. [25] [26] - + File: img/batch_norm.jpeg Graphic file (type jpg) Package pdftex.def Info: img/batch_norm.jpeg used on input line 164. (pdftex.def) Requested size: 390.0pt x 134.28722pt. [27 <./img/batch_norm.jpeg>] - + File: img/RNN_general_architecture.png Graphic file (type png) Package pdftex.def Info: img/RNN_general_architecture.png used on input line 1 @@ -1946,7 +1946,7 @@ [] - + File: img/LSTM_cell.png Graphic file (type png) Package pdftex.def Info: img/LSTM_cell.png used on input line 217. @@ -1976,7 +1976,7 @@ \openout2 = `06_RNN_used.aux'. (06_RNN_used.tex - + File: img/RNN-Pred-Arch.png Graphic file (type png) Package pdftex.def Info: img/RNN-Pred-Arch.png used on input line 9. @@ -1995,7 +1995,7 @@ [] - + File: img/RNN-Classifier-Arch.png Graphic file (type png) Package pdftex.def Info: img/RNN-Classifier-Arch.png used on input line 45. @@ -2025,40 +2025,84 @@ [] - -File: img/XGB_tf-ft_hist.png Graphic file (type png) - -Package pdftex.def Info: img/XGB_tf-ft_hist.png used on input line 20. -(pdftex.def) Requested size: 312.00119pt x 226.13411pt. - -File: img/XGB_ROC-curve.png Graphic file (type png) - -Package pdftex.def Info: img/XGB_ROC-curve.png used on input line 25. + +Underfull \hbox (badness 10000) in paragraph at lines 11--12 + + [] + + +File: img/RNN_tf-ft_hist.png Graphic file (type png) + +Package pdftex.def Info: img/RNN_tf-ft_hist.png used on input line 16. +(pdftex.def) Requested size: 312.00119pt x 223.1071pt. + +File: img/RNN_ROC-curve.png Graphic file (type png) + +Package pdftex.def Info: img/RNN_ROC-curve.png used on input line 21. (pdftex.def) Requested size: 312.00119pt x 213.3579pt. [37 ] -Underfull \hbox (badness 10000) in paragraph at lines 33--34 +Underfull \hbox (badness 10000) in paragraph at lines 29--30 [] -[38 <./img/XGB_tf-ft_hist.png> <./img/XGB_ROC-curve.png>] -Underfull \hbox (badness 10000) in paragraph at lines 35--36 +[38 <./img/RNN_tf-ft_hist.png> <./img/RNN_ROC-curve.png>] + +File: img/XGB_tf-ft_hist.png Graphic file (type png) + +Package pdftex.def Info: img/XGB_tf-ft_hist.png used on input line 40. +(pdftex.def) Requested size: 312.00119pt x 226.13411pt. + +File: img/XGB_ROC-curve.png Graphic file (type png) + +Package pdftex.def Info: img/XGB_ROC-curve.png used on input line 45. +(pdftex.def) Requested size: 312.00119pt x 213.3579pt. + [39] +Underfull \hbox (badness 10000) in paragraph at lines 53--54 [] -) [39] +[40 <./img/XGB_tf-ft_hist.png> <./img/XGB_ROC-curve.png>] +Underfull \hbox (badness 10000) in paragraph at lines 55--56 + + [] + + +Package caption Warning: \label without proper reference on input line 64. +See the caption package documentation for explanation. + + +LaTeX Warning: Reference `RNN-XGB_ROC' on page 41 undefined on input line 59. + + +File: img/RNN-XGB_ROC-curve_comparison.png Graphic file (type png) + +Package pdftex.def Info: img/RNN-XGB_ROC-curve_comparison.png used on input li +ne 63. +(pdftex.def) Requested size: 312.00119pt x 213.3579pt. +[41 <./img/RNN-XGB_ROC-curve_comparison.png>] +Underfull \hbox (badness 10000) in paragraph at lines 74--75 + + [] + + +Underfull \hbox (badness 10000) in paragraph at lines 76--82 + + [] + +) [42] \openout2 = `08_Appendix.aux'. - (08_Appendix.tex) [40 + (08_Appendix.tex) [43 -] (00_main.bbl [41 +] (00_main.bbl [44 -] [42]) +] [45]) Package atveryend Info: Empty hook `BeforeClearDocument' on input line 72. - [43] + [46] Package atveryend Info: Empty hook `AfterLastShipout' on input line 72. (00_main.aux (01_Standard_Model.aux) (02_mu_to_3e_decay.aux) (03_experimental_setup.aux) @@ -2067,7 +2111,10 @@ Package atveryend Info: Executing hook `AtVeryEndDocument' on input line 72. Package atveryend Info: Executing hook `AtEndAfterFileList' on input line 72. Package rerunfilecheck Info: File `00_main.out' has not changed. -(rerunfilecheck) Checksum: 075DAF2E576AE5EEB0BE329E17F60ABB;3780. +(rerunfilecheck) Checksum: F467950E883FD22A1766479B9392E7BD;3897. + + +LaTeX Warning: There were undefined references. LaTeX Warning: There were multiply-defined labels. @@ -2075,16 +2122,19 @@ Package atveryend Info: Empty hook `AtVeryVeryEnd' on input line 72. ) Here is how much of TeX's memory you used: - 20013 strings out of 492973 - 338198 string characters out of 3135932 - 433719 words of memory out of 3000000 - 23429 multiletter control sequences out of 15000+200000 + 20043 strings out of 492973 + 338828 string characters out of 3135932 + 434140 words of memory out of 3000000 + 23448 multiletter control sequences out of 15000+200000 548944 words of font info for 87 fonts, out of 3000000 for 9000 1141 hyphenation exceptions out of 8191 47i,19n,65p,1101b,571s stack positions out of 5000i,500n,10000p,200000b,50000s -pdfTeX warning (dest): name{Hfootnote.28} has been referenced but does not ex +pdfTeX warning (dest): name{Hfootnote.29} has been referenced but does not ex ist, replaced by a fixed one +pdfTeX warning (dest): name{Hfootnote.28} has been referenced but does not exis +t, replaced by a fixed one + pdfTeX warning (dest): name{Hfootnote.27} has been referenced but does not exis t, replaced by a fixed one @@ -2183,9 +2233,9 @@ onts/cm/cmsy6.pfb> -Output written on 00_main.pdf (43 pages, 1683081 bytes). +Output written on 00_main.pdf (46 pages, 1712305 bytes). PDF statistics: - 776 PDF objects out of 1000 (max. 8388607) - 180 named destinations out of 1000 (max. 500000) - 558 words of extra memory for PDF output out of 10000 (max. 10000000) + 814 PDF objects out of 1000 (max. 8388607) + 188 named destinations out of 1000 (max. 500000) + 589 words of extra memory for PDF output out of 10000 (max. 10000000) diff --git a/Report/00_main.out b/Report/00_main.out index 51a2181..25c4530 100644 --- a/Report/00_main.out +++ b/Report/00_main.out @@ -50,5 +50,7 @@ \BOOKMARK [2][-]{subsection.8.2}{RNN classifier with RNN track prediction input}{section.8}% 50 \BOOKMARK [2][-]{subsection.8.3}{XGBoost}{section.8}% 51 \BOOKMARK [2][-]{subsection.8.4}{Comparison in performance of the RNN and XGBoost}{section.8}% 52 -\BOOKMARK [2][-]{subsection.8.5}{Outlook}{section.8}% 53 -\BOOKMARK [1][-]{section.9}{Acknowledgements}{}% 54 +\BOOKMARK [1][-]{section.9}{Results}{}% 53 +\BOOKMARK [2][-]{subsection.9.1}{Results}{section.9}% 54 +\BOOKMARK [2][-]{subsection.9.2}{Outlook and potential}{section.9}% 55 +\BOOKMARK [1][-]{section.10}{Acknowledgements}{}% 56 diff --git a/Report/00_main.pdf b/Report/00_main.pdf index 0e4cd53..8b3de49 100644 --- a/Report/00_main.pdf +++ b/Report/00_main.pdf Binary files differ diff --git a/Report/00_main.synctex.gz b/Report/00_main.synctex.gz index 45d1d42..55d4672 100644 --- a/Report/00_main.synctex.gz +++ b/Report/00_main.synctex.gz Binary files differ diff --git a/Report/00_main.toc b/Report/00_main.toc index b2f7ce8..1cf051c 100644 --- a/Report/00_main.toc +++ b/Report/00_main.toc @@ -49,7 +49,9 @@ \contentsline {section}{\numberline {8}Results}{37}{section.8} \contentsline {subsection}{\numberline {8.1}Best $\chi ^2$}{37}{subsection.8.1} \contentsline {subsection}{\numberline {8.2}RNN classifier with RNN track prediction input}{37}{subsection.8.2} -\contentsline {subsection}{\numberline {8.3}XGBoost}{37}{subsection.8.3} -\contentsline {subsection}{\numberline {8.4}Comparison in performance of the RNN and XGBoost}{39}{subsection.8.4} -\contentsline {subsection}{\numberline {8.5}Outlook}{39}{subsection.8.5} -\contentsline {section}{\numberline {9}Acknowledgements}{40}{section.9} +\contentsline {subsection}{\numberline {8.3}XGBoost}{39}{subsection.8.3} +\contentsline {subsection}{\numberline {8.4}Comparison in performance of the RNN and XGBoost}{41}{subsection.8.4} +\contentsline {section}{\numberline {9}Results}{42}{section.9} +\contentsline {subsection}{\numberline {9.1}Results}{42}{subsection.9.1} +\contentsline {subsection}{\numberline {9.2}Outlook and potential}{42}{subsection.9.2} +\contentsline {section}{\numberline {10}Acknowledgements}{43}{section.10} diff --git a/Report/07_Analysis.aux b/Report/07_Analysis.aux index 714ae85..0cf57a9 100644 --- a/Report/07_Analysis.aux +++ b/Report/07_Analysis.aux @@ -3,43 +3,52 @@ \@writefile{toc}{\contentsline {section}{\numberline {8}Results}{37}{section.8}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.1}Best $\chi ^2$}{37}{subsection.8.1}} \@writefile{toc}{\contentsline {subsection}{\numberline {8.2}RNN classifier with RNN track prediction input}{37}{subsection.8.2}} -\@writefile{toc}{\contentsline {subsection}{\numberline {8.3}XGBoost}{37}{subsection.8.3}} -\newlabel{XGB_tp_fp_hist}{{14a}{38}{Number of false positives and false negatives depending cut\relax }{figure.caption.18}{}} -\newlabel{sub@XGB_tp_fp_hist}{{a}{38}{Number of false positives and false negatives depending cut\relax }{figure.caption.18}{}} -\newlabel{XGB_ROC}{{14b}{38}{ROC curve for the XGBoost model\relax }{figure.caption.18}{}} -\newlabel{sub@XGB_ROC}{{b}{38}{ROC curve for the XGBoost model\relax }{figure.caption.18}{}} +\newlabel{RNN_tp_fp_hist}{{14a}{38}{Number of false positives and false negatives depending cut\relax }{figure.caption.18}{}} +\newlabel{sub@RNN_tp_fp_hist}{{a}{38}{Number of false positives and false negatives depending cut\relax }{figure.caption.18}{}} +\newlabel{RNN_ROC}{{14b}{38}{ROC curve for the RNN model\relax }{figure.caption.18}{}} +\newlabel{sub@RNN_ROC}{{b}{38}{ROC curve for the RNN model\relax }{figure.caption.18}{}} \@writefile{lof}{\contentsline {figure}{\numberline {14}{\ignorespaces XGBoost classifier figures\relax }}{38}{figure.caption.18}} \citation{ML:ROC_AUC:Bradley:1997:UAU:1746432.1746434} +\@writefile{toc}{\contentsline {subsection}{\numberline {8.3}XGBoost}{39}{subsection.8.3}} +\newlabel{XGB_tp_fp_hist}{{15a}{40}{Number of false positives and false negatives depending cut\relax }{figure.caption.19}{}} +\newlabel{sub@XGB_tp_fp_hist}{{a}{40}{Number of false positives and false negatives depending cut\relax }{figure.caption.19}{}} +\newlabel{XGB_ROC}{{15b}{40}{ROC curve for the XGBoost model\relax }{figure.caption.19}{}} +\newlabel{sub@XGB_ROC}{{b}{40}{ROC curve for the XGBoost model\relax }{figure.caption.19}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {15}{\ignorespaces XGBoost classifier figures\relax }}{40}{figure.caption.19}} +\@writefile{toc}{\contentsline {subsection}{\numberline {8.4}Comparison in performance of the RNN and XGBoost}{41}{subsection.8.4}} +\newlabel{RNN-XGB_ROC}{{\caption@xref {RNN-XGB_ROC}{ on input line 64}}{41}{Comparison in performance of the RNN and XGBoost}{figure.caption.20}{}} +\@writefile{lof}{\contentsline {figure}{\numberline {16}{\ignorespaces Comparison ROC curves of RNN and XGBoost model\relax }}{41}{figure.caption.20}} \citation{gent1992special} \citation{graves2013speech} -\@writefile{toc}{\contentsline {subsection}{\numberline {8.4}Comparison in performance of the RNN and XGBoost}{39}{subsection.8.4}} -\@writefile{toc}{\contentsline {subsection}{\numberline {8.5}Outlook}{39}{subsection.8.5}} +\@writefile{toc}{\contentsline {section}{\numberline {9}Results}{42}{section.9}} +\@writefile{toc}{\contentsline {subsection}{\numberline {9.1}Results}{42}{subsection.9.1}} +\@writefile{toc}{\contentsline {subsection}{\numberline {9.2}Outlook and potential}{42}{subsection.9.2}} \@setckpt{07_Analysis}{ -\setcounter{page}{40} +\setcounter{page}{43} \setcounter{equation}{11} \setcounter{enumi}{0} \setcounter{enumii}{0} \setcounter{enumiii}{0} \setcounter{enumiv}{0} -\setcounter{footnote}{28} +\setcounter{footnote}{29} \setcounter{mpfootnote}{0} \setcounter{part}{0} -\setcounter{section}{8} -\setcounter{subsection}{5} +\setcounter{section}{9} +\setcounter{subsection}{2} \setcounter{subsubsection}{0} \setcounter{paragraph}{0} \setcounter{subparagraph}{0} -\setcounter{figure}{14} +\setcounter{figure}{16} \setcounter{table}{3} \setcounter{parentequation}{0} \setcounter{AM@survey}{0} \setcounter{ContinuedFloat}{0} -\setcounter{subfigure}{2} +\setcounter{subfigure}{0} \setcounter{subtable}{0} \setcounter{float@type}{4} \setcounter{Item}{0} -\setcounter{Hfootnote}{28} -\setcounter{bookmark@seq@number}{53} +\setcounter{Hfootnote}{29} +\setcounter{bookmark@seq@number}{55} \setcounter{@stackindex}{1} \setcounter{ROWcellindex@}{0} \setcounter{TABrowindex@}{2} diff --git a/Report/07_Analysis.tex b/Report/07_Analysis.tex index 3b53c4d..e89e200 100644 --- a/Report/07_Analysis.tex +++ b/Report/07_Analysis.tex @@ -8,11 +8,31 @@ \subsection{RNN classifier with RNN track prediction input} -The RNN's that we put in sequence (First track prediction then classification) are a much more complex model. When trained the were able to label all the tracks right with an accuracy of $87\%$. Note that the $75\%$ limit of always choosing one track for every event was exceeded\footnote{Usually the one that is considered the best by the corresponding algorithm}. +The RNN's that we put in sequence (First track prediction then classification) are a much more complex model. When trained they were able to label all the tracks right with an accuracy of around $87.63\%$. Note that the $75\%$ limit of always choosing one track for every event was exceeded\footnote{Usually the one that is considered the best by the corresponding algorithm}.\\ + +\begin{figure}[H] +\begin{center} +\begin{subfigure}{0.8\textwidth} +\includegraphics[width=1\textwidth]{img/RNN_tf-ft_hist.png} +\caption{Number of false positives and false negatives depending cut} +\label{RNN_tp_fp_hist} +\end{subfigure} +\begin{subfigure}{0.8\textwidth} +\includegraphics[width=1\textwidth]{img/RNN_ROC-curve.png} +\caption{ROC curve for the RNN model} +\label{RNN_ROC} +\end{subfigure} +\caption{XGBoost classifier figures} +\end{center} +\end{figure} + +As shown in figure \ref{RNN_tp_fp_hist} depending on where we apply the cut, we have a changing number of false positives and false negatives. In figure \ref{RNN_tp_fp_hist} the blue bins are false positives and the orange bins are false negatives. Depending on what is more important for the experiment\footnote{E.g. all positives have to be correct $\rightarrow$ increase cut}. One can also qualitatively judge the performance here, as in the optimal case all the false positives would gather at the area where the cut goes to $0$. Analogously we want all the false negatives to gather at the cut around $1$. Here we see that this is fulfilled really well. So already by this graph we see that the system will perform well.\\ + +Figure \ref{RNN_ROC} shows the ROC curve \cite{ML:ROC_AUC:Bradley:1997:UAU:1746432.1746434} of the RNN classifier. Generally the more area under the ROC curve the better the classifier. In the perfect case, where everything gets labelled $100\%$ correctly, the area under the curve(ROC AUC) would be $1$ and random guessing would be around $0.5$. Here we have an area of $0.93$. This is already really close to the optimal case. \subsection{XGBoost} -Also an XGBoost classifier was implemented and trained to have some more comparison to the performance of our RNN classification. XGBoost models train much faster than NN and are often a serious competitor to them as often they reach similar performances. The input of XGBoost model was the same as for the RNN classification. The accuracy of this classifier of labelling the tracks was at $80.74\%$ with a cut applied at $0.5$. Note that here we also exceeded the $75\%$ even though with a smaller accuracy than the RNN. +Also an XGBoost classifier\footnote{Depth = 3 and number of estimators =3} was implemented and trained to have some more comparison to the performance of our RNN classification. XGBoost models train much faster than NN and are often a serious competitor to them as often they reach similar performances. Based on that, they are often used as baselines for RNN classifiers and a RNN classifier is considered good if they surpass the XGBoost model. The input of XGBoost model was the same as for the RNN classification. The accuracy of this classifier of labelling the tracks was at $80.74\%$ with a cut applied at $0.5$. Note that here we also exceeded the $75\%$ even though with a smaller accuracy than the RNN. \begin{figure}[H] \begin{center} @@ -30,15 +50,39 @@ \end{center} \end{figure} -As shown in figure \ref{XGB_tp_fp_hist} depending on where we apply the cut, we have a changing number of false positives and false negatives. In figure \ref{XGB_tp_fp_hist} the blue bins are false positives and the orange bins are false negatives. Depending on what is more important for the experiment\footnote{E.g. all positives have to be correct $\rightarrow$ increase cut}\\ +In figure \ref{XGB_tp_fp_hist} the blue bins are false positives and the orange bins are false negatives. Here we see that the bins are more evenly spread and gather less at the edges. So already qualitatively we can guess that it will perform worse than our RNN's.\\ -Figure \ref{XGB_ROC} shows the ROC curve \cite{ML:ROC_AUC:Bradley:1997:UAU:1746432.1746434} of the XGB classifier. Generally the more area under the ROC curve the better the classifier. In the perfect case, where everything gets labelled $100\%$ correctly, the area under the curve would be 1. Here we have an area of $0.88$.\\ +Figure \ref{XGB_ROC} shows the ROC curve of the XGB classifier. Generally the more area under the ROC curve the better the classifier. In the perfect case, where everything gets labelled $100\%$ correctly, the area under the curve would be 1. Here we have an area of $0.88$.\\ \subsection{Comparison in performance of the RNN and XGBoost} -The RNN classifier performs with around $6\%$ better accuracy than the XGBoost classifier. Also by comparing the the ROC curves in figure \textbf{ROC of both}, one can clearly see that the area under the RNN ROC curve is bigger. +The RNN classifier performs with around $6\%$ better accuracy than the XGBoost classifier. Also by comparing the the ROC curves in figure \ref{RNN-XGB_ROC}, one can clearly see that the area under the RNN ROC curve is bigger. In numbers we have around $0.05>$ more area under the curve for the RNN model. The RNN classifier performs significantly better in labelling the 8 hit tracks than the XGBoost model. -\subsection{Outlook} +\begin{figure}[H] +\begin{center} +\includegraphics[width=0.8\textwidth]{img/RNN-XGB_ROC-curve_comparison.png} +\label{RNN-XGB_ROC} +\caption{Comparison ROC curves of RNN and XGBoost model} +\end{center} +\end{figure} +\newpage + +\section{Results} + +\subsection{Results} + +The RNN models perform significantly better at labelling the 8 hit tracks than all other classifiers and methods.\\ + +\begin{tabular}{c | c c} +Model & Accuracy with cut at $0.5$ $[\%]$ & ROC AUC \\ \hline +Best $\chi^2$ & $52.01\%$ & / \\ +XGBoost & $80.74$ & 0.88 \\ +RNN & $87.63\%$ & 0.93 +\end{tabular}\\ + +Using this system of RNN's proves to be viable solution to this problem and brings a huge jump in accuracy also over other machine learning solutions. + +\subsection{Outlook and potential} Where do we want to go from here? One way to improve the algorithm would for example be to create a fully connected neural network \cite{gent1992special}. By doing this both RNN's would be connected and would train as a unit. This would have the positive effect of not having to retrain the classifying RNN as well whenever the first on gets changed. \\ Another goal could be to make this type of RNN appliable to more types of problems. So for example instead of being restricted to tracks of a specific length (here eight hits) one could make it more general to be able to deal with an arbitrary length of the track. This would be especially useful for this experiment, as a lot of particles don't just recurl once but many times over (in the central station). Hereby the are creating a lot of background, which minimalizing is crucial to reach our desired sensitivity of $10^{-16}$.\\ diff --git a/Report/08_Appendix.aux b/Report/08_Appendix.aux index dd7b30c..8eceef3 100644 --- a/Report/08_Appendix.aux +++ b/Report/08_Appendix.aux @@ -1,32 +1,32 @@ \relax \providecommand\hyper@newdestlabel[2]{} -\@writefile{toc}{\contentsline {section}{\numberline {9}Acknowledgements}{40}{section.9}} +\@writefile{toc}{\contentsline {section}{\numberline {10}Acknowledgements}{43}{section.10}} \@setckpt{08_Appendix}{ -\setcounter{page}{41} +\setcounter{page}{44} \setcounter{equation}{11} \setcounter{enumi}{0} \setcounter{enumii}{0} \setcounter{enumiii}{0} \setcounter{enumiv}{0} -\setcounter{footnote}{28} +\setcounter{footnote}{29} \setcounter{mpfootnote}{0} \setcounter{part}{0} -\setcounter{section}{9} +\setcounter{section}{10} \setcounter{subsection}{0} \setcounter{subsubsection}{0} \setcounter{paragraph}{0} \setcounter{subparagraph}{0} -\setcounter{figure}{14} +\setcounter{figure}{16} \setcounter{table}{3} \setcounter{parentequation}{0} \setcounter{AM@survey}{0} \setcounter{ContinuedFloat}{0} -\setcounter{subfigure}{2} +\setcounter{subfigure}{0} \setcounter{subtable}{0} \setcounter{float@type}{4} \setcounter{Item}{0} -\setcounter{Hfootnote}{28} -\setcounter{bookmark@seq@number}{54} +\setcounter{Hfootnote}{29} +\setcounter{bookmark@seq@number}{56} \setcounter{@stackindex}{1} \setcounter{ROWcellindex@}{0} \setcounter{TABrowindex@}{2} diff --git a/Report/img/RNN-XGB_ROC-curve_comparison.png b/Report/img/RNN-XGB_ROC-curve_comparison.png new file mode 100644 index 0000000..0c1ff94 --- /dev/null +++ b/Report/img/RNN-XGB_ROC-curve_comparison.png Binary files differ diff --git a/Report/img/RNN_ROC-curve.png b/Report/img/RNN_ROC-curve.png new file mode 100644 index 0000000..650d331 --- /dev/null +++ b/Report/img/RNN_ROC-curve.png Binary files differ diff --git a/Report/img/RNN_tf-ft_hist.png b/Report/img/RNN_tf-ft_hist.png new file mode 100644 index 0000000..761706b --- /dev/null +++ b/Report/img/RNN_tf-ft_hist.png Binary files differ