text:
\chapter{Sprint 6}\label{chapter:sprint6}
%Common / Overall goal for all Knox groups

\input{sections/Chapter 8 Sprint 6/planning}
\input{sections/Chapter 8 Sprint 6/UI}
\input{sections/Chapter 8 Sprint 6/database_redesign}
\input{sections/Chapter 8 Sprint 6/deploy_rdf_api}
\input{sections/Chapter 8 Sprint 6/Review}
\input{sections/Chapter 8 Sprint 6/retrospective}

ext: tex

max_stars_repo_path: sections/Chapter 8 Sprint 6/Sprint6.tex

max_issues_repo_path: sections/Chapter 8 Sprint 6/Sprint6.tex

max_forks_repo_path: sections/Chapter 8 Sprint 6/Sprint6.tex


text:
\section{Proofs}\label{ssec:proof}

We show three results regarding the bias of the SBW estimator under the classical errors-in-variables model. First, we show that the bias of the SBW estimator that sets $\delta = 0$ (i.e. reweights the treated units to exactly balance the control units) is equal to the bias of the OLS estimator. Second, we show that if the observed covariate values can be replaced by their conditional expectations $\eta_1$ given the noisy observations, then the SBW estimator will be unbiased. Finally, we consider the case where we can estimate $\eta_1$ using auxillary data to estimate the covariance matrix of the error terms and show that the SBW estimator is consistent if we replace $\eta_1$ by an estimate $\hat{\eta_1}$ in the constraint set. In contrast to our application, we take the perspective throughout that $X$ is random, noting that this estimator still has desirable properties even when we view $X$ is fixed (see, e.g., \cite{gleser1992importance}).

Consider a dataset that consists of $i = 1, ..., n$ randomly sampled units where we observe the outcomes $Y_i$ and a treatment assignment indicator $A_i \in \{0, 1\}$. Let $n_a$ be the number of units in each treatment group, i.e, $n_a = \sum_{i = 1}^n \mathbbm{1}(A_i = a)$. Let $X_i \in \mathbb{R}^q$ be a vector of covariate values where $X_i \mid A_i = a \stackrel{iid}\sim MVN(\upsilon_a, \Sigma_{XX})$.

Our target parameter is $\psi^1 = \mathbb{E}\{Y_i^1 \mid A_i = 0\}$.\footnote{Here we switch to the super-population target because we view $X$ as random.} We assume unconfoundedness ($Y_i^a \perp A_i \mid X_i$), consistency ($Y_i^A_i = A_iY_i + (1-A_i)Y_i$), and that $\mathbb{E}\{Y_i \mid X_i, A_i = a\} = \alpha_a + X_i^T\beta_a$. For simplicity we also assume throughout that there exist positive weights $\gamma$ that exactly balance the covariates, i.e. $\sum_{i: A_i = 1}\gamma_iX_{i, r} = \bar{X}_{0, r}$ for all $r = 1, ..., q$ and $\sum_{i: A_i = 1}\gamma_i = 1$, $\gamma_i > 0$.

We begin by establishing the following identity:

\begin{equation}
\psi^1 = \mu_y + (\upsilon_0 - \upsilon_1)^T\beta_1
\end{equation}

where $\mu_y = \mathbb{E}\{Y_i \mid A_i = 1\}$ and $\upsilon_a = \mathbb{E}\{X_i \mid A_i = a\}$.

\begin{proof}

Using our causal and modeling assumptions we have that:

\begin{align*}
\mathbb{E}\{Y_i^1 \mid X_i, A_i = 0\} &= \mathbb{E}\{Y_i^1 \mid X_i, A_i = 1\} \\
&= \mathbb{E}\{Y_i \mid X_i, A_i = 1\} \\
&= \alpha_1 + X_i^T\beta_1 \\
&= \mu_y + (X_i - \upsilon_1)^T\beta \\
&\implies \psi^1 = \mu_y + (\upsilon_0 - \upsilon_1)^T\beta_1
\end{align*}

where the first equality follows from unconfoundedness, the second equality from consistency, the third from our parametric modeling assumptions, and the fourth by definition of $\alpha$. The final implication comes from plugging $\upsilon_0$ in place of $X_i$.

\end{proof}

We now outline the classical errors-in-variables model. Let $Y_i(X_i, a) = \alpha_a + X_i^T\beta_a + \epsilon_i$ where $\epsilon_i \stackrel{iid}\sim N(0, \sigma^2)$. We consider the case where we observe $W_i$, a vector of mean-unbiased proxies for the true (unobserved) covariate vector $X_i$; i.e., $W_i = X_i + v_i$, where $v_i \stackrel{iid}\sim MVN(0, \Sigma_{vv})$. Consider the model:

\begin{equation}
(\epsilon_i, v_i) \sim MVN((0, 0), \begin{pmatrix}
\sigma^2 & 0 \\
0 & \Sigma_{vv}
\end{pmatrix}
\end{equation}

In other words the error in the outcome model is uncorrelated with the error in the covariates. Let $u_i = (\epsilon_i, v_i)$. We further assume that the covariates are uncorrelated with any of the error terms:

\begin{equation}
(X_i, u_i) \mid A_i = a \sim MVN((\upsilon_a, 0), \begin{pmatrix}
\Sigma_{XX} & 0 \\
0 & \Sigma_{uu}
\end{pmatrix}
\end{equation}

Then we see that $(X_i, W_i) \mid A_i = a \stackrel{iid}{\sim} MVN((\upsilon_a, \upsilon_a), \Sigma)$\footnote{In contrast to our application, we instead assume here that $\Sigma_{WW \mid A = 1} = \Sigma_{WW \mid A = 0} = \Sigma_{WW}$ and $\Sigma_{XX \mid A = 1} = \Sigma_{XX \mid A = 0} = \Sigma_{XX}$. This is not a necessary assumption, but helps simplify notation.} where

$$
\Sigma = \begin{pmatrix}
\Sigma_{XX} & \Sigma_{XX} \\
\Sigma_{XX} & \Sigma_{WW}
\end{pmatrix}
$$

and $\Sigma_{WW} = \Sigma_{XX} + \Sigma_{vv}$. Let $\kappa = \Sigma_{WW}^{-1}\Sigma_{XX}$. By the normality of the joint distribution of $X_i$ and $W_i$, we also know that

\begin{equation}
\mathbb{E}\{X_i \mid W_i, A_i = a\} = \upsilon_a + \kappa^T(W_i - \upsilon_a)
\end{equation}

We consider estimating $\psi^1$ in this setting. Let

$$
\hat{\psi}^{1, reg} = \bar{Y}_1 + (\bar{W}_0 - \bar{W}_1)^T\hat{\beta}_1
$$

where $\hat{\beta}_1$ is the OLS estimator of $\beta_1$, $\bar{W}_a = n_a^{-1}\sum_{i:A_i = a} W_i$ and $\bar{Y}_a$ is defined analogously. Let

$$
\hat{\psi}^{1, sbw} = \sum_{i: A_i = a} \gamma_i Y_i
$$

be the SBW estimator, where

\begin{align*}
\gamma &= \arg\min_{\theta} \sum_{i: A_i = 1}\theta_i^2 \text{ such that } W_{1, r}^T\theta = \bar{W}_{0, r} \ \ \ (r = 1, ..., q) \\
&\theta_i > 0, \sum_{i: A_i = 1}\theta_i = 1
\end{align*}

where $W_{a, r}$ is the $n_a$ dimensional vector associated with covariate $r$.

\begin{proposition}\label{cl1}
The bias of $\hat{\psi}^{1, reg}$ is equal to the bias of $\hat{\psi}^{1, sbw}$; specifically,

$$
\mathbb{E}\{\psi^{1, reg} - \psi^1\} = \mathbb{E}\{\psi^{1, sbw} - \psi^1\} = (\upsilon_0 - \upsilon_1)^T(\kappa - I_d)\beta
$$
\end{proposition}

\begin{proof}
Recall that $\mathbb{E}(\hat{\beta}_1) = \kappa\beta_1$ (c.l. \cite{gleser1992importance}). Consider the error of $\hat{\psi}^{1, reg}$:

\begin{align*}
    \hat{\psi}^{1, reg} - \psi^1 &= \bar{Y}_1 + (\bar{W}_0 - \bar{W}_1)^T\hat{\beta}_1 - (\mu_y + (\upsilon_0 - \upsilon_1)^T\beta_1) \\
    &= \underbrace{(\bar{Y}_1 - \mu_y)}_{T_1} + \underbrace{(\bar{W}_0 - \upsilon_0)^T\hat{\beta}_1}_{T_2} - \underbrace{(\bar{W}_1 - \upsilon_1)^T\hat{\beta}_1}_{T_3} + \underbrace{(\upsilon_0 - \upsilon_1)^T(\hat{\beta}_1 - \beta_1)}_{T_4} \\
    \implies \mathbb{E}\{\hat{\psi}^{1, reg} - \psi^1\} &= (\upsilon_0 - \upsilon_1)^T(\kappa - I_d)\beta_1
\end{align*}

The first equality holds by definition and the second by rearranging terms. The second equality consists of four terms. $T_1$ is simply the estimation error from a sample average, which has expectation zero. $T_2$ is the product of the estimation error from a sample average and $\hat{\beta}$; this also has expectation zero because $\bar{X}_0$ is estimated on a different part of the sample than $\hat{\beta}$, so these errors are independent. $T_3$ also has expectation zero because $\bar{W}_1^T\hat{\beta}_1 = \bar{Y}_1$, which has expectation $\mu_y$, and $\upsilon_1^T\hat{\beta}_1$ also has expectation $\mu_y$. We are then left with $T_4$; we substitute $\mathbb{E}{\hat{\beta}} = \kappa\beta$ to get the final result.

We now derive the bias of $\hat{\psi}^{1, sbw}$:

\begin{align*}
    \hat{\psi}^{1, sbw} - \psi^1 &= \sum_{i: A_i = 1}\gamma_iY_i - (\alpha_1 + \upsilon_0^T\beta_1) \\
    &= \sum_{i: A_i = 1} \gamma_i(\alpha_1 + X_i^T\beta_1 + \epsilon_i) - (\alpha_1 + \bar{W}_0^T\beta_1 + (\upsilon_0 - \bar{W}_0)^T\beta_1) \\
    &= \sum_{i: A_i = 1} (\gamma_i(W_i - v_i)^T\beta_1 + \gamma_i\epsilon_i) - \bar{W}_0^T\beta_1 + (\upsilon_0 - \bar{W}_0)^T\beta_1 \\
    &= \underbrace{-\sum_{i: A_i = 1}\gamma_iv_i^T\beta_1}_{T_1} + \underbrace{\sum_{i: A_i = 1}\gamma_i\epsilon_i}_{T_2}  + \underbrace{(\upsilon_0 - \bar{W}_0)^T\beta_1}_{T_3}
\end{align*}

Conditioning on $W_i$, we can take expectations over $X_i$, and see that $T_2$ has expectation zero (noting that the weights, conditional on $W_i$, are independent of these errors). $T_3$ is simply the scaled sum of mean zero estimation error and therefore has expectation zero. We conclude by considering $T_1$ and again take expectations over $X_i$ conditional on $W_i$:

\begin{align*}
    \sum_{i: A_i = 1} \gamma_i\mathbb{E}\{X_i - W_i \mid W_i\}^T\beta_1 &= \sum_{i: A_i = 1} \gamma_i (\upsilon_1 + \kappa^T(W_i - \upsilon_1))^T\beta_1 - \sum_{i: A_i = 1}\gamma_i W_i^T\beta_1 \\
    &= (\upsilon_1 + \kappa^T(\bar{W}_0 - \upsilon_1))^T\beta_1 - \bar{W}_0^T\beta_1 \\
    &= (\kappa^T(\bar{W}_0 - \upsilon_1))^T\beta_1 - (\bar{W}_0 - \upsilon_1)^T\beta_1  \\
    &= (\bar{W}_0 - \upsilon_1)^T(\kappa - I_d)\beta_1 \\
    &= (\upsilon_0 - \upsilon_1)^T(\kappa - I_d)\beta_1 + (\bar{W}_0 - \upsilon_0)^T(\kappa - I_d)\beta_1 \\
    \implies \mathbb{E}\{\hat{\psi}^{1, sbw} - \psi^1\} &= (\upsilon_0 - \upsilon_1)^T(\kappa - I_d)\beta_1
\end{align*}

The final line holds because in the second to last line, we can take expectation over $W_i$ and see that the second term is a scaled sum of mean zero estimation error and has expectation zero.
\end{proof}

Let $\hat{\psi}^{1, sbw}(\eta_1)$ be the SBW estimator that reweights $\eta_1(W_i)$ rather than $W_i$; i.e. $\hat{\psi}^{sbw}(1, \eta_1) = \sum_{i: A_i = 1}\gamma_i^\star Y_i$ where

\begin{align*}
\gamma^\star = &\arg\min_{\theta} \sum_{i: A_i = 1}\theta_i^2 \text{ such that } \eta_1(W_{1, r})^T\theta = \upsilon_{0, r} \ \ \ (r = 1, ..., q) \\
&\theta_i > 0, \sum_{i: A_i = 1} \theta_i = 1
\end{align*}

\begin{proposition}
The estimator $\hat{\psi}^{1, sbw}(\eta_1)$ is unbiased, i.e.
$\mathbb{E}\{\hat{\psi}^{1, sbw}(\eta_1)\} = \psi^1$
\end{proposition}

\begin{proof}

By linearity we know that

\begin{align*}
Y_i^1 = \alpha_1 + \eta_a(W_i)^T\beta_1 + (X_i - \eta_a(W_i))^T\beta_1
\end{align*}

We then have that:

\begin{align*}
    \hat{\psi}^{1, sbw}(\eta_1) - \psi^1 &= \sum_{i: A_i = 1}\gamma_i^\star Y_i - (\alpha_1 + \upsilon_0^T\beta_1) \\
    &= \sum_{i: A_i = 1}\gamma_i^\star\alpha_1 + \sum_{i: A_i = 1}\gamma_i^\star\eta_1(W_i)^T\beta_1 + \sum_{i: A_i = 1}\gamma_i^\star(X_i - \eta_1(W_i))^T\beta_1 - (\alpha + \upsilon_0^T\beta_1) \\
    &= \sum_{i: A_i = 1}\gamma_i^\star(X_i - \eta_1(W_i))^T\beta_1
\end{align*}

Conditional on $W$, the weights are fixed and $X_i - \eta_1(W_i)$ has expectation zero; therefore, the estimator is unbiased.

\end{proof}
\begin{remark}

While we have assumed no model error, this estimator still has variance, conditional on $W_i$, equal to

$$
\sum_{i: A_i = 1} \gamma_i^{\star^2}\beta_1^T Cov(X_i \mid W_i)\beta_1
$$

where, assuming that $(X_i, W_i)$ are jointly normally distributed, $Cov(X_i \mid W_i) = \Sigma_{XX} - \Sigma_{XX}\Sigma_{WW}^{-1}\Sigma_{XX}$. Therefore, the variance of this estimator is higher than if we knew the true $X_i$ unless $\Sigma_{WW} = \Sigma_{XX}$ (i.e. we observe $X_i$).

\end{remark}

\begin{remark}
Assuming there is a model error simply leads to the additional term $\sum_{i: A_i = 1}\gamma_i\epsilon_i$. This again has expectation zero, because the weights remain independent of the error in the outcome model, and adds a term to the total variance (conditional on $W_i$) equal to

$$
\sigma^2\sum_{i: A_i = 1}\gamma_i^{\star^2}
$$
\end{remark}

\begin{remark}
If we view both $W$ and $X$ as fixed and substitute $\bar{X}_a$ for $\upsilon_a$ (again assuming this quantity is known), the remaining term contributes to a finite-sample bias rather than variance. Moreover, the squared-bias term (equivalent to the MSE when $\sigma^2 = 0$) will decrease with the variance of the weights.
\end{remark}

Proposition 2 assumes that we know $\eta_1$ and $\upsilon_a$; however, in practice we estimate it from the data. Moreover, the estimation of $\hat{\eta}_1$ typically involves both the observed dataset and auxillary data, which we have not yet specified here. We now consider a simple version of this case.

Recall that $\eta_1(W_i) = \upsilon_1 + \kappa^T(W_i - \upsilon_1)$, where $\kappa = \Sigma_{WW}^{-1}\Sigma_{XX}$. We can easily estimate $\upsilon_a$ consistently using $\bar{W}_a$; the challenge is estimating $\kappa$.

Following \cite{gleser1992importance}, consider the setting where we observe $T$ independent vectors of measurements $W_i^\star$ from known values $X_i^\star$, so that $v_i^\star = W_i^\star - X_i^\star$. Assume that $\Sigma_{v^\star v^\star} = \Sigma_{vv}$. We can then estimate $\hat{\Sigma}_{vv} = \frac{1}{T}\sum_{i=1}^T(W_i^\star - X_i^\star)'(W_i^\star - X_i^\star)$. We can then estimate

$$
\hat{\kappa} = (n\hat{\Sigma}_{WW})^{-1}(n(\hat{\Sigma}_{WW} - \hat{\Sigma}_{vv}))
$$

assuming $n(\hat{\Sigma}_{WW} - \hat{\Sigma}_{vv})$ is positive semi-definite, and where $\hat{\Sigma}_{WW} = \frac{1}{n}\sum_{i=1}^n (W_i - \bar{W})(W_i - \bar{W})'$.

\begin{proposition}
Let $\tilde{\gamma}$ be the weights that solve the SBW objective with $\sum_{i: A_i = 1}\tilde{\gamma}_i\hat{\eta}_1(W_{i, r}) = \bar{W}_{0, r}$ for all $r = 1, ..., q$. The estimator $\sum_{i: A_i = 1}\tilde{\gamma}_iY_i \to \psi^1$ as $n \to \infty$, $T \to \infty$.
\end{proposition}

\begin{proof}

Rewriting the error expression from Lemma A.2 leads to the additional terms $\sum_{i: A_i = 1}\tilde{\gamma}_i(\eta_1(W_i) - \hat{\eta_1}(W_i))^T\beta_1$ and $(\upsilon_0 - \bar{W}_0)^T\beta_1$. The second term is scaled estimation error and has expectation zero. It therefore suffices to show that

$$
\sum_{i: A_i = 1}\tilde{\gamma}_i(\eta_1(W_i) - \hat{\eta_1}(W_i))^T\beta_1 \to 0
$$

By the weak law of large numbers, $\hat{\Sigma}_{WW} - \hat{\Sigma}_{vv} \to \Sigma_{XX}$ as $n \to \infty$, $T \to \infty$; similarly $\hat{\Sigma}_{WW} \to \Sigma_{WW}$ as $n \to \infty$. By the continuous mapping theorem $\hat{\kappa} \to \kappa$. Since $\bar{W}_0 \to \upsilon_0$, we have that $\hat{\eta_1} \to \eta_1$.

Let $\gamma^\star$ be the weights defined in Lemma A.3. We can then rewrite the error term above as

\begin{align*}
\sum_{i: A_i = 1}(\tilde{\gamma}_i - \gamma_i^\star)(\hat{\eta}_1(W_i) - \eta(W_i))^T\beta_1 + \sum_{i: A_i = 1}\gamma_i^\star(\hat{\eta}_1(W_i) - \eta_1(W_i))^T\beta_1
\end{align*}

As $n \to \infty$, $T \to \infty$, $\hat{\eta}_1 \to \eta_1$, and therefore by definition of the SBW objective, $\tilde{\gamma}_i \to \gamma_i^\star$. These two terms therefore both converge in probability to zero.

\end{proof}

Our estimation of $\eta_1$ follows a somewhat different procedure which we outline in Appendix B.

\subsection{H-SBW Objective}

We show that the H-SBW estimator minimizes the variance of any estimator within the constraint set given a constant within-group correlation structure. Here we assume that the true covariates are measured. Consider the outcome model:

\begin{equation}
    Y_{sc}(X_{sc}, a) = \alpha_a + X_{sc}^T\beta_a + c_s + \epsilon_{sc}
\end{equation}

where $\mathbb{E}\{c_s \mid X_{sc}, A_{sc}\} = \mathbb{E}\{\epsilon_{sc} \mid X_{sc}, A_{sc}\} = \mathbb{E}\{c_s\epsilon_{sc} \mid X_{sc}, A_{sc}\} = 0$, and $Var(\epsilon_{sc}) = \sigma^2$ and $Var(c_s) = \eta^2$. Let $\rho = \frac{\eta^2}{\sigma^2 + \eta^2}$.

The conditional variance of any weighting estimator $\sum_{sc: A_{sc} = 1}\gamma_{sc}Y_{sc}$ using weights $\gamma$ where $\gamma^T1 = n_t$ is then equal to:

\begin{align}
    Var(n_t^{-1}\sum_{sc: A_s = 1}\gamma_{sc}Y_{sc} \mid X_{sc}, A_{sc}) &= n_t^{-2}\sum_{s: A_s = 1}^{m_1}(\sum_{c = 1}^{p_s}\gamma_{sc}^2(\sigma^2 + \eta^2) + \sum_{c \ne d}\gamma_{sc}\gamma_{sd}\eta^2) \\
    &\propto \sum_{s: A_s = 1}(\sum_{c = 1}^{p_s}(\gamma_{sc}^2 + \sum_{c \ne d}\rho \gamma_{sc}\gamma_{sd})
\end{align}

where the second line follows by dividing by $\eta^2 + \sigma^2$. This suggests the H-SBW objective, which minimizes the (conditional) variance of this estimator for known $\rho$.


text:
\documentclass[a4paper,twocolumn]{article}

\usepackage[english]{babel}
\usepackage[utf8]{inputenc}
\usepackage{graphicx}
\usepackage{fullpage}

\usepackage[small]{titlesec}
% patch titlesec bug of not showing (sub)title numbering http://tex.stackexchange.com/a/300259
\usepackage{etoolbox}
\makeatletter
\patchcmd{\ttlh@hang}{\parindent\z@}{\parindent\z@\leavevmode}{}{}
\patchcmd{\ttlh@hang}{\noindent}{}{}{}
\makeatother

\title{Distilling the Knowledge in a Neural Network $-$ summary}
\author{Matěj Nikl}

\begin{document}
\maketitle
\noindent
A process called \textit{distillation} is a kind of learning that can be used to transfer a knowledge from a cumbersome model (or an ensemble of models) to a smaller/single model, which is more suitable for deployment, while preserving the knowledge and thus preserving the performance.

To do so, the standard back-propagation algorithm (with a different objective function) is used.

\section{Softmax with temperature}
Neural networks typically produce class probabilities by using a \textit{softmax} output layer that converts the logit, $z_i$, computed for each class into a probability, $q_i$, by comparing $z_i$ with the other logits:
\[
    q_i = \frac{e^{z_i/T}}{\sum_j e^{z_j/T}}
\]
where a temperature $T$ is normally set to 1. Using a higher value for $T$ produces a softer probability distribution over classes.

\begin{figure}[!h]
    \includegraphics[width=\columnwidth]{softmax_example.eps}
    \caption{Comparison of softmax outputs $q$ given inputs $z$ using different temperatures}
\end{figure}

\section{Distillation}

A side-effect of learning is that a trained model assigns probabilities to all of the incorrect answers and even when these probabilities are very small, some of them are much larger than others $-$ the relative probabilities of incorrect answers tell us a lot about how the model tends to generalize. An image of a car, for example, may only have a very small chance of being mistaken for a truck, but that mistake is still much more probable than mistaking it for an apple.

By using all the class probabilities produced by a cumbersome model as \textit{soft targets} for a small model, the knowledge can be transferred.

For this transfer stage, the same training set or even a different \textit{transfer} set can be used.

For an ensemble of models, an arithmetic or geometric mean of their individual predictive distributions as the soft targets can be used.

Moreover, when the soft targets have high entropy, they provide much more information per training case than hard targets and much less variance in the gradient between training cases, so the small model can often be trained on much less data and using a much higher learning rate.

\begin{table*}[t]
    \center
    \begin{tabular}{| l | c | c |}
        \hline
        System                 & Test Frame Accuracy & WER    \\ \hline
        Baseline               & 58.9\%              & 10.9\% \\
        10x Ensemble           & 61.1\%              & 10.7\% \\
        Distilled single model & 60.8\%              & 10.7\% \\
        \hline
    \end{tabular}
    \caption{Frame classification accuracy and Word Error Rate (WER) of a speech recognition DNN showing that the distilled single model performs about as well as the averaged prediction of 10 models that were used to create the soft targets.}
\end{table*}

\begin{table*}[t]
    \center
    \begin{tabular}{| l | c | c |}
        \hline
        System \& training set             & Train Frame Accuracy & Test Frame Accuracy \\ \hline
        Baseline (100\% of training set)   & 63.4\%               & 58.9\%              \\
        Baseline (3\% of training set)     & 67.3\%               & 44.5\%              \\
        Soft Targets (3\% of training set) & 65.4\%               & 57.0\%              \\
        \hline
    \end{tabular}
    \caption{Soft targets allow a new model to generalize well from only 3\% of the training set. The soft targets are obtained by training on the full training set.}
\end{table*}

\subsection{Computing the gradient}
Using the Kullback-Leibler divergence (cross-entropy) as a loss function:
\[
    C = D_{\mathrm{KL}}(p\|q) = - \sum_i p_i \log(q_i)
\]
Each case in the transfer set contributes a cross-entropy gradient, $dC/dz_i$, with respect to each logit, $z_i$, of the distilled model. If the cumbersome model has logits $v_i$, which produce soft target probabilities $p_i$ and the transfer training is done at a temperature of $T$, the gradient is given by:
\[
    \frac{\partial C}{\partial z_i} = \frac{1}{T}(q_i - p_i) = \frac{1}{T}\left( \frac{e^{z_i/T}}{\sum_j e^{z_j/T}} - \frac{e^{v_i/T}}{\sum_j e^{v_j/T}} \right)
\]
If the temperature is high compared with the magnitude of the logits, we can approximate ($e^x \approx 1+x$ for $x \approx 0$):
\[
    \frac{\partial C}{\partial z_i} \approx \frac{1}{T}\left( \frac{1 + z_i/T}{N + \sum_j z_j/T} - \frac{1 + v_i/T}{N + \sum_j v_j/T} \right)
\]
If we now assume that the logits have been zero-meaned separately for each transfer case so that $\sum_j z_j = \sum_j v_j = 0$, we can further simplify to:
\[
    \frac{\partial C}{\partial z_i} \approx \frac{1}{NT^2} (z_i - v_i)
\]
So in the high temperature limit, distillation is equivalent to minimizing the L2 distance (MSE) between the soft target probabilities and the distilled model probabilities.

\subsection{Combining the objective functions}
When the correct labels are known for all or at least some of the transfer set, this method can be significantly improved by also training the distilled model to produce the correct labels. The best way to do it is to use a weighted average of the above defined objective function of cross-entropy with the soft targets and the objective function of cross-entropy with the correct labels (the hard targets). The best results are generally obtained by using a \textit{considerably} lower weight on the second objective function.

\section{Ensembles of specialists on very big datasets}
Even though training an ensemble of models is a very simple way to take advantage of parallel computation, and distillation can help us at the test time, the amount of computation required at training time for a very large dataset is excessive.

The proposed improvement for this situation is, instead of training an ensemble of standalone models, to train \textit{specialist} models that can help with distinguishing between highly confusable classes, while improving the overall performance.

Each specialist model starts with weights initialized as the main \textit{generalist} model and focuses solely on its own subset of (for the generalist model) highly confusable classes, and thus learns much faster.

The ultimate goal of this idea was to distill the knowledge from the specialist models back into the generalist model, which the researchers haven't been able to accomplish yet.

\end{document}


text:
Tensegrities share many of the design, fabrication, modeling, sensing,
and control challenges of the broader category of soft robots
\cite{Pfeifer:2012aa, Kim:2013aa, Majidi:2014aa},
which are made out of intrinsically soft and/or extensible
materials.

\subsection{Opportunities in Morphological Computation}

%~\cite{2917079}

Both tensegrity and soft robots closely relate to the notion of
embodied intelligence, where morphology and materials can take over
some of the functions normally attributed to control to achieve a
system that is overall simpler, more robust and adaptive than those
based on the classical control paradigm. This principle
is known as \emph{morphological computation}~\cite{Pfeifer:2012aa,
Hauser}.  Many approaches to morphological
computation \cite{Zambrano:2014aa, Hauser} seek to reduce the
complexity of control systems through intelligent mechanism designs,
which effectively exhibit complex behaviors while reducing the use of
explicit control systems.  An example would be a compliant, soft hand
that naturally grasps a wide range of object shapes while executing
the same simple control law in all cases~\cite{Deimel:2015aa}.  These
systems may reduce the amount of sensing, actuation, or explicit
modeling and decision making associated with traditional approaches to
the task.

Tensegrity robots show this quality when they passively conform to the
environment and re-balance forces throughout their
structure \cite{Khazanov:2014aa}.  This is a very desirable property,
which enables the design of robots capable of a wide range of tasks
and activities. Such platforms are generally more versatile and robust
in the face of noise and the unpredictability of operating in messy
real-world environments.

%A well designed morphology can lead to drastic reductions in control
%requirements, as well as improved controllability. On the other hand,
%a poorly designed morphology may lead to low controllability, require
%complex control algorithms, or in the worse case, simply be inadequate
%for the task.

\begin{figure*}[t]
\includegraphics[width = \textwidth]{tex/img/review_overview_fig}
%\vspace{-.2in}
\caption{Overview of research topics discussed. An arrow indicates a topic that can contribute to the
development of another.  }
%\vspace{-.2in}
\label{fig:overview}
\end{figure*}

\subsection{Methods for Controlling Soft Robots}

Soft materials can bend, twist, stretch, compress, buckle, wrinkle and
so on. Such motions may involve an infinite number of Degrees of Freedom (DoF) indicating
that the control if soft robots requires new approaches
\cite{Rus:2015aa, Trimmer:2014aa}. While some progress can
be made with more traditional control approaches, such as Model
Predictive Control (MPC), these efforts typically require
accurate models of the robot and environment, which can be difficult
to acquire given the complex physical properties of soft robots.
Thus, many efforts focus specifically on biomimetic
systems \cite{Kim:2013aa}, which aim to reproduce the control
behaviors of their biological counterparts and often provide a new
understanding of soft organisms~\cite{Lin:2011aa}.

%This is similar to developments in tensegrity
%structures \cite{skelton_tensegrity_2009}.

To better understand the challenges in controlling soft and tensegrity
based robots, new static, kinematic and dynamic models have been
developed to capture the ability of bending and flexing
\cite{Saunders:2011aa, skelton_tensegrity_2009}.  For many
years, a popular abstraction for soft robots has been that of
piece-wise constant curvature ({\tt PCC})
\cite{Webster:2010aa}, which does not
capture all aspects of real mechanisms. Recently some non-constant
curvature models have been proposed to better model soft mechanisms.
\cite{Renda:2014aa}.  The need for
expressive models has also led to the development of simulation tools
targeted to soft robots \cite{Germann:2013aa}. This is also an
important development in tensegrity
robotics \cite{Caluwaerts2013rsif}, in which open source physics based
simulation tools have recently become
available \cite{SunSpiralSoftware}.  Table~\ref{tbl:resources} gives
pointers to simulation tools and analytical models for tensegrity
structures.

There have been both model-based and model-free approaches for the
low-level control of soft robots \cite{Rigatos:2009aa}. On the
model-based side, a recent effort utilizes finite
elements \cite{Largilliere:2015aa}, while a recent data-driven,
model-free approaches utilizes graph-theory \cite{Vikas:2015aa}.
There is no consensus, however, yet regarding the appropriate
methodology for control, and especially planning, for soft robots
given their highly continuous, complex and intrinsically compliant
deformation \cite{Rus:2015aa}.  This has motivated efforts in
proposing behavior-based control architectures for soft robots that
may be applicable to tensegrities as well \cite{Armbrust:2015aa}.  A
critical challenge in achieving deliberative control and planning,
shared between tensegrity structures and soft robots, is the
difficulty of solving the inverse kinematics problem. There are
solutions in certain setups, such as for semi-soft
manipulators \cite{Neppalli:2009aa}.

% or under the {\tt PCC} assumption \cite{Marchese:2014aa}.

\subsection{Planning for Tensegrities}

Similar to soft robots, the very properties that make tensegrities
ideal for physical interaction with the environment, such as
compliance, and multi-path load distribution, present some significant
challenges to traditional planning approaches and lead to uncertainty
during motion execution.

For instance, compliance allows tensegrity robots to adapt their shape
to workspaces of unknown or uncertain geometry. But when a force is
applied, the robot can deform in a non-linear manner and will often
excite oscillatory behaviors \cite{SunSpiral2013Tensegrity-Base}.  The
results of contacts are therefore very hard to predict to the level of
accuracy required for traditional trajectory and route planning
techniques.  These issues have limited investigation of planning
algorithms and focused most existing efforts on controllers for the
generation of local gaits \cite{Paul2006a} and quasi-static
paths \cite{Porta:2015aa}. Yet, they have also inspired the
development of approaches that go beyond the traditional control
toolkit and allow adaptation to multiple different terrain
types~\cite{MirletzSoftRobotics,burms2015online}.  Similar
developments in soft robotics technologies are taking place, which
address the complications of self-loading and non-linear compliance
\cite{Rus:2015aa}.  Recently, planning methods have been introduced
which begin to address these needs for soft
robots \cite{Bonilla:2015aa, Marchese:2016aa}.

To move this field forward for both tensegrity and soft robots, both
high-level approaches of model-based and model-free planning should be
investigated as highlighted in Fig.~\ref{fig:overview}.  Model-based
planning approaches should be able to reason over more complex
dynamical models of tensegrity systems and provide robust trajectories
over probabilistic state representations that capture the diversity of
possible executions given the inherent uncertainty. Another direction
is to study feedback-based motion planners that provide a robust
composition of controllers with performance guarantees. In the
model-free domain, methods should capture the dynamics of low-level
controllers and then plan over the resulting dynamics. The low-level
controllers should manage the details of environmental interaction
while successfully driving the system to the next waypoint.

%The rest of this paper will aim to review work in low-level control
%and planning for tensegrity robots and outline directions for further
%research in this domain given developments in the motion planning and
%robot learning fields.

%Many advances have been made in understanding how to control robots
%with these physical properties, but those very same desirable
%properties introduce significant challenges to long-range planning
%algorithms.  Specifically, the adaptability of the robots and their
%ability to behave in complex ways which are outside the scope of the
%control system makes long range planning difficult because there is
%much greater uncertainty about the evolution of the robot's state as
%it executes the desired plan.

\begin{table}[b]
%\vspace{-.2in}
\centering
\begin{tabular}{r|r}
\textbf{resource} & \textbf{references} \\ \hline
\rowcolor[HTML]{BBDAFF} dynamics models   	& \cite{skelton_tensegrity_2009,hernandez2009reconfigurable,MiratsTur2009,sultan2002,Wroldsen2006A-Discussion-on}\\
kinematics \& statics
& \cite{C.R.1978,6907473,skelton_tensegrity_2009, 2003Tensegrity:-Str,Guest2010,Juan2008,pellegrino2014deployable}           \\
\rowcolor[HTML]{BBDAFF} simulation        	& \cite{SunSpiralSoftware,MirletzSoftRobotics,skelton_tensegrity_2009,Caluwaerts2013rsif,Rovira2009Control-and-Sim} \\
hardware design
                        & \cite{Bliss2013Central-Pattern,bohm2013vibration,Fest2004,Shibata2009,Caluwaerts2013rsif,miratstur2011athree-dof,kimrobust,Khazanov:2014aa,ChanMarch2004} \\
                        & \cite{Koizumi2012b,Paul2006a,Rieffel2009,Mirletz2015,sabelhaus2015system,bruce2014design,veuve2015deployment} \\
\rowcolor[HTML]{BBDAFF} state estimation        & \cite{Caluwaerts:2015aa}
\end{tabular}
\caption{\vspace{-.05in}Resources for tensegrity control.}
%\vspace{-.4in}
\label{tbl:resources}
\end{table}

%dos2015design,fest2003adjustable


text:
\subsection{Introduction}

\subsection{subgroup analysis}

\subsection{interaction terms}

\subsection{efficient policy learning}

\subsection{Het DML}
\(y=a(z)+d b(z)\)
Het effects is \(b(z)\)
We build groups instead of arbitrary function. So we estimate \(E[b(z)|G]\)

Use part of the data set to estimate

\(\hat y =\hat a (z)+D\hat b(z)\)

Use \(s=\hat b(z)\) to stratify. Key point is defining subgroups algorithmically. Less opportunity for hacking

\subsection{Continuous treatment effects}

\subsection{Intent-to-treat}

max_stars_repo_name: adamdboult/nodeHomePage

max_issues_repo_name: adamdboult/nodeHomePage

max_forks_repo_name: adamdboult/nodeHomePage


text:
%!TEX root = lucene4IR2016workshop_report.tex
\section{Keynotes and Invited Talks}
%<<<<<<< HEAD
%During the course of the workshops a series of talks were given on how Lucene is being used in industry, teaching and for evaluation along with more technical talks on the inner workings of Lucene's scoring algorithm as well as how learning to rank is being included into Solr\footnote{\scriptsize{Slides are available from \url{www.github.com/leifos/lucene4ir}}}.
%
%\subsection*{Introduction Talk: Why are we here?}
%{\bf Leif Azzopardi, University of Strathclyde}:
%Leif explained how after attending the lively Reproducibility workshop~\cite{arguello2016repro} at ACM SIGIR 2015, he wondered where the Lucene team was, and if Lucene and the community is so big, why they don't come to IR conferences - he posited that perhaps we haven't been inclusive and welcoming as we could to such a large community of search practitioners. He further asserted that this has lead to a failure in transferring our knowledge into one of the largest open source toolkits available. He argued that if we as academics want to increase our impact then we need to improve how we transfer our knowledge to industry. One way is working with large search engines, but what about other industries and organisations that need search and use toolkits like Lucene? He argued that we need to start speaking the same language i.e. work with Lucene et al. and look for opportunities on how we can contribute and develop resources for training and teaching IR and how to undertake evaluations and data science using widely used, supported and commonly accepted Open Source toolkits. He described how this workshop was a good starting point and opportunity to explore how academia and industry can better work together, where we can identify common goals, needs and resources that are needed to foster this relationship.
%=======
During the course of the workshops a series of talks on how Lucene is being used in Industry, Teaching and for Evaluation along with more technical talks on the inner workings of how Lucene's scoring algorithm works and how learning to rank is being included into Solr, were presented\footnote{\scriptsize{Slides are available from \url{www.github.com/leifos/lucene4ir}}}. A summary of each talk is below.

\subsection*{Introduction Talk: Why are we here?}
{\bf Leif Azzopardi, University of Strathclyde}:
Leif explained how after attending the lively Reproducibility workshop~\cite{arguello2016repro} at ACM SIGIR 2015, he wondered where the Lucene team was, and why, if Lucene and the community is so big, why they don't come to IR conferences - he posited that perhaps we haven't been very inclusive or welcoming to such a large community of search practitioners. He further asserted that this has reduced our capacity to transfer our knowledge and experience into one of the largest Open Source toolkits available. He argued that if we as academics want to increase our impact then we need to improve how we transfer our knowledge to industry. One way is working with large search engines, but what about other industries and organisations that need search and use toolkits like Lucene? He argued that we need to start speaking the same language i.e. work with Lucene et al and look for opportunities on how we can contribute and develop resources for training and teaching IR and how to undertake evaluations and data science using widely used, supported and commonly accepted Open Source toolkits. He described how this workshop was a good starting point and opportunity to explore how academia and industry can better work together, where we can identify common goals, needs and resources that are needed to foster this relationship.
%>>>>>>> f0e54b9adc801bc5760e93a856bf677df92c95d5

\input{charlie}

\input{juanma}

\input{rup}

\input{jake}

\input{diego}


text:
\subsection*{MORR.Core.Data.Transcoding}

\begin{interface}{IEncoder}
    \clsdiagram{resources/Classes/Core/MORR/Data/Transcoding/IEncoder.png}

    \clsdcl{public interface IEncoder}

    \clsdsp{Encodes the provided samples into a file at a path.}

    \begin{attributes}
        \attribute{ManualResetEvent EncodeFinished \{ get; \}}{An event raised when encoding finishes.}
    \end{attributes}

    \begin{methods}
        \begin{method}{void Encode(DirectoryPath recordingDirectoryPath)}{Encodes all provided samples to a recording at the specified location.}
            \begin{parameters}
                \para{DirectPath recordingDirectoryPath}{The DirectoryPath to contain the recording.}
            \end{parameters}
            \begin{exceptions}
                \excp{EncodingException}{Exception thrown if the encoding fails.}
            \end{exceptions}
        \end{method}
    \end{methods}
\end{interface}

\begin{interface}{IDecoder}
    \clsdiagram{resources/Classes/Core/MORR/Data/Transcoding/IDecoder.png}

    \clsdcl{public interface IDecoder}

    \clsdsp{Decodes samples stored in a file at a path and provides the decoded samples.}

    \begin{attributes}
        \attribute{ManualResetEvent DecodeFinished \{ get; \}}{An event raised when decoding finishes.}
    \end{attributes}

    \begin{methods}
        \begin{method}{void Decode(DirectoryPath recordingDirectoryPath)}{Decodes samples contained in a recording at a specified location.}
            \begin{parameters}
                \para{DirectoryPath recordingDirectoryPath}{The DirectoryPath of the file to decode from.}
            \end{parameters}
            \begin{exceptions}
                \excp{DecodingException}{Exception thrown if decoding fails.}
            \end{exceptions}
        \end{method}
    \end{methods}
\end{interface}

\subsection*{MORR.Core.Data.Transcoding.Exceptions}

\begin{class}{EncodingException}
    \clsdiagram{resources/Classes/Core/MORR/Data/Transcoding/Exceptions/EncodingException.png}

    \clsdcl{public class EncodingException : Exception}

    \clsdsp{A generic encoding exception that all specialized encoding exceptions derive from.}

    \begin{constructors}
        \begin{constructor}{public EncodingException()}{Creates a new instance of a EncodingException without any specific error message.}
        \end{constructor}
        \begin{constructor}{public EncodingException(string message)}{Creates a new instance of a EncodingException with the specified error message.}
            \begin{parameters}
                \para{string message}{The error message to construct the instance with.}
            \end{parameters}
        \end{constructor}
        \begin{constructor}{public EncodingException(string message, Exception innerException)}{Creates a new instance of a EncodingException with the specified error message and inner exception.}
            \begin{parameters}
                \para{string message}{The error message to construct the instance with.}
                \para{Exception innerException}{The inner exception to construct the instance with.}
            \end{parameters}
        \end{constructor}
    \end{constructors}
\end{class}

\begin{class}{DecodingException}
    \clsdiagram{resources/Classes/Core/MORR/Data/Transcoding/Exceptions/DecodingException.png}

    \clsdcl{public class DecodingException : Exception}

    \clsdsp{A generic decoding exception that all specialized decoding exceptions derive from.}

    \begin{constructors}
        \begin{constructor}{public DecodingException()}{Creates a new instance of a DecodingException without any specific error message.}
        \end{constructor}
        \begin{constructor}{public DecodingException(string message)}{Creates a new instance of a DecodingException with the specified error message.}
            \begin{parameters}
                \para{string message}{The error message to construct the instance with.}
            \end{parameters}
        \end{constructor}
        \begin{constructor}{public DecodingException(string message, Exception innerException)}{Creates a new instance of a DecodingException with the specified error message and inner exception.}
            \begin{parameters}
                \para{string message}{The error message to construct the instance with.}
                \para{Exception innerException}{The inner exception to construct the instance with.}
            \end{parameters}
        \end{constructor}
    \end{constructors}
\end{class}

\subsection*{MORR.Core.Data.Transcoding.Json}

\begin{class}{JsonDecoder}
    \clsdiagram[width=\textwidth]{resources/Classes/Core/MORR/Data/Transcoding/Json/JsonDecoder.png}

    \clsdcl{public class JsonDecoder : \linebreak DefaultDecodeableEventQueue<JsonIntermediateFormatSample>, IDecoder}

    \clsdsp{Decodes samples from a recording previously created with the JsonEncoder.}

    \begin{constructors}
		\begin{constructor}{public JsonDecoder())}{Creates a new instance of the JsonDecoder}
		\end{constructor}
		\begin{constructor}{public JsonDecoder(IFileSystem fileSystem))}{Creates a new instance of the JsonDecoder with specified FileSystem}
			\begin{parameters}
				\para{IFileSystem fileSystem}{The file system to construct the instance with.}
			\end{parameters}
		\end{constructor}
	\end{constructors}

    \begin{attributes}
        \attribute{public ManualResetEvent DecodeFinished \{ get; \}}{An event raised when decoding finishes.}
    \end{attributes}

    \begin{methods}
        \begin{method}{void Decode(DirectoryPath recordingDirectoryPath)}{Decodes samples contained in a recording at a specified location.}
            \begin{parameters}
                \para{DirectoryPath recordingDirectoryPath}{The DirectoryPath of the file to decode from.}
            \end{parameters}
            \begin{exceptions}
                \excp{DecodingException}{Exception thrown if decoding fails.}
            \end{exceptions}
        \end{method}
    \end{methods}
\end{class}

\begin{class}{JsonEncoder}
    \clsdiagram{resources/Classes/Core/MORR/Data/Transcoding/Json/JsonEncoder.png}

    \clsdcl{public class JsonEncoder : IEncoder}

    \clsdsp{Encodes samples into a recording in JSON format.}

    \begin{attributes}
        \attribute{public ManualResetEvent EncodeFinished \{ get; \}}{An event raised when encoding finishes.}
    \end{attributes}

     \begin{constructors}
		\begin{constructor}{public JsonEncoder())}{Creates a new instance of the JsonEncoder}
		\end{constructor}
		\begin{constructor}{public JsonEncoder(IFileSystem fileSystem))}{Creates a new instance of the JsonEncoder with specified FileSystem}
			\begin{parameters}
				\para{IFileSystem fileSystem}{The file system to construct the instance with.}
			\end{parameters}
		\end{constructor}
	\end{constructors}

    \begin{methods}
        \begin{method}{void Encode(DirectoryPath recordingDirectoryPath)}{Encodes all provided samples to a recording at the specified location.}
            \begin{parameters}
                \para{DirectoryPath recordingDirectoryPath}{The DirectoryPath of the file to encode to.}
            \end{parameters}
            \begin{exceptions}
                \excp{EncodingException}{Exception thrown if encoding fails.}
            \end{exceptions}
        \end{method}
    \end{methods}
\end{class}

\begin{class}{JsonDecoderConfiguration}
    \clsdiagram{resources/Classes/Core/MORR/Data/Transcoding/Json/JsonDecoderConfiguration.png}

    \clsdcl{public class JsonDecoderConfiguration : IConfiguration}

    \clsdsp{Encapsulates all JSON decoding configuration options.}

    \begin{attributes}
        \attribute{public FilePath RelativeFilePath \{ get; set; \}}{The path to the file to store the data in relative to the recording directory.}
    \end{attributes}

    \begin{methods}
        \begin{method}{public void Parse(RawConfiguration configuration)}{Parses the configuration from the provided value.}
            \begin{parameters}
                \para{RawConfiguration configuration}{The configuration value to parse from.}
            \end{parameters}
        \end{method}
    \end{methods}
\end{class}

\begin{class}{JsonEncoderConfiguration}
    \clsdiagram{resources/Classes/Core/MORR/Data/Transcoding/Json/JsonEncoderConfiguration.png}

    \clsdcl{public class JsonEncoderConfiguration : IConfiguration}

    \clsdsp{Encapsulates all JSON encoding configuration options.}

    \begin{attributes}
        \attribute{public FilePath RelativeFilePath \{ get; set; \}}{The path to the file to store the data in relative to the recording directory.}
    \end{attributes}

    \begin{methods}
        \begin{method}{public void Parse(RawConfiguration configuration)}{Parses the configuration from the provided value.}
            \begin{parameters}
                \para{RawConfiguration configuration}{The configuration value to parse from.}
            \end{parameters}
        \end{method}
    \end{methods}
\end{class}

\subsection*{MORR.Core.Data.Transcoding.Mpeg}

\begin{class}{MpegEncoder}
    \clsdiagram{resources/Classes/Core/MORR/Data/Transcoding/Mpeg/MPEGEncoder.png}

    \clsdcl{public class MpegEncoder : IEncoder}

    \clsdsp{Encodes samples into a recording in MPEG format.}

    \begin{attributes}
        \attribute{public ManualResetEvent EncodeFinished \{ get; \}}{An event raised when encoding finishes.}
    \end{attributes}

     \begin{constructors}
		\begin{constructor}{public MpegEncoder())}{Creates a new instance of the MpegEncoder}
		\end{constructor}
		\begin{constructor}{public MpegEncoder(IFileSystem fileSystem))}{Creates a new instance of the MpegEncoder with specified FileSystem}
			\begin{parameters}
				\para{IFileSystem fileSystem}{The file system to construct the instance with.}
			\end{parameters}
		\end{constructor}
	\end{constructors}

    \begin{methods}
        \begin{method}{void Encode(DirectoryPath recordingDirectoryPath)}{Encodes all provided samples to a recording at the specified location.}
            \begin{parameters}
                \para{DirectoryPath recordingDirectoryPath}{The DirectoryPath of the file to encode to.}
            \end{parameters}
            \begin{exceptions}
                \excp{EncodingException}{Exception thrown if encoding fails.}
            \end{exceptions}
        \end{method}
    \end{methods}
\end{class}

\begin{class}{MpegEncoderConfiguration}
    \clsdiagram{resources/Classes/Core/MORR/Data/Transcoding/Mpeg/MPEGEncoderConfiguration.png}

    \clsdcl{public class MpegEncoderConfiguration : IConfiguration}

    \clsdsp{Encapsulates all MPEG encoding configuration options.}

    \begin{attributes}
        \attribute{public uint Width \{ get; set; \}}{The width in pixels of the resulting video stream.}
        \attribute{public uint Height \{ get; set; \}}{The height in pixels of the resulting video stream.}
        \attribute{public uint KiloBitsPerSecond \{ get; set; \}}{The bit rate in kilo bits per second of the resulting video stream.}
        \attribute{public uint FramesPerSecond \{ get; set; \}}{The frame rate in frames per second of the resulting video stream.}
        \attribute{public FilePath RelativeFilePath \{ get; set; \}}{The path to the file to store the data in relative to the recording directory.}
    \end{attributes}

    \begin{methods}
        \begin{method}{public void Parse(RawConfiguration configuration)}{Parses the configuration from the provided value.}
            \begin{parameters}
                \para{RawConfiguration configuration}{The configuration value to parse from.}
            \end{parameters}
        \end{method}
    \end{methods}
\end{class}

max_stars_repo_name: insightmind/MORR


text:
\documentclass{article}
\usepackage[pdftex]{graphicx}

\title{Pythorient - a lightweight script focussing on the triple junction analysis of EBSD data}

\author{J\"orn Leuthold}

\date{\today}
% Hint: \title{what ever}, \author{who care} and \date{when ever} could stand
% before or after the \begin{document} command
% BUT the \maketitle command MUST come AFTER the \begin{document} command!
\begin{document}

\maketitle

\begin{abstract}
Pythorient is developed for the identification of CSL $\Sigma$ triple junction (TJ) configurations, which is particularly useful for the characterization of the microstructure in terms of grain boundary engineering. During recrystallization at elevated temperatures the migration of grain boundaries (GB) result in an increased probability of the formation of a $\Sigma3-\Sigma3-\Sigma3^n$ $(n \geq 2)$ TJ in low stacking fault metals. This special orientation relationship affects the properties of a material in terms of crystal plasticity, creep and corrosion resistance. Understanding the processes in the microstructure which lead to a inhomogeneous distribution of CSL TJs help to improve such macroscopic properties.

\end{abstract}

\section{Brief overview of the program}
Pythorient reads HexGrid files exported from OIM Analysis (vers. 5.3). Cleaning procedures have to be done in advance. Depending on the overall quality of the EBSD measurement, a dilation cleaning -though dangerous in term of creation of artifacts- is recommended in order to remove non-indexed points. Also the rain-fall segmentation to extract the grain partition has to be carried out in OIM Analysis (which is done by default). By default the data file contains:   phi1, Phi, phi2, x, y, IQ, CI, Fit, ID, edge, phase. The Euler angles must be given in rad (not deg). General GBs and TJs are determined from Grain ID. CSL GBs and TJs are computed with respect to the crystal symmetry and point group (so far only Cubic and fcc). For plotting either a common map (so far IPF, IQ) or density estimation map of a given TJ configuration is created. The density estimation is carried out by a boxsum or gaussian method.
\begin{figure}[ht]
\centering
\includegraphics[width=\linewidth]{figs/Generated_OIM}
\caption{Example of TJ analysis on a artificially generated microstructure. For density estimation only $\Sigma3-X-X$ TJs are taken into account.}
\label{fig:generated}
\end{figure}

\begin{figure}[ht]
\centering
\includegraphics[width=\linewidth]{figs/Cu_TJ_analysis}
\caption{TJ analysis on a Cu sample subjected to annealing after severe plastic deformation.}
\label{fig:Cu_SPD}
\end{figure}
\section{Testing}
The program was tested in Python 2.7 and 3.4 using the Anaconda package manager and the basic modules numpy and scipy. Additionally numba is required for performance enhancement.

\end{document}

max_stars_repo_name: joernleu/Pythorient

max_issues_repo_name: joernleu/Pythorient

max_forks_repo_name: joernleu/Pythorient


text:
\section{Limitations and Future Work}
\label{sec:conclusion}

Disclose all limitations, and describe how potential future works can address these and lead to more interesting and ground breaking stuff.

I am continuously updating this document based on your questions and feedbacks.

I believe in the (near?) future all research papers, at least in computer science, will be published in a form that contains everything in one place, including math, code, and data, so that readers can parse the descriptions and repeat the experiments, like what we can do with Jupyter/Ipython notebooks today.


text:
%[TODO: from Feature Detection]
\section{Feature Detection}

\ifCPy

\cvCPyFunc{Canny}
Implements the Canny algorithm for edge detection.

\cvdefC{
void cvCanny(\par const CvArr* image,
\par CvArr* edges,
\par double threshold1,
\par double threshold2,
\par int aperture\_size=3 );
}\cvdefPy{Canny(image,edges,threshold1,threshold2,aperture\_size=3)-> None}
\begin{description}
\cvarg{image}{Single-channel input image}
\cvarg{edges}{Single-channel image to store the edges found by the function}
\cvarg{threshold1}{The first threshold}
\cvarg{threshold2}{The second threshold}
\cvarg{aperture\_size}{Aperture parameter for the Sobel operator (see \cvCPyCross{Sobel})}
\end{description}

The function finds the edges on the input image \texttt{image} and marks them in the output image \texttt{edges} using the Canny algorithm. The smallest value between \texttt{threshold1} and \texttt{threshold2} is used for edge linking, the largest value is used to find the initial segments of strong edges.

\cvCPyFunc{CornerEigenValsAndVecs}
Calculates eigenvalues and eigenvectors of image blocks for corner detection.

\cvdefC{
void cvCornerEigenValsAndVecs( \par const CvArr* image,\par CvArr* eigenvv,\par int blockSize,\par int aperture\_size=3 );

}\cvdefPy{CornerEigenValsAndVecs(image,eigenvv,blockSize,aperture\_size=3)-> None}

\begin{description}
\cvarg{image}{Input image}
\cvarg{eigenvv}{Image to store the results. It must be 6 times wider than the input image}
\cvarg{blockSize}{Neighborhood size (see discussion)}
\cvarg{aperture\_size}{Aperture parameter for the Sobel operator (see \cvCPyCross{Sobel})}
\end{description}

For every pixel, the function \texttt{cvCornerEigenValsAndVecs} considers a $\texttt{blockSize} \times \texttt{blockSize}$ neigborhood S(p). It calcualtes the covariation matrix of derivatives over the neigborhood as:

\[
M = \begin{bmatrix}
\sum_{S(p)}(dI/dx)^2 & \sum_{S(p)}(dI/dx \cdot dI/dy)^2 \\
\sum_{S(p)}(dI/dx \cdot dI/dy)^2 & \sum_{S(p)}(dI/dy)^2
\end{bmatrix}
\]

After that it finds eigenvectors and eigenvalues of the matrix and stores them into destination image in form
$(\lambda_1, \lambda_2, x_1, y_1, x_2, y_2)$ where
\begin{description}
\item[$\lambda_1, \lambda_2$]are the eigenvalues of $M$; not sorted
\item[$x_1, y_1$]are the eigenvectors corresponding to $\lambda_1$
\item[$x_2, y_2$]are the eigenvectors corresponding to $\lambda_2$
\end{description}

\cvCPyFunc{CornerHarris}
Harris edge detector.

\cvdefC{
void cvCornerHarris(
\par const CvArr* image,
\par CvArr* harris\_dst,
\par int blockSize,
\par int aperture\_size=3,
\par double k=0.04 );
}
\cvdefPy{CornerHarris(image,harris\_dst,blockSize,aperture\_size=3,k=0.04)-> None}

\begin{description}
\cvarg{image}{Input image}
\cvarg{harris\_dst}{Image to store the Harris detector responses. Should have the same size as \texttt{image}}
\cvarg{blockSize}{Neighborhood size (see the discussion of \cvCPyCross{CornerEigenValsAndVecs})}
\cvarg{aperture\_size}{Aperture parameter for the Sobel operator (see \cvCPyCross{Sobel}).}
% format. In the case of floating-point input format this parameter is the number of the fixed float filter used for differencing
\cvarg{k}{Harris detector free parameter. See the formula below}
\end{description}

The function runs the Harris edge detector on the image. Similarly to \cvCPyCross{CornerMinEigenVal} and \cvCPyCross{CornerEigenValsAndVecs}, for each pixel it calculates a $2\times2$ gradient covariation matrix $M$ over a $\texttt{blockSize} \times \texttt{blockSize}$ neighborhood. Then, it stores

\[
det(M) - k \, trace(M)^2
\]

to the destination image. Corners in the image can be found as the local maxima of the destination image.

\cvCPyFunc{CornerMinEigenVal}
Calculates the minimal eigenvalue of gradient matrices for corner detection.

\cvdefC{
void cvCornerMinEigenVal(
\par const CvArr* image,
\par CvArr* eigenval,
\par int blockSize,
\par int aperture\_size=3 );
}\cvdefPy{CornerMinEigenVal(image,eigenval,blockSize,aperture\_size=3)-> None}
\begin{description}
\cvarg{image}{Input image}
\cvarg{eigenval}{Image to store the minimal eigenvalues. Should have the same size as \texttt{image}}
\cvarg{blockSize}{Neighborhood size (see the discussion of \cvCPyCross{CornerEigenValsAndVecs})}
\cvarg{aperture\_size}{Aperture parameter for the Sobel operator (see \cvCPyCross{Sobel}).}
%  format. In the case of floating-point input format this parameter is the number of the fixed float filter used for differencing
\end{description}

The function is similar to \cvCPyCross{CornerEigenValsAndVecs} but it calculates and stores only the minimal eigen value of derivative covariation matrix for every pixel, i.e. $min(\lambda_1, \lambda_2)$ in terms of the previous function.

\cvCPyFunc{FindCornerSubPix}
Refines the corner locations.

\cvdefC{
void cvFindCornerSubPix(
\par const CvArr* image,
\par CvPoint2D32f* corners,
\par int count,
\par CvSize win,
\par CvSize zero\_zone,
\par CvTermCriteria criteria );
}\cvdefPy{FindCornerSubPix(image,corners,win,zero\_zone,criteria)-> corners}

\begin{description}
\cvarg{image}{Input image}
\ifC
\cvarg{corners}{Initial coordinates of the input corners; refined coordinates on output}
\cvarg{count}{Number of corners}
\fi
\ifPy
\cvarg{corners}{Initial coordinates of the input corners as a list of (x, y) pairs}
\fi
\cvarg{win}{Half of the side length of the search window. For example, if \texttt{win}=(5,5), then a $5*2+1 \times 5*2+1 = 11 \times 11$ search window would be used}
\cvarg{zero\_zone}{Half of the size of the dead region in the middle of the search zone over which the summation in the formula below is not done. It is used sometimes to avoid possible singularities of the autocorrelation matrix. The value of (-1,-1) indicates that there is no such size}
\cvarg{criteria}{Criteria for termination of the iterative process of corner refinement. That is, the process of corner position refinement stops either after a certain number of iterations or when a required accuracy is achieved. The \texttt{criteria} may specify either of or both the maximum number of iteration and the required accuracy}
\end{description}

The function iterates to find the sub-pixel accurate location of corners, or radial saddle points, as shown in on the picture below.
\ifPy
It returns the refined coordinates as a list of (x, y) pairs.
\fi

\includegraphics[width=1.0\textwidth]{pics/cornersubpix.png}

Sub-pixel accurate corner locator is based on the observation that every vector from the center $q$ to a point $p$ located within a neighborhood of $q$ is orthogonal to the image gradient at $p$ subject to image and measurement noise. Consider the expression:

\[
\epsilon_i = {DI_{p_i}}^T \cdot (q - p_i)
\]

where ${DI_{p_i}}$ is the image gradient at the one of the points $p_i$ in a neighborhood of $q$. The value of $q$ is to be found such that $\epsilon_i$ is minimized. A system of equations may be set up with $\epsilon_i$ set to zero:

\[
\sum_i(DI_{p_i} \cdot {DI_{p_i}}^T) q = \sum_i(DI_{p_i} \cdot {DI_{p_i}}^T \cdot p_i)
\]

where the gradients are summed within a neighborhood ("search window") of $q$. Calling the first gradient term $G$ and the second gradient term $b$ gives:

\[
q = G^{-1} \cdot b
\]

The algorithm sets the center of the neighborhood window at this new center $q$ and then iterates until the center keeps within a set threshold.

\cvCPyFunc{GoodFeaturesToTrack}
Determines strong corners on an image.

\cvdefC{
void cvGoodFeaturesToTrack(
\par const CvArr* image
\par CvArr* eigImage, CvArr* tempImage
\par CvPoint2D32f* corners
\par int* cornerCount
\par double qualityLevel
\par double minDistance
\par const CvArr* mask=NULL
\par int blockSize=3
\par int useHarris=0
\par double k=0.04 );
}
\cvdefPy{GoodFeaturesToTrack(image,eigImage,tempImage,cornerCount,qualityLevel,minDistance,mask=NULL,blockSize=3,useHarris=0,k=0.04)-> corners}

\begin{description}
\cvarg{image}{The source 8-bit or floating-point 32-bit, single-channel image}
\cvarg{eigImage}{Temporary floating-point 32-bit image, the same size as \texttt{image}}
\cvarg{tempImage}{Another temporary image, the same size and format as \texttt{eigImage}}
\ifC
\cvarg{corners}{Output parameter; detected corners}
\cvarg{cornerCount}{Output parameter; number of detected corners}
\else
\cvarg{cornerCount}{number of corners to detect}
\fi
\cvarg{qualityLevel}{Multiplier for the max/min eigenvalue; specifies the minimal accepted quality of image corners}
\cvarg{minDistance}{Limit, specifying the minimum possible distance between the returned corners; Euclidian distance is used}
\cvarg{mask}{Region of interest. The function selects points either in the specified region or in the whole image if the mask is NULL}
\cvarg{blockSize}{Size of the averaging block, passed to the underlying \cvCPyCross{CornerMinEigenVal} or \cvCPyCross{CornerHarris} used by the function}
\cvarg{useHarris}{If nonzero, Harris operator (\cvCPyCross{CornerHarris}) is used instead of default \cvCPyCross{CornerMinEigenVal}}
\cvarg{k}{Free parameter of Harris detector; used only if ($\texttt{useHarris} != 0$)}
\end{description}

The function finds the corners with big eigenvalues in the image. The function first calculates the minimal
eigenvalue for every source image pixel using the \cvCPyCross{CornerMinEigenVal}
function and stores them in \texttt{eigImage}. Then it performs
non-maxima suppression (only the local maxima in $3\times 3$ neighborhood
are retained). The next step rejects the corners with the minimal
eigenvalue less than $\texttt{qualityLevel} \cdot max(\texttt{eigImage}(x,y))$.
Finally, the function ensures that the distance between any two corners is not smaller than \texttt{minDistance}. The weaker corners (with a smaller min eigenvalue) that are too close to the stronger corners are rejected.

Note that the if the function is called with different values \texttt{A} and \texttt{B} of the parameter \texttt{qualityLevel}, and \texttt{A} > {B}, the array of returned corners with \texttt{qualityLevel=A} will be the prefix of the output corners array with \texttt{qualityLevel=B}.

\cvCPyFunc{HoughLines2}
Finds lines in a binary image using a Hough transform.

\cvdefC{
CvSeq* cvHoughLines2( \par CvArr* image,\par void* storage,\par int method,\par double rho,\par double theta,\par int threshold,\par double param1=0,\par double param2=0 );
}
\cvdefPy{HoughLines2(image,storage,method,rho,theta,threshold,param1=0,param2=0)-> lines}

\begin{description}
\cvarg{image}{The 8-bit, single-channel, binary source image. In the case of a probabilistic method, the image is modified by the function}
\cvarg{storage}{The storage for the lines that are detected. It can
be a memory storage (in this case a sequence of lines is created in
the storage and returned by the function) or single row/single column
matrix (CvMat*) of a particular type (see below) to which the lines'
parameters are written. The matrix header is modified by the function
so its \texttt{cols} or \texttt{rows} will contain the number of lines
detected. If \texttt{storage} is a matrix and the actual number
of lines exceeds the matrix size, the maximum possible number of lines
is returned (in the case of standard hough transform the lines are sorted
by the accumulator value)}
\cvarg{method}{The Hough transform variant, one of the following:
\begin{description}
  \cvarg{CV\_HOUGH\_STANDARD}{classical or standard Hough transform. Every line is represented by two floating-point numbers $(\rho, \theta)$, where $\rho$ is a distance between (0,0) point and the line, and $\theta$ is the angle between x-axis and the normal to the line. Thus, the matrix must be (the created sequence will be) of \texttt{CV\_32FC2} type}
  \cvarg{CV\_HOUGH\_PROBABILISTIC}{probabilistic Hough transform (more efficient in case if picture contains a few long linear segments). It returns line segments rather than the whole line. Each segment is represented by starting and ending points, and the matrix must be (the created sequence will be) of \texttt{CV\_32SC4} type}
  \cvarg{CV\_HOUGH\_MULTI\_SCALE}{multi-scale variant of the classical Hough transform. The lines are encoded the same way as \texttt{CV\_HOUGH\_STANDARD}}
\end{description}}
\cvarg{rho}{Distance resolution in pixel-related units}
\cvarg{theta}{Angle resolution measured in radians}
\cvarg{threshold}{Threshold parameter. A line is returned by the function if the corresponding accumulator value is greater than \texttt{threshold}}
\cvarg{param1}{The first method-dependent parameter:
\begin{itemize}
  \item For the classical Hough transform it is not used (0).
  \item For the probabilistic Hough transform it is the minimum line length.
  \item For the multi-scale Hough transform it is the divisor for the distance resolution $\rho$. (The coarse distance resolution will be $\rho$ and the accurate resolution will be $(\rho / \texttt{param1})$).
\end{itemize}}
\cvarg{param2}{The second method-dependent parameter:
\begin{itemize}
  \item For the classical Hough transform it is not used (0).
  \item For the probabilistic Hough transform it is the maximum gap between line segments lying on the same line to treat them as a single line segment (i.e. to join them).
  \item For the multi-scale Hough transform it is the divisor for the angle resolution $\theta$. (The coarse angle resolution will be $\theta$ and the accurate resolution will be $(\theta / \texttt{param2})$).
\end{itemize}}
\end{description}

The function implements a few variants of the Hough transform for line detection.

\ifC
\textbf{Example. Detecting lines with Hough transform.}
\begin{lstlisting}
/* This is a standalone program. Pass an image name as a first parameter
of the program.  Switch between standard and probabilistic Hough transform
by changing "#if 1" to "#if 0" and back */
#include <cv.h>
#include <highgui.h>
#include <math.h>

int main(int argc, char** argv)
{
    IplImage* src;
    if( argc == 2 && (src=cvLoadImage(argv[1], 0))!= 0)
    {
        IplImage* dst = cvCreateImage( cvGetSize(src), 8, 1 );
        IplImage* color_dst = cvCreateImage( cvGetSize(src), 8, 3 );
        CvMemStorage* storage = cvCreateMemStorage(0);
        CvSeq* lines = 0;
        int i;
        cvCanny( src, dst, 50, 200, 3 );
        cvCvtColor( dst, color_dst, CV_GRAY2BGR );
#if 1
        lines = cvHoughLines2( dst,
                               storage,
                               CV_HOUGH_STANDARD,
                               1,
                               CV_PI/180,
                               100,
                               0,
                               0 );

        for( i = 0; i < MIN(lines->total,100); i++ )
        {
            float* line = (float*)cvGetSeqElem(lines,i);
            float rho = line[0];
            float theta = line[1];
            CvPoint pt1, pt2;
            double a = cos(theta), b = sin(theta);
            double x0 = a*rho, y0 = b*rho;
            pt1.x = cvRound(x0 + 1000*(-b));
            pt1.y = cvRound(y0 + 1000*(a));
            pt2.x = cvRound(x0 - 1000*(-b));
            pt2.y = cvRound(y0 - 1000*(a));
            cvLine( color_dst, pt1, pt2, CV_RGB(255,0,0), 3, 8 );
        }
#else
        lines = cvHoughLines2( dst,
                               storage,
                               CV_HOUGH_PROBABILISTIC,
                               1,
                               CV_PI/180,
                               80,
                               30,
                               10 );
        for( i = 0; i < lines->total; i++ )
        {
            CvPoint* line = (CvPoint*)cvGetSeqElem(lines,i);
            cvLine( color_dst, line[0], line[1], CV_RGB(255,0,0), 3, 8 );
        }
#endif
        cvNamedWindow( "Source", 1 );
        cvShowImage( "Source", src );

        cvNamedWindow( "Hough", 1 );
        cvShowImage( "Hough", color_dst );

        cvWaitKey(0);
    }
}
\end{lstlisting}

This is the sample picture the function parameters have been tuned for:

\includegraphics[width=0.5\textwidth]{pics/building.jpg}

And this is the output of the above program in the case of probabilistic Hough transform (\texttt{\#if 0} case):

\includegraphics[width=0.5\textwidth]{pics/houghp.png}
\fi

\cvCPyFunc{PreCornerDetect}
Calculates the feature map for corner detection.

\cvdefC{
void cvPreCornerDetect(
\par const CvArr* image,
\par CvArr* corners,
\par int apertureSize=3 );
}
\cvdefPy{PreCornerDetect(image,corners,apertureSize=3)-> None}
\begin{description}
\cvarg{image}{Input image}
\cvarg{corners}{Image to store the corner candidates}
\cvarg{apertureSize}{Aperture parameter for the Sobel operator (see \cvCPyCross{Sobel})}
\end{description}

The function calculates the function

\[
D_x^2 D_{yy} + D_y^2 D_{xx} - 2 D_x D_y D_{xy}
\]

where $D_?$ denotes one of the first image derivatives and $D_{??}$ denotes a second image derivative.

The corners can be found as local maximums of the function below:

\ifC
\begin{lstlisting}
// assume that the image is floating-point
IplImage* corners = cvCloneImage(image);
IplImage* dilated_corners = cvCloneImage(image);
IplImage* corner_mask = cvCreateImage( cvGetSize(image), 8, 1 );
cvPreCornerDetect( image, corners, 3 );
cvDilate( corners, dilated_corners, 0, 1 );
cvSubS( corners, dilated_corners, corners );
cvCmpS( corners, 0, corner_mask, CV_CMP_GE );
cvReleaseImage( &corners );
cvReleaseImage( &dilated_corners );
\end{lstlisting}

\else
\lstinputlisting{python_fragments/precornerdetect.py}
\fi

\ifC
\cvCPyFunc{SampleLine}
Reads the raster line to the buffer.

\cvdefC{
int cvSampleLine(
\par const CvArr* image
\par CvPoint pt1
\par CvPoint pt2
\par void* buffer
\par int connectivity=8 );
}

\begin{description}
\cvarg{image}{Image to sample the line from}
\cvarg{pt1}{Starting line point}
\cvarg{pt2}{Ending line point}
\cvarg{buffer}{Buffer to store the line points; must have enough size to store
$max( |\texttt{pt2.x} - \texttt{pt1.x}|+1, |\texttt{pt2.y} - \texttt{pt1.y}|+1 )$
points in the case of an 8-connected line and
$ (|\texttt{pt2.x}-\texttt{pt1.x}|+|\texttt{pt2.y}-\texttt{pt1.y}|+1) $
in the case of a 4-connected line}
\cvarg{connectivity}{The line connectivity, 4 or 8}
\end{description}

The function implements a particular application of line iterators. The function reads all of the image points lying on the line between \texttt{pt1} and \texttt{pt2}, including the end points, and stores them into the buffer.

\fi

\fi

\ifCpp

\cvCppFunc{Canny}
Finds edges in an image using Canny algorithm.

\cvdefCpp{void Canny( const Mat\& image, Mat\& edges,\par
            double threshold1, double threshold2,\par
            int apertureSize=3, bool L2gradient=false );}
\begin{description}
\cvarg{image}{Single-channel 8-bit input image}
\cvarg{edges}{The output edge map. It will have the same size and the same type as \texttt{image}}
\cvarg{threshold1}{The first threshold for the hysteresis procedure}
\cvarg{threshold2}{The second threshold for the hysteresis procedure}
\cvarg{apertureSize}{Aperture size for the \cvCppCross{Sobel} operator}
\cvarg{L2gradient}{Indicates, whether the more accurate $L_2$ norm $=\sqrt{(dI/dx)^2 + (dI/dy)^2}$ should be used to compute the image gradient magnitude (\texttt{L2gradient=true}), or a faster default $L_1$ norm $=|dI/dx|+|dI/dy|$ is enough (\texttt{L2gradient=false})}
\end{description}

The function finds edges in the input image \texttt{image} and marks them in the output map \texttt{edges} using the Canny algorithm. The smallest value between \texttt{threshold1} and \texttt{threshold2} is used for edge linking, the largest value is used to find the initial segments of strong edges, see
\url{http://en.wikipedia.org/wiki/Canny_edge_detector}

\cvCppFunc{cornerEigenValsAndVecs}
Calculates eigenvalues and eigenvectors of image blocks for corner detection.

\cvdefCpp{void cornerEigenValsAndVecs( const Mat\& src, Mat\& dst,\par
                            int blockSize, int apertureSize,\par
                            int borderType=BORDER\_DEFAULT );}
\begin{description}
\cvarg{src}{Input single-channel 8-bit or floating-point image}
\cvarg{dst}{Image to store the results. It will have the same size as \texttt{src} and the type \texttt{CV\_32FC(6)}}
\cvarg{blockSize}{Neighborhood size (see discussion)}
\cvarg{apertureSize}{Aperture parameter for the \cvCppCross{Sobel} operator}
\cvarg{boderType}{Pixel extrapolation method; see \cvCppCross{borderInterpolate}}
\end{description}

For every pixel $p$, the function \texttt{cornerEigenValsAndVecs} considers a \texttt{blockSize} $\times$ \texttt{blockSize} neigborhood $S(p)$. It calculates the covariation matrix of derivatives over the neighborhood as:

\[
M = \begin{bmatrix}
\sum_{S(p)}(dI/dx)^2 & \sum_{S(p)}(dI/dx dI/dy)^2 \\
\sum_{S(p)}(dI/dx dI/dy)^2 & \sum_{S(p)}(dI/dy)^2
\end{bmatrix}
\]

Where the derivatives are computed using \cvCppCross{Sobel} operator.

After that it finds eigenvectors and eigenvalues of $M$ and stores them into destination image in the form
$(\lambda_1, \lambda_2, x_1, y_1, x_2, y_2)$ where
\begin{description}
\item[$\lambda_1, \lambda_2$]are the eigenvalues of $M$; not sorted
\item[$x_1, y_1$]are the eigenvectors corresponding to $\lambda_1$
\item[$x_2, y_2$]are the eigenvectors corresponding to $\lambda_2$
\end{description}

The output of the function can be used for robust edge or corner detection.

See also: \cvCppCross{cornerMinEigenVal}, \cvCppCross{cornerHarris}, \cvCppCross{preCornerDetect}

\cvCppFunc{cornerHarris}
Harris edge detector.

\cvdefCpp{void cornerHarris( const Mat\& src, Mat\& dst, int blockSize,\par
                  int apertureSize, double k,\par
                  int borderType=BORDER\_DEFAULT );}
\begin{description}
\cvarg{src}{Input single-channel 8-bit or floating-point image}
\cvarg{dst}{Image to store the Harris detector responses; will have type \texttt{CV\_32FC1} and the same size as \texttt{src}}
\cvarg{blockSize}{Neighborhood size (see the discussion of \cvCppCross{cornerEigenValsAndVecs})}
\cvarg{apertureSize}{Aperture parameter for the \cvCppCross{Sobel} operator}
\cvarg{k}{Harris detector free parameter. See the formula below}
\cvarg{boderType}{Pixel extrapolation method; see \cvCppCross{borderInterpolate}}
\end{description}

The function runs the Harris edge detector on the image. Similarly to \cvCppCross{cornerMinEigenVal} and \cvCppCross{cornerEigenValsAndVecs}, for each pixel $(x, y)$ it calculates a $2\times2$ gradient covariation matrix $M^{(x,y)}$ over a $\texttt{blockSize} \times \texttt{blockSize}$ neighborhood. Then, it computes the following characteristic:

\[
\texttt{dst}(x,y) = \mathrm{det} M^{(x,y)} - k \cdot \left(\mathrm{tr} M^{(x,y)}\right)^2
\]

Corners in the image can be found as the local maxima of this response map.

\cvCppFunc{cornerMinEigenVal}
Calculates the minimal eigenvalue of gradient matrices for corner detection.

\cvdefCpp{void cornerMinEigenVal( const Mat\& src, Mat\& dst,\par
                        int blockSize, int apertureSize=3,\par
                        int borderType=BORDER\_DEFAULT );}
\begin{description}
\cvarg{src}{Input single-channel 8-bit or floating-point image}
\cvarg{dst}{Image to store the minimal eigenvalues; will have type \texttt{CV\_32FC1} and the same size as \texttt{src}}
\cvarg{blockSize}{Neighborhood size (see the discussion of \cvCppCross{cornerEigenValsAndVecs})}
\cvarg{apertureSize}{Aperture parameter for the \cvCppCross{Sobel} operator}
\cvarg{boderType}{Pixel extrapolation method; see \cvCppCross{borderInterpolate}}
\end{description}

The function is similar to \cvCppCross{cornerEigenValsAndVecs} but it calculates and stores only the minimal eigenvalue of the covariation matrix of derivatives, i.e. $\min(\lambda_1, \lambda_2)$ in terms of the formulae in \cvCppCross{cornerEigenValsAndVecs} description.

\cvCppFunc{cornerSubPix}
Refines the corner locations.

\cvdefCpp{void cornerSubPix( const Mat\& image, vector<Point2f>\& corners,\par
                   Size winSize, Size zeroZone,\par
                   TermCriteria criteria );}
\begin{description}
\cvarg{image}{Input image}
\cvarg{corners}{Initial coordinates of the input corners; refined coordinates on output}
\cvarg{winSize}{Half of the side length of the search window. For example, if \texttt{winSize=Size(5,5)}, then a $5*2+1 \times 5*2+1 = 11 \times 11$ search window would be used}
\cvarg{zeroZone}{Half of the size of the dead region in the middle of the search zone over which the summation in the formula below is not done. It is used sometimes to avoid possible singularities of the autocorrelation matrix. The value of (-1,-1) indicates that there is no such size}
\cvarg{criteria}{Criteria for termination of the iterative process of corner refinement. That is, the process of corner position refinement stops either after a certain number of iterations or when a required accuracy is achieved. The \texttt{criteria} may specify either of or both the maximum number of iteration and the required accuracy}
\end{description}

The function iterates to find the sub-pixel accurate location of corners, or radial saddle points, as shown in on the picture below.

\includegraphics[width=1.0\textwidth]{pics/cornersubpix.png}

Sub-pixel accurate corner locator is based on the observation that every vector from the center $q$ to a point $p$ located within a neighborhood of $q$ is orthogonal to the image gradient at $p$ subject to image and measurement noise. Consider the expression:

\[
\epsilon_i = {DI_{p_i}}^T \cdot (q - p_i)
\]

where ${DI_{p_i}}$ is the image gradient at the one of the points $p_i$ in a neighborhood of $q$. The value of $q$ is to be found such that $\epsilon_i$ is minimized. A system of equations may be set up with $\epsilon_i$ set to zero:

\[
\sum_i(DI_{p_i} \cdot {DI_{p_i}}^T) - \sum_i(DI_{p_i} \cdot {DI_{p_i}}^T \cdot p_i)
\]

where the gradients are summed within a neighborhood ("search window") of $q$. Calling the first gradient term $G$ and the second gradient term $b$ gives:

\[
q = G^{-1} \cdot b
\]

The algorithm sets the center of the neighborhood window at this new center $q$ and then iterates until the center keeps within a set threshold.

\cvCppFunc{goodFeaturesToTrack}
Determines strong corners on an image.

\cvdefCpp{void goodFeaturesToTrack( const Mat\& image, vector<Point2f>\& corners,\par
                         int maxCorners, double qualityLevel, double minDistance,\par
                         const Mat\& mask=Mat(), int blockSize=3,\par
                         bool useHarrisDetector=false, double k=0.04 );}
\begin{description}
\cvarg{image}{The input 8-bit or floating-point 32-bit, single-channel image}
\cvarg{corners}{The output vector of detected corners}
\cvarg{maxCorners}{The maximum number of corners to return. If there are more corners than that will be found, the strongest of them will be returned}
\cvarg{qualityLevel}{Characterizes the minimal accepted quality of image corners; the value of the parameter is multiplied by the by the best corner quality measure (which is the min eigenvalue, see \cvCppCross{cornerMinEigenVal}, or the Harris function response, see \cvCppCross{cornerHarris}). The corners, which quality measure is less than the product, will be rejected. For example, if the best corner has the quality measure = 1500, and the \texttt{qualityLevel=0.01}, then all the corners which quality measure is less than 15 will be rejected.}
\cvarg{minDistance}{The minimum possible Euclidean distance between the returned corners}
\cvarg{mask}{The optional region of interest. If the image is not empty (then it needs to have the type \texttt{CV\_8UC1} and the same size as \texttt{image}), it will specify the region in which the corners are detected}
\cvarg{blockSize}{Size of the averaging block for computing derivative covariation matrix over each pixel neighborhood, see \cvCppCross{cornerEigenValsAndVecs}}
\cvarg{useHarrisDetector}{Indicates, whether to use \hyperref[cornerHarris]{Harris} operator or \cvCppCross{cornerMinEigenVal}}
\cvarg{k}{Free parameter of Harris detector}
\end{description}

The function finds the most prominent corners in the image or in the specified image region, as described
in \cite{Shi94}:
\begin{enumerate}
\item the function first calculates the corner quality measure at every source image pixel using the \cvCppCross{cornerMinEigenVal} or \cvCppCross{cornerHarris}
\item then it performs non-maxima suppression (the local maxima in $3\times 3$ neighborhood
are retained).
\item the next step rejects the corners with the minimal eigenvalue less than $\texttt{qualityLevel} \cdot \max_{x,y} qualityMeasureMap(x,y)$.
\item the remaining corners are then sorted by the quality measure in the descending order.
\item finally, the function throws away each corner $pt_j$ if there is a stronger corner $pt_i$ ($i < j$) such that the distance between them is less than \texttt{minDistance}
\end{enumerate}

The function can be used to initialize a point-based tracker of an object.

Note that the if the function is called with different values \texttt{A} and \texttt{B} of the parameter \texttt{qualityLevel}, and \texttt{A} > {B}, the vector of returned corners with \texttt{qualityLevel=A} will be the prefix of the output vector with \texttt{qualityLevel=B}.

See also: \cvCppCross{cornerMinEigenVal}, \cvCppCross{cornerHarris}, \cvCppCross{calcOpticalFlowPyrLK}, \cvCppCross{estimateRigidMotion}, \cvCppCross{PlanarObjectDetector}, \cvCppCross{OneWayDescriptor}

\cvCppFunc{HoughCircles}
Finds circles in a grayscale image using a Hough transform.

\cvdefCpp{void HoughCircles( Mat\& image, vector<Vec3f>\& circles,\par
                 int method, double dp, double minDist,\par
                 double param1=100, double param2=100,\par
                 int minRadius=0, int maxRadius=0 );}
\begin{description}
\cvarg{image}{The 8-bit, single-channel, grayscale input image}
\cvarg{circles}{The output vector of found circles. Each vector is encoded as 3-element floating-point vector $(x, y, radius)$}
\cvarg{method}{Currently, the only implemented method is \texttt{CV\_HOUGH\_GRADIENT}, which is basically \emph{21HT}, described in \cite{Yuen90}.}
\cvarg{dp}{The inverse ratio of the accumulator resolution to the image resolution. For example, if \texttt{dp=1}, the accumulator will have the same resolution as the input image, if \texttt{dp=2} - accumulator will have half as big width and height, etc}
\cvarg{minDist}{Minimum distance between the centers of the detected circles. If the parameter is too small, multiple neighbor circles may be falsely detected in addition to a true one. If it is too large, some circles may be missed}
\cvarg{param1}{The first method-specific parameter. in the case of \texttt{CV\_HOUGH\_GRADIENT} it is the higher threshold of the two passed to \cvCppCross{Canny} edge detector (the lower one will be twice smaller)}
\cvarg{param2}{The second method-specific parameter. in the case of \texttt{CV\_HOUGH\_GRADIENT} it is the accumulator threshold at the center detection stage. The smaller it is, the more false circles may be detected. Circles, corresponding to the larger accumulator values, will be returned first}
\cvarg{minRadius}{Minimum circle radius}
\cvarg{maxRadius}{Maximum circle radius}
\end{description}

The function finds circles in a grayscale image using some modification of Hough transform. Here is a short usage example:

\begin{lstlisting}
#include <cv.h>
#include <highgui.h>
#include <math.h>

using namespace cv;

int main(int argc, char** argv)
{
    Mat img, gray;
    if( argc != 2 && !(img=imread(argv[1], 1)).data)
        return -1;
    cvtColor(img, gray, CV_BGR2GRAY);
    // smooth it, otherwise a lot of false circles may be detected
    GaussianBlur( gray, gray, Size(9, 9), 2, 2 );
    vector<Vec3f> circles;
    HoughCircles(gray, circles, CV_HOUGH_GRADIENT,
                 2, gray->rows/4, 200, 100 );
    for( size_t i = 0; i < circles.size(); i++ )
    {
         Point center(cvRound(circles[i][0]), cvRound(circles[i][1]));
         int radius = cvRound(circles[i][2]);
         // draw the circle center
         circle( img, center, 3, Scalar(0,255,0), -1, 8, 0 );
         // draw the circle outline
         circle( img, center, radius, Scalar(0,0,255), 3, 8, 0 );
    }
    namedWindow( "circles", 1 );
    imshow( "circles", img );
    return 0;
}
\end{lstlisting}

Note that usually the function detects the circles' centers well, however it may fail to find the correct radii. You can assist the function by specifying the radius range (\texttt{minRadius} and \texttt{maxRadius}) if you know it, or you may ignore the returned radius, use only the center and find the correct radius using some additional procedure.

See also: \cvCppCross{fitEllipse}, \cvCppCross{minEnclosingCircle}

\cvCppFunc{HoughLines}
Finds lines in a binary image using standard Hough transform.

\cvdefCpp{void HoughLines( Mat\& image, vector<Vec2f>\& lines,\par
                 double rho, double theta, int threshold,\par
                 double srn=0, double stn=0 );}
\begin{description}
\cvarg{image}{The 8-bit, single-channel, binary source image. The image may be modified by the function}
\cvarg{lines}{The output vector of lines. Each line is represented by a two-element vector $(\rho, \theta)$. $\rho$ is the distance from the coordinate origin $(0,0)$ (top-left corner of the image) and $\theta$ is the line rotation angle in radians ($0 \sim \textrm{vertical line}, \pi/2 \sim \textrm{horizontal line}$)}
\cvarg{rho}{Distance resolution of the accumulator in pixels}
\cvarg{theta}{Angle resolution of the accumulator in radians}
\cvarg{threshold}{The accumulator threshold parameter. Only those lines are returned that get enough votes ($>\texttt{threshold}$)}
\cvarg{srn}{For the multi-scale Hough transform it is the divisor for the distance resolution \texttt{rho}. The coarse accumulator distance resolution will be \texttt{rho} and the accurate accumulator resolution will be \texttt{rho/srn}. If both \texttt{srn=0} and \texttt{stn=0} then the classical Hough transform is used, otherwise both these parameters should be positive.}
\cvarg{stn}{For the multi-scale Hough transform it is the divisor for the distance resolution \texttt{theta}}
\end{description}

The function implements standard or standard multi-scale Hough transform algorithm for line detection. See \cvCppCross{HoughLinesP} for the code example.

\cvCppFunc{HoughLinesP}
Finds lines segments in a binary image using probabilistic Hough transform.

\cvdefCpp{void HoughLinesP( Mat\& image, vector<Vec4i>\& lines,\par
                  double rho, double theta, int threshold,\par
                  double minLineLength=0, double maxLineGap=0 );}
\begin{description}
\cvarg{image}{The 8-bit, single-channel, binary source image. The image may be modified by the function}
\cvarg{lines}{The output vector of lines. Each line is represented by a 4-element vector $(x_1, y_1, x_2, y_2)$, where $(x_1,y_1)$ and $(x_2, y_2)$ are the ending points of each line segment detected.}
\cvarg{rho}{Distance resolution of the accumulator in pixels}
\cvarg{theta}{Angle resolution of the accumulator in radians}
\cvarg{threshold}{The accumulator threshold parameter. Only those lines are returned that get enough votes ($>\texttt{threshold}$)}
\cvarg{minLineLength}{The minimum line length. Line segments shorter than that will be rejected}
\cvarg{maxLineGap}{The maximum allowed gap between points on the same line to link them.}
\end{description}

The function implements probabilistic Hough transform algorithm for line detection, described in \cite{Matas00}. Below is line detection example:

\begin{lstlisting}
/* This is a standalone program. Pass an image name as a first parameter
of the program.  Switch between standard and probabilistic Hough transform
by changing "#if 1" to "#if 0" and back */
#include <cv.h>
#include <highgui.h>
#include <math.h>

using namespace cv;

int main(int argc, char** argv)
{
    Mat src, dst, color_dst;
    if( argc != 2 || !(src=imread(argv[1], 0)).data)
        return -1;

    Canny( src, dst, 50, 200, 3 );
    cvtColor( dst, color_dst, CV_GRAY2BGR );

#if 0
    vector<Vec2f> lines;
    HoughLines( dst, lines, 1, CV_PI/180, 100 );

    for( size_t i = 0; i < lines.size(); i++ )
    {
        float rho = lines[i][0];
        float theta = lines[i][1];
        double a = cos(theta), b = sin(theta);
        double x0 = a*rho, y0 = b*rho;
        Point pt1(cvRound(x0 + 1000*(-b)),
                  cvRound(y0 + 1000*(a)));
        Point pt2(cvRound(x0 - 1000*(-b)),
                  cvRound(y0 - 1000*(a)));
        line( color_dst, pt1, pt2, Scalar(0,0,255), 3, 8 );
    }
#else
    vector<Vec4i> lines;
    HoughLinesP( dst, lines, 1, CV_PI/180, 80, 30, 10 );
    for( size_t i = 0; i < lines.size(); i++ )
    {
        line( color_dst, Point(lines[i][0], lines[i][1]),
            Point(lines[i][2], lines[i][3]), Scalar(0,0,255), 3, 8 );
    }
#endif
    namedWindow( "Source", 1 );
    imshow( "Source", src );

    namedWindow( "Detected Lines", 1 );
    imshow( "Detected Lines", color_dst );

    waitKey(0);
    return 0;
}
\end{lstlisting}

This is the sample picture the function parameters have been tuned for:

\includegraphics[width=0.5\textwidth]{pics/building.jpg}

And this is the output of the above program in the case of probabilistic Hough transform

\includegraphics[width=0.5\textwidth]{pics/houghp.png}

\cvCppFunc{preCornerDetect}
Calculates the feature map for corner detection

\cvdefCpp{void preCornerDetect( const Mat\& src, Mat\& dst, int apertureSize,\par
                     int borderType=BORDER\_DEFAULT );}
\begin{description}
\cvarg{src}{The source single-channel 8-bit of floating-point image}
\cvarg{dst}{The output image; will have type \texttt{CV\_32F} and the same size as \texttt{src}}
\cvarg{apertureSize}{Aperture size of \cvCppCross{Sobel}}
\cvarg{borderType}{The pixel extrapolation method; see \cvCppCross{borderInterpolate}}
\end{description}

The function calculates the complex spatial derivative-based function of the source image

\[
\texttt{dst} = (D_x \texttt{src})^2 \cdot D_{yy} \texttt{src} + (D_y \texttt{src})^2 \cdot D_{xx} \texttt{src} - 2 D_x \texttt{src} \cdot D_y \texttt{src} \cdot D_{xy} \texttt{src}
\]

where $D_x$, $D_y$ are the first image derivatives, $D_{xx}$, $D_{yy}$ are the second image derivatives and $D_{xy}$ is the mixed derivative.

The corners can be found as local maximums of the functions, as shown below:

\begin{lstlisting}
Mat corners, dilated_corners;
preCornerDetect(image, corners, 3);
// dilation with 3x3 rectangular structuring element
dilate(corners, dilated_corners, Mat(), 1);
Mat corner_mask = corners == dilated_corners;
\end{lstlisting}

\fi


text:
%!TEX root = ../main.tex
\chapter{Evaluation}
The evaluation chapter measures the achievement of the aim set in the introduction. Also, to better place the artefact in the landscape of existing solutions, the evaluation is presented in direct comparison with GSB and the solutions presented by the literature.

\section{Google Safe Browsing}
The work presented throughout this dissertation aims to challenge and improve the current status quo of browser embedded anti-phishing detection systems. The artefact has successfully almost doubled the classification accuracy while respecting the priorities of such software. By design, it is geared towards delivering a minimal amount of false-positives, at the expense of a slight growth in the number of false negatives. This way, its alerts are trustworthy and do not produce noise. Furthermore, it manages to do so using just the URL of the webpage and does not rely on third-party services for any other information.

Table \ref{tab:ARTEFACT_VS_GSB} shows a comparison between the classification accuracy of both GSB and the artefact. We can observe that the artefact managed to surpass GSB's effectiveness in every instance of the testing process.

Besides classification accuracy, the artefact presents a much simpler structure while causing a disproportionally low number of false-negatives and false-positives. The addition of blacklists, whitelists and other mechanisms is expected to reduce the number of inaccurate predictions further.

\begin{singlespace}
	\begin{center}
		\label{tab:ARTEFACT_VS_GSB}
		\begin{tabular}{  m{16em}  m{12em}  m{7em} } \toprule

			                                    & \textbf{Google Safe Browsing} & \textbf{Artefact} \\ \midrule

			\textbf{Phishtank 01.04.2020}       & 50.76\%                       & 95.61\%           \\

			\textbf{Phishtank 05.04.2020}       & 48.32\%                       & 95.67\%           \\

			\textbf{Phishtank 05.05.2020 (API)} & 44.47\%                       & 94.80\%           \\

			\textbf{Phishtank 06.05.2020 (API)} & 44.33\%                       & 94.78\%           \\

			\textbf{Phishtank 07.05.2020 (API)} & 44.94\%                       & 94.75\%           \\

			\textbf{Phishtank 08.05.2020 (API)} & 45.34\%                       & 94.63\%           \\

			\textbf{Phishtank 09.05.2020 (API)} & 45.33\%                       & 94.64\%           \\

			\textbf{Phishtank 10.05.2020 (API)} & 43.60\%                       & 95.08\%           \\ \bottomrule
		\end{tabular}
		\captionsetup{type=table}\caption{Classification accuracy comparison between GSB and the artefact}
	\end{center}
\end{singlespace}

\section{Literature}
Besides accomplishing the aim of surpassing GSB's performance, the artefact presents competitive results even when compared to what the literature presents. Table \ref{tab:EXISTENT_SOLUTIONS_COMPARISON} shows that when placing the artefact in the comparison of solutions presented by \cite{Adebowale}, its accuracy comes above all others. Moreover, it is trained on a wide range of international phishing URLs, which gives the model language independence.

\begin{center}
	\begin{tabular}{  m{0.5em}  m{6em}  m{13.3em}  m{8.5em}  m{2.3em}  m{4.3em}  } \toprule

		                                & \textbf{Plugin }      & \textbf{Features \& Techniques}      & \textbf{Browser}    & \textbf{Acc}   & \textbf{Type}     \\ \midrule

		\multicolumn{1}{r}{\textbf{1}}  & GoldPhish             & Heuristics \& Features-based         & IE                  & 98             & Free              \\

		\multicolumn{1}{r}{\textbf{2}}  & Cloudmark             & Heuristics                           & IE                  & 94             & Free              \\

		\multicolumn{1}{r}{\textbf{3}}  & SmartScreen           & Blacklist and whitelist              & IE                  & 95.9           & Free              \\

		\multicolumn{1}{r}{\textbf{4}}  & Netcraft              & Blacklist and whitelist              & Chrome, Firefox     & 90             & Free              \\

		\multicolumn{1}{r}{\textbf{5}}  & SpoofGuard            & Heuristics \& Features-based         & IE                  & 91             & Free              \\

		\multicolumn{1}{r}{\textbf{6}}  & Phishdentity          & Google search-by-image API           & IE                  & 97.2           & Research          \\

		\multicolumn{1}{r}{\textbf{7}}  & PhishTackle           & Heuristics \& Features-based         & IE                  & 91.3           & Research          \\

		\multicolumn{1}{r}{\textbf{8}}  & PhishGuard            & Heuristics                           & Firefox             & 94             & Research          \\

		\multicolumn{1}{r}{\textbf{9}}  & PhishIdentifier       & Heuristics                           & Firefox             & 92             & Research          \\

		\multicolumn{1}{r}{\textbf{10}} & PhishTester           & Heuristics \& Features-based         & IE                  & 97.1           & Research          \\

		\multicolumn{1}{r}{\textbf{11}} & CANTINA+              & Heuristics \& Features-based         & IE                  & 98.06          & Research          \\

		\multicolumn{1}{r}{\textbf{12}} & PhishAri              & Features-based                       & Chrome              & 92.52          & Research          \\

		\multicolumn{1}{r}{\textbf{13}} & PhishShield           & Heuristics \& Feature-based          & Chrome              & 96.57          & Research          \\

		\multicolumn{1}{r}{\textbf{14}} & PhishNet              & Blacklist                            & Chrome              & 95             & Research          \\

		\multicolumn{1}{r}{\textbf{15}} & PhishDef              & Heuristics \& Feature-based          & Chrome              & 97             & Research          \\

		\multicolumn{1}{r}{\textbf{16}} & GSB                   & Blacklist                            & Chrome, Firefox     & 93.3           & Free              \\

		\multicolumn{1}{r}{\textbf{17}} & PhishZoo              & Heuristics                           & Chrome              & 96.10          & Research          \\

		\multicolumn{1}{r}{\textbf{18}} & Seclayer              & Heuristics                           & IE, Chrome          & 91             & Free              \\

		\multicolumn{1}{r}{\textbf{19}} & IPDPS                 & Heuristics, Feature and image        & IE, Chrome, Firefox & 98.55          & Research          \\ \midrule

		\multicolumn{1}{r}{\textbf{20}} & \textbf{The artefact} & \textbf{Heuristics \& Feature-based} & \textbf{N/A}        & \textbf{99.29} & \textbf{Research} \\ \bottomrule
	\end{tabular}
	\captionsetup{type=table}\caption{A comparison of existing solutions \citep{Adebowale}}
	\label{tab:EXISTENT_SOLUTIONS_COMPARISON}
\end{center}

\section{Chapter overview}
This chapter assessed classifier's performance and aim accomplishment by comparing the results presented in the previous chapter against both GSB and the relevant solutions proposed by the literature. This comparison shows that the artefact outperforms GSB by a large margin, and manages to rank first when placed between literature solutions.

max_stars_repo_name: abutnaru/dissertation

max_issues_repo_name: abutnaru/dissertation

max_forks_repo_name: abutnaru/dissertation


text:
\section{Google: Learning Hand-Eye Coordination for Robotic Grasping}

\begin{frame}{\insertsec}
Paper published by Sergey Levine \emph{et al.} (2016).
Its objective is for a robot to learn how to grab pieces by using only image data and a movement
and rotation vector.

\begin{center}
    \includegraphics[width=.9\textwidth]{images/google_robot}
\end{center}
\end{frame}

\begin{frame}
Network with both image input and scalar input, each value is repeated
\(img.width \times img.height\) times to match the CNN size.
\begin{center}
    \includegraphics[width=\textwidth]{images/google_robot_network}
\end{center}
\end{frame}

max_stars_repo_name: jmigual/FIB-TFG

max_issues_repo_name: jmigual/FIB-TFG

max_forks_repo_name: jmigual/FIB-TFG