#LyX 2.0 created this file. For more info see http://www.lyx.org/
\lyxformat 413
\begin_document
\begin_header
\textclass article
\begin_preamble
\usepackage{dsfont}

% No date please
\date{}
\fontfamily{cmss}
\selectfont
\end_preamble
\use_default_options false
\maintain_unincluded_children false
\language english
\language_package default
\inputencoding auto
\fontencoding global
\font_roman default
\font_sans helvet
\font_typewriter default
\font_default_family default
\use_non_tex_fonts false
\font_sc false
\font_osf false
\font_sf_scale 100
\font_tt_scale 100

\graphics default
\default_output_format default
\output_sync 0
\bibtex_command default
\index_command default
\paperfontsize default
\spacing single
\use_hyperref false
\papersize default
\use_geometry false
\use_amsmath 1
\use_esint 0
\use_mhchem 0
\use_mathdots 1
\cite_engine basic
\use_bibtopic false
\use_indices false
\paperorientation portrait
\suppress_date false
\use_refstyle 0
\index Index
\shortcut idx
\color #008000
\end_index
\secnumdepth 3
\tocdepth 3
\paragraph_separation indent
\paragraph_indentation default
\quotes_language english
\papercolumns 1
\papersides 1
\paperpagestyle default
\tracking_changes false
\output_changes false
\html_math_output 0
\html_css_as_file 0
\html_be_strict false
\end_header

\begin_body

\begin_layout Standard
\begin_inset ERT
status collapsed

\begin_layout Plain Layout

% This is for 
\backslash
LyX
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset FormulaMacro
\newcommand{\bfrac}[2]{\frac{#1}{#2}}
\end_inset


\end_layout

\begin_layout Standard
\begin_inset FormulaMacro
\newcommand{\nth}{^{\textrm{th}}}
\end_inset


\end_layout

\begin_layout Standard
\begin_inset FormulaMacro
\newcommand{\R}{R}
\end_inset


\end_layout

\begin_layout Standard
\begin_inset FormulaMacro
\renewcommand{\N}{N}
\end_inset


\end_layout

\begin_layout Standard
\begin_inset FormulaMacro
\renewcommand{\Z}{Z}
\end_inset


\end_layout

\begin_layout Standard
\begin_inset FormulaMacro
\renewcommand{\tra}{^{T}}
\end_inset


\end_layout

\begin_layout Standard
\begin_inset FormulaMacro
\renewcommand{\xx}{\mathbf{x}}
\end_inset


\end_layout

\begin_layout Standard
\begin_inset ERT
status collapsed

\begin_layout Plain Layout

% This is for 
\backslash
LaTeX
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
fontfamily{cmss}
\backslash
selectfont
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
renewcommand{
\backslash
R}{{
\backslash
mathds{R}}} 
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
renewcommand{
\backslash
N}{{
\backslash
mathds{N}}}
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
renewcommand{
\backslash
Z}{{
\backslash
mathds{Z}}}
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
renewcommand{
\backslash
tra}{^{
\backslash
top}}
\end_layout

\begin_layout Plain Layout

\end_layout

\begin_layout Plain Layout


\backslash
renewcommand{
\backslash
bfrac}[2]{
\backslash
frac{{
\backslash
textstyle #1 }}{{
\backslash
textstyle #2 }}}
\end_layout

\end_inset


\end_layout

\begin_layout Title
Mini-HOWTO on using Octave for Unconstrained Nonlinear Optimization
\begin_inset Foot
status collapsed

\begin_layout Plain Layout
Author : Etienne Grossmann 
\family typewriter
<etienne@egdn.net>
\family default
 (soon replaced by 
\begin_inset Quotes eld
\end_inset

Octave-Forge developers
\begin_inset Quotes erd
\end_inset

?).
 This document is free documentation; you can redistribute it and/or modify
 it under the terms of the GNU Free Documentation License as published by
 the Free Software Foundation.
\begin_inset Newline newline
\end_inset

.
\begin_inset space ~
\end_inset


\begin_inset space ~
\end_inset


\begin_inset space ~
\end_inset

This is distributed in the hope that it will be useful, but WITHOUT ANY
 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 FOR A PARTICULAR PURPOSE.
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Note Comment
status collapsed

\begin_layout Plain Layout
Keywords: nonlinear optimization, octave, tutorial, Nelder-Mead, Conjugate
 Gradient, Levenberg-Marquardt
\end_layout

\end_inset


\end_layout

\begin_layout Standard

\emph on
This document refers to the frontend function 
\begin_inset Quotes eld
\end_inset

minimize
\begin_inset Quotes erd
\end_inset

, which is now deprecated and will be removed in a future version of the
 optim package.
 An alternative frontend function is 
\begin_inset Quotes eld
\end_inset

nonlin_min
\begin_inset Quotes erd
\end_inset

, not described here.
 The backends of 
\begin_inset Quotes eld
\end_inset

minimize
\begin_inset Quotes erd
\end_inset

 (
\begin_inset Quotes eld
\end_inset

nelder_mead_min
\begin_inset Quotes erd
\end_inset

, 
\begin_inset Quotes eld
\end_inset

bfgsmin
\begin_inset Quotes erd
\end_inset

, 
\begin_inset Quotes eld
\end_inset

d2_min
\begin_inset Quotes erd
\end_inset

) can also be called directly.
 This document will be removed in the future.
\end_layout

\begin_layout Standard
Nonlinear optimization problems are very common and when a solution cannot
 be found analytically, one usually tries to find it numerically.
 This document shows how to perform unconstrained nonlinear minimization
 using the Octave language for numerical computation.
 We assume to be so lucky as to have an initial guess from which to start
 an iterative method, and so impatient as to avoid as much as possible going
 into the details of the algorithm.
 In the following examples, we consider multivariable problems, but the
 single variable case is solved in exactly the same way.
\end_layout

\begin_layout Standard
All the algorithms used below return numerical approximations of 
\emph on
local minima
\emph default
 of the optimized function.
 In the following examples, we minimize a function with a single minimum
 (Figure
\begin_inset space ~
\end_inset


\begin_inset CommandInset ref
LatexCommand ref
reference "fig:function"

\end_inset

), which is relatively easily found.
 In practice, success of optimization algorithms greatly depend on the optimized
 function and on the starting point.
\end_layout

\begin_layout Section*

\end_layout

\begin_layout Standard
\begin_inset ERT
status collapsed

\begin_layout Plain Layout


\backslash
fontfamily{cmss} 
\backslash
selectfont A simple example
\end_layout

\end_inset


\end_layout

\begin_layout Standard
\begin_inset Float figure
wide false
sideways false
status open

\begin_layout Plain Layout
\align center
\begin_inset ERT
status collapsed

\begin_layout Plain Layout


\backslash
raisebox{6mm}{
\end_layout

\end_inset


\begin_inset Graphics
	filename figures/2D_slice-3.eps2
	width 30text%

\end_inset


\begin_inset ERT
status collapsed

\begin_layout Plain Layout

}
\end_layout

\end_inset


\begin_inset space ~
\end_inset


\end_layout

\begin_layout Plain Layout
\begin_inset Caption

\begin_layout Plain Layout
\begin_inset CommandInset label
LatexCommand label
name "fig:function"

\end_inset

 2D slice of the function that is minimized throughout this tutorial.
 Although not obvious at first sight, it has a unique minimum.
\end_layout

\end_inset


\end_layout

\end_inset


\end_layout

\begin_layout Standard
We will use a call of the type
\end_layout

\begin_layout LyX-Code
[x_best, best_value, niter] = minimize (func, x_init)
\end_layout

\begin_layout Standard
to find the minimum of 
\begin_inset Formula 
\[
\begin{array}{cccc}
f\,: & \left(x_{1},.x_{2},x_{3}\right)\in\R^{3} & \longrightarrow & \left(x_{1}-1\right)^{2}/9+\left(x_{3}-1\right)^{2}/9+\left(x_{3}-1\right)^{2}/9\\
 &  &  & -\cos\left(x_{1}-1\right)-\cos\left(x_{2}-1\right)-\cos\left(x_{3}-1\right).
\end{array}
\]

\end_inset


\end_layout

\begin_layout Standard
The following commands should find a local minimum of 
\begin_inset Formula \ensuremath{f()}

\end_inset

, using the Nelder-Mead (aka 
\begin_inset Quotes eld
\end_inset

downhill simplex
\begin_inset Quotes erd
\end_inset

) algorithm and starting from a randomly chosen point 
\family typewriter
x0
\family default

\begin_inset space ~
\end_inset

:
\end_layout

\begin_layout LyX-Code
function cost = foo (xx)
\end_layout

\begin_layout LyX-Code
  xx--;  
\end_layout

\begin_layout LyX-Code
  cost = sum (-cos(xx)+xx.^2/9);
\end_layout

\begin_layout LyX-Code
endfunction
\end_layout

\begin_layout LyX-Code
x0 = [-1, 3, -2];
\end_layout

\begin_layout LyX-Code
[x,v,n] = minimize ("foo", x0)
\end_layout

\begin_layout Standard
The output should look like
\begin_inset space ~
\end_inset

:
\end_layout

\begin_layout LyX-Code
x =
\end_layout

\begin_layout LyX-Code
  1.00000 1.00000 1.00000
\end_layout

\begin_layout LyX-Code

\end_layout

\begin_layout LyX-Code
v = -3.0000
\end_layout

\begin_layout LyX-Code
n = 248
\end_layout

\begin_layout Standard
This means that a minimum has been found in 
\begin_inset Formula \ensuremath{\left(1,1,1\right)}

\end_inset

 and that the value at that point is 
\begin_inset Formula \ensuremath{-3}

\end_inset

.
 This is correct, since all the points of the form 
\begin_inset Formula \ensuremath{x_{1}=1+2i\pi,\, x_{2}=1+2j\pi,\, x_{3}=1+2k\pi}

\end_inset

, for some 
\begin_inset Formula \ensuremath{i,j,k\in\N}

\end_inset

, minimize 
\begin_inset Formula \ensuremath{f()}

\end_inset

.
 The number of function evaluations, 248, is also returned.
 Note that this number depends on the starting point.
 You will most likely obtain different numbers if you change 
\family typewriter
x0
\family default
.
\end_layout

\begin_layout Standard
The Nelder-Mead algorithm is quite robust, but unfortunately it is not very
 efficient.
 For high-dimensional problems, its execution time may become prohibitive.
\end_layout

\begin_layout Section*

\end_layout

\begin_layout Standard
\begin_inset ERT
status collapsed

\begin_layout Plain Layout


\backslash
fontfamily{cmss} 
\backslash
selectfont Using the first differential
\end_layout

\end_inset


\end_layout

\begin_layout Standard
Fortunately, when a function, like 
\begin_inset Formula \ensuremath{f()}

\end_inset

 above, is differentiable, more efficient optimization algorithms can be
 used.
 If 
\family typewriter
minimize()
\family default
 is given the differential of the optimized function, using the 
\family typewriter
"df"
\family default
 option, it will use a conjugate gradient method.
\end_layout

\begin_layout LyX-Code
## Function returning partial derivatives
\end_layout

\begin_layout LyX-Code
function dc = diffoo (x)
\end_layout

\begin_layout LyX-Code
    x = x(:)' - 1;
\end_layout

\begin_layout LyX-Code
    dc = sin (x) + 2*x/9;
\end_layout

\begin_layout LyX-Code
endfunction
\end_layout

\begin_layout LyX-Code
[x, v, n] = minimize ("foo", x0, "df", "diffoo")
\end_layout

\begin_layout Standard
This produces the output
\begin_inset space ~
\end_inset

:
\end_layout

\begin_layout LyX-Code
x =
\end_layout

\begin_layout LyX-Code
  1.00000 1.00000 1.00000
\end_layout

\begin_layout LyX-Code
v = -3 
\end_layout

\begin_layout LyX-Code
n =
\end_layout

\begin_layout LyX-Code
  108 6
\end_layout

\begin_layout Standard
The same minimum has been found, but only 108 function evaluations were
 needed, together with 6 evaluations of the differential.
 Here, 
\family typewriter
diffoo()
\family default
 takes the same argument as 
\family typewriter
foo()
\family default
 and returns the partial derivatives of 
\begin_inset Formula \ensuremath{f()}

\end_inset

 with respect to the corresponding variables.
 It doesn't matter if it returns a row or column vector or a matrix, as
 long as the 
\begin_inset Formula \ensuremath{i\nth}

\end_inset

 element of 
\family typewriter
diffoo(x)
\family default
 is the partial derivative of 
\begin_inset Formula \ensuremath{f()}

\end_inset

 with respect to 
\begin_inset Formula \ensuremath{x_{i}}

\end_inset

 .
\end_layout

\begin_layout Section*

\end_layout

\begin_layout Standard
\begin_inset ERT
status collapsed

\begin_layout Plain Layout


\backslash
fontfamily{cmss} 
\backslash
selectfont Using numerical approximations of the first differential
\end_layout

\end_inset


\end_layout

\begin_layout Standard
Sometimes, the minimized function is differentiable, but actually writing
 down its differential is more work than one would like.
 Numerical differentiation offers a solution which is less efficient in
 terms of computation cost, but easy to implement.
 The 
\family typewriter
"ndiff"
\family default
 option of 
\family typewriter
minimize()
\family default
 uses numerical differentiation to execute exactly the same algorithm as
 in the previous example.
 However, because numerical approximation of the differentia is used, the
 outpud may differ slightly
\begin_inset space ~
\end_inset

:
\end_layout

\begin_layout LyX-Code
[x, v, n] = minimize ("foo", x0, "ndiff")
\end_layout

\begin_layout Standard
wich yields
\begin_inset space ~
\end_inset

:
\end_layout

\begin_layout LyX-Code
x =
\end_layout

\begin_layout LyX-Code
  1.00000 1.00000 1.00000
\end_layout

\begin_layout LyX-Code
v = -3 
\end_layout

\begin_layout LyX-Code
n =
\end_layout

\begin_layout LyX-Code
  78 6
\end_layout

\begin_layout Standard
Note that each time the differential is numerically approximated, 
\family typewriter
foo()
\family default
 is called 6 times (twice per input element), so that 
\family typewriter
foo()
\family default
 is evaluated a total of (78+6*6=) 114 times in this example.
\end_layout

\begin_layout Section*

\end_layout

\begin_layout Standard
\begin_inset ERT
status collapsed

\begin_layout Plain Layout


\backslash
fontfamily{cmss} 
\backslash
selectfont Using the first and second differentials
\end_layout

\end_inset


\end_layout

\begin_layout Standard
When the function is twice differentiable and one knows how to compute its
 first and second differentials, still more efficient algorithms can be
 used (in our case, a variant of Levenberg-Marquardt).
 The option 
\family typewriter
"d2f"
\family default
 allows to specify a function that returns the value of the function, the
 first and second differentials of the minimized function.
 Entering the commands
\begin_inset space ~
\end_inset

: 
\end_layout

\begin_layout LyX-Code
function [c, dc, d2c] = d2foo (x)
\end_layout

\begin_layout LyX-Code
    c = foo(x);
\end_layout

\begin_layout LyX-Code
    dc = diffoo(x);
\end_layout

\begin_layout LyX-Code
    d2c = diag (cos (x(:)-1) + 2/9);
\end_layout

\begin_layout LyX-Code
end
\end_layout

\begin_layout LyX-Code
[x,v,n] = minimize ("foo", x0, "d2f", "d2foo") 
\end_layout

\begin_layout Standard
produces the output
\begin_inset space ~
\end_inset

:
\end_layout

\begin_layout LyX-Code
x =
\end_layout

\begin_layout LyX-Code
  1.0000 1.0000 1.0000
\end_layout

\begin_layout LyX-Code
v = -3
\end_layout

\begin_layout LyX-Code
n =
\end_layout

\begin_layout LyX-Code
  34 5
\end_layout

\begin_layout Standard
This time, 34 function evaluations, and 5 evaluations of 
\family typewriter
d2foo()
\family default
 were needed.
\end_layout

\begin_layout Section*

\end_layout

\begin_layout Standard
\begin_inset ERT
status collapsed

\begin_layout Plain Layout


\backslash
fontfamily{cmss} 
\backslash
selectfont Summary
\end_layout

\end_inset


\end_layout

\begin_layout Standard
We have just seen the most basic ways of solving nonlinear unconstrained
 optimization problems.
 The online help system of Octave (try e.g.
 
\begin_inset Quotes eld
\end_inset


\family typewriter
help minimize
\family default

\begin_inset Quotes erd
\end_inset

) will yield information on other issues, such as 
\emph on
passing extra arguments
\emph default
 to the minimized function, 
\emph on
controling the termination
\emph default
 of the optimization process, choosing the algorithm etc.
\end_layout

\begin_layout LyX-Code

\end_layout

\end_body
\end_document
