Scaled descent

Slides

View slides in full screen / PDF

In-class demos

using Optim, Plots
f(x) = (1.0 - x[1])^2 + 100.0 * (x[2] - x[1]^2)^2
x0 = [0.0, 0.0]
opts = Optim.Options(store_trace=true, extended_trace=true);

Precompiling Optim...
    380.2 ms  ✓ FiniteDiff
    391.5 ms  ✓ FiniteDiff → FiniteDiffSparseArraysExt
    614.4 ms  ✓ NLSolversBase
    937.4 ms  ✓ LineSearches
   1719.2 ms  ✓ Optim
  5 dependencies successfully precompiled in 4 seconds. 61 already precompiled.

res = []
for method in [GradientDescent, BFGS, Newton]
   r = optimize(f, x0, method(), autodiff=:forward, opts)
   push!(res, r)
end

trace = res[1].trace
fvals = [t.value for t in trace]
plot(fvals[1:100], label="steepest descent")
trace = res[2].trace
fvals = [t.value for t in trace]
plot!(fvals, label="quasi Newton")
trace = res[3].trace
fvals = [t.value for t in trace]
plot!(fvals, label="Newton")