diff --git a/test/NLPModelsJuMP.jl b/test/NLPModelsJuMP.jl new file mode 100644 index 0000000..6c14615 --- /dev/null +++ b/test/NLPModelsJuMP.jl @@ -0,0 +1,53 @@ +module TestWithNLPModelsJuMP + +using Test + +using JuMP +using ArrayDiff +import MathOptInterface as MOI +import NLPModelsJuMP +import JSOSolvers + +function runtests() + for name in names(@__MODULE__; all = true) + if startswith("$(name)", "test_") + @testset "$(name)" begin + getfield(@__MODULE__, name)() + end + end + end + return +end + +function test_neural_nlpmodels_jump() + n = 2 + X = [1.0 0.5; 0.3 0.8] + target = [0.5 0.2; 0.1 0.7] + model = Model(NLPModelsJuMP.Optimizer) + set_attribute(model, "solver", JSOSolvers.LBFGSSolver) + set_attribute( + model, + MOI.AutomaticDifferentiationBackend(), + ArrayDiff.Mode(), + ) + @variable(model, W1[1:n, 1:n], container = ArrayDiff.ArrayOfVariables) + @variable(model, W2[1:n, 1:n], container = ArrayDiff.ArrayOfVariables) + # Use distinct starting values to break symmetry + start_W1 = [0.3 -0.2; 0.1 0.4] + start_W2 = [-0.1 0.5; 0.2 -0.3] + for i in 1:n, j in 1:n + set_start_value(W1[i, j], start_W1[i, j]) + set_start_value(W2[i, j], start_W2[i, j]) + end + Y = W2 * tanh.(W1 * X) + loss = sum((Y .- target) .^ 2) + @objective(model, Min, loss) + optimize!(model) + @test termination_status(model) == MOI.LOCALLY_SOLVED + @test objective_value(model) < 1e-6 + return +end + +end + +TestWithNLPModelsJuMP.runtests() diff --git a/test/Optimisers.jl b/test/Optimisers.jl new file mode 100644 index 0000000..864cc30 --- /dev/null +++ b/test/Optimisers.jl @@ -0,0 +1,56 @@ +module TestWithOptimisers + +using Test + +using JuMP +using ArrayDiff +import LinearAlgebra +import MathOptInterface as MOI +import NLPModelsJuMP + +include(joinpath(@__DIR__, "OptimisersSolver.jl")) + +function runtests() + for name in names(@__MODULE__; all = true) + if startswith("$(name)", "test_") + @testset "$(name)" begin + getfield(@__MODULE__, name)() + end + end + end + return +end + +function test_neural_optimisers() + n = 2 + X = [1.0 0.5; 0.3 0.8] + target = [0.5 0.2; 0.1 0.7] + model = Model(NLPModelsJuMP.Optimizer) + set_attribute(model, "solver", OptimisersSolver) + set_attribute( + model, + MOI.AutomaticDifferentiationBackend(), + ArrayDiff.Mode(), + ) + @variable(model, W1[1:n, 1:n], container = ArrayDiff.ArrayOfVariables) + @variable(model, W2[1:n, 1:n], container = ArrayDiff.ArrayOfVariables) + # Use distinct starting values to break symmetry + start_W1 = [0.3 -0.2; 0.1 0.4] + start_W2 = [-0.1 0.5; 0.2 -0.3] + for i in 1:n, j in 1:n + set_start_value(W1[i, j], start_W1[i, j]) + set_start_value(W2[i, j], start_W2[i, j]) + end + Y = W2 * tanh.(W1 * X) + loss = sum((Y .- target) .^ 2) + @objective(model, Min, loss) + set_attribute(model, "max_iter", 20_000) + set_attribute(model, "tol", 1e-6) + optimize!(model) + @test objective_value(model) < 1e-3 + return +end + +end + +TestWithOptimisers.runtests() diff --git a/test/OptimisersSolver.jl b/test/OptimisersSolver.jl new file mode 100644 index 0000000..78b6164 --- /dev/null +++ b/test/OptimisersSolver.jl @@ -0,0 +1,69 @@ +import SolverCore +import NLPModels +import Optimisers + +# An NLPModels solver that runs an `Optimisers.AbstractRule` (e.g. `Adam`) on +# the variable vector of an unconstrained `AbstractNLPModel` using `obj` and +# `grad!`. Designed to be plugged into `NLPModelsJuMP.Optimizer` via +# `set_attribute(model, "solver", OptimisersSolver)`. +mutable struct OptimisersSolver{R<:Optimisers.AbstractRule} <: + SolverCore.AbstractOptimizationSolver + rule::R + x::Vector{Float64} + g::Vector{Float64} +end + +function OptimisersSolver( + nlp::NLPModels.AbstractNLPModel; + rule::Optimisers.AbstractRule = Optimisers.Adam(0.05), +) + nvar = NLPModels.get_nvar(nlp.meta) + return OptimisersSolver(rule, zeros(Float64, nvar), zeros(Float64, nvar)) +end + +function SolverCore.reset!(solver::OptimisersSolver) + fill!(solver.x, 0.0) + fill!(solver.g, 0.0) + return solver +end + +function SolverCore.reset!( + solver::OptimisersSolver, + nlp::NLPModels.AbstractNLPModel, +) + return SolverCore.reset!(solver) +end + +function SolverCore.solve!( + solver::OptimisersSolver, + nlp::NLPModels.AbstractNLPModel, + stats::SolverCore.GenericExecutionStats; + max_iter::Int = 10_000, + tol::Real = 1e-6, + verbose::Int = 0, +) + SolverCore.reset!(stats) + copyto!(solver.x, NLPModels.get_x0(nlp.meta)) + state = Optimisers.setup(solver.rule, solver.x) + start = time() + iter = 0 + status = :max_iter + while iter < max_iter + NLPModels.grad!(nlp, solver.x, solver.g) + if LinearAlgebra.norm(solver.g) < tol + status = :first_order + break + end + state, solver.x = Optimisers.update!(state, solver.x, solver.g) + iter += 1 + if verbose > 0 && iter % verbose == 0 + @info "Optimisers" iter obj = NLPModels.obj(nlp, solver.x) + end + end + SolverCore.set_iter!(stats, iter) + SolverCore.set_status!(stats, status) + SolverCore.set_solution!(stats, solver.x) + SolverCore.set_objective!(stats, NLPModels.obj(nlp, solver.x)) + SolverCore.set_time!(stats, time() - start) + return stats +end diff --git a/test/Project.toml b/test/Project.toml index 050654f..68b2a1c 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,15 +2,22 @@ ArrayDiff = "c45fa1ca-6901-44ac-ae5b-5513a4852d50" Calculus = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9" GenOpt = "f2c049d8-7489-4223-990c-4f1c121a4cde" +JSOSolvers = "10dff2fc-5484-5881-a0e0-c90441020f8a" JuMP = "4076af6c-e467-56ae-b986-b466b2749572" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" MathOptInterface = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" +NLPModels = "a4795742-8479-5a88-8948-cc11e1c8c1a6" +NLPModelsJuMP = "792afdf1-32c1-5681-94e0-d7bf7a5df49e" NLopt = "76087f3c-5699-56af-9a33-bf431cd00edd" +Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" OrderedCollections = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" Revise = "295af30f-e4ad-537b-8983-00126c2a3abe" +SolverCore = "ff4d7338-4cf1-434d-91df-b86cb86fb843" SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [sources] ArrayDiff = {path = ".."} +NLopt = {url = "https://github.com/jump-dev/NLopt.jl/", rev = "bl/diff_backend"} +NLPModelsJuMP = {url = "https://github.com/JuliaSmoothOptimizers/NLPModelsJuMP.jl", rev = "bl/arraydiff"} diff --git a/test/runtests.jl b/test/runtests.jl index 327ddb2..fb70b82 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,5 +1,11 @@ include("ReverseAD.jl") include("ArrayDiff.jl") include("JuMP.jl") -# Currently needs https://github.com/jump-dev/NLopt.jl/pull/273 -#include("NLopt.jl") +if VERSION >= v"1.11" + # [sources] not supported on Julia v1.10 + # Needs https://github.com/jump-dev/NLopt.jl/pull/273 + include("NLopt.jl") + # Needs https://github.com/JuliaSmoothOptimizers/NLPModelsJuMP.jl/pull/229 + include("NLPModelsJuMP.jl") + include("Optimisers.jl") +end