adam.hpp

Go to the documentation of this file.

#ifndef MLPACK_CORE_OPTIMIZERS_ADAM_ADAM_HPP
#define MLPACK_CORE_OPTIMIZERS_ADAM_ADAM_HPP

#include <mlpack/prereqs.hpp>

#include <mlpack/core/optimizers/sgd/sgd.hpp>
#include "adam_update.hpp"
#include "adamax_update.hpp"
#include "amsgrad_update.hpp"
#include "nadam_update.hpp"
#include "nadamax_update.hpp"
#include "optimisticadam_update.hpp"

namespace mlpack {
namespace optimization {

template<typename UpdateRule = AdamUpdate>
class AdamType
{
 public:
  AdamType(const double stepSize = 0.001,
           const size_t batchSize = 32,
           const double beta1 = 0.9,
           const double beta2 = 0.999,
           const double eps = 1e-8,
           const size_t maxIterations = 100000,
           const double tolerance = 1e-5,
           const bool shuffle = true);

  template<typename DecomposableFunctionType>
  double Optimize(DecomposableFunctionType& function, arma::mat& iterate)
  {
    return optimizer.Optimize(function, iterate);
  }

  double StepSize() const { return optimizer.StepSize(); }
  double& StepSize() { return optimizer.StepSize(); }

  size_t BatchSize() const { return optimizer.BatchSize(); }
  size_t& BatchSize() { return optimizer.BatchSize(); }

  double Beta1() const { return optimizer.UpdatePolicy().Beta1(); }
  double& Beta1() { return optimizer.UpdatePolicy().Beta1(); }

  double Beta2() const { return optimizer.UpdatePolicy().Beta2(); }
  double& Beta2() { return optimizer.UpdatePolicy().Beta2(); }

  double Epsilon() const { return optimizer.UpdatePolicy().Epsilon(); }
  double& Epsilon() { return optimizer.UpdatePolicy().Epsilon(); }

  size_t MaxIterations() const { return optimizer.MaxIterations(); }
  size_t& MaxIterations() { return optimizer.MaxIterations(); }

  double Tolerance() const { return optimizer.Tolerance(); }
  double& Tolerance() { return optimizer.Tolerance(); }

  bool Shuffle() const { return optimizer.Shuffle(); }
  bool& Shuffle() { return optimizer.Shuffle(); }

 private:
  SGD<UpdateRule> optimizer;
};

using Adam = AdamType<AdamUpdate>;

using AdaMax = AdamType<AdaMaxUpdate>;

using AMSGrad = AdamType<AMSGradUpdate>;

using Nadam = AdamType<NadamUpdate>;

using NadaMax = AdamType<NadaMaxUpdate>;

using OptimisticAdam = AdamType<OptimisticAdamUpdate>;

} // namespace optimization
} // namespace mlpack

// Include implementation.
#include "adam_impl.hpp"

#endif