Add a hyper-parameter tuning module

src/mlpack/core/CMakeLists.txt

@@ -5,6 +5,7 @@ set(DIRS
   cv
   data
   dists
+  hpt
   kernels
   math
   metrics

src/mlpack/core/hpt/CMakeLists.txt

@@ -0,0 +1,15 @@
+set(SOURCES
+  bind.hpp
+  cv_function.hpp
+  cv_function_impl.hpp
+  deduce_hp_types.hpp
+  hpt.hpp
+  hpt_impl.hpp
+)
+
+set(DIR_SRCS)
+foreach(file ${SOURCES})
+  set(DIR_SRCS ${DIR_SRCS} ${CMAKE_CURRENT_SOURCE_DIR}/${file})
+endforeach()
+
+set(MLPACK_SRCS ${MLPACK_SRCS} ${DIR_SRCS} PARENT_SCOPE)

src/mlpack/core/hpt/bind.hpp

@@ -0,0 +1,111 @@
+/**
+ * @file bind.hpp
+ * @author Kirill Mishchenko
+ *
+ * Facilities for supporting bound arguments.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_HPT_BIND_HPP
+#define MLPACK_CORE_HPT_BIND_HPP
+
+#include <type_traits>
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace hpt {
+
+template<typename>
+struct PreBoundArg;
+
+/**
+ * Mark the given argument as one that should be bound. It can be applied to
+ * arguments that are passed to the Optimize method of HyperParameterTuner.
+ *
+ * The implementation avoids data coping. If the passed argument is an l-value
+ * reference, we store it as a const l-value rerefence inside the returned
+ * PreBoundArg object. If the passed argument is an r-value reference,
+ * ligth-weight coping (by taking possesion of the r-value) will be made during
+ * the initialization of the returned PreBoundArg object.
+ */
+template<typename T>
+PreBoundArg<T> Bind(T&& value)
+{
+  return PreBoundArg<T>{std::forward<T>(value)};
+}
+
+/**
+ * A struct for storing information about a bound argument. Objects of this type
+ * are supposed to be passed into the CVFunction constructor.
+ *
+ * This struct is not meant to be used directly by users. Rather use the
+ * mlpack::hpt::Bind function.
+ *
+ * @tparam T The type of the bound argument.
+ * @tparam I The index of the bound argument.
+ */
+template<typename T, size_t I>
+struct BoundArg
+{
+  //! The index of the bound argument.
+  static const size_t index = I;
+
+  //! The value of the bound argument.
+  const T& value;
+};
+
+/**
+ * A struct for marking arguments as ones that should be bound (it can be useful
+ * for the Optimize method of HyperParameterTuner). Arguments of this type are
+ * supposed to be converted into structs of the type BoundArg by adding
+ * information about argument positions.
+ *
+ * This struct is not meant to be used directly by users. Rather use the
+ * mlpack::hpt::Bind function.
+ */
+template<typename T>
+struct PreBoundArg
+{
+  using Type = T;
+
+  const T value;
+};
+
+/**
+ * The specialization of the template for references.
+ *
+ * This struct is not meant to be used directly by users. Rather use the
+ * mlpack::hpt::Bind function.
+ */
+template<typename T>
+struct PreBoundArg<T&>
+{
+  using Type = T;
+
+  const T& value;
+};
+
+/**
+ * A type function for checking whether the given type is PreBoundArg.
+ */
+template<typename T>
+class IsPreBoundArg
+{
+  template<typename>
+  struct Implementation : std::false_type {};
+
+  template<typename Type>
+  struct Implementation<PreBoundArg<Type>> : std::true_type {};
+
+ public:
+  static const bool value = Implementation<typename std::decay<T>::type>::value;
+};
+
+} // namespace hpt
+} // namespace mlpack
+
+#endif

src/mlpack/core/hpt/cv_function.hpp

@@ -0,0 +1,143 @@
+/**
+ * @file cv_function.hpp
+ * @author Kirill Mishchenko
+ *
+ * A cross-validation wrapper for optimizers.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_HPT_CV_FUNCTION_HPP
+#define MLPACK_CORE_HPT_CV_FUNCTION_HPP
+
+#include <mlpack/core.hpp>
+
+namespace mlpack {
+namespace hpt {
+
+/**
+ * This wrapper serves for adapting the interface of the cross-validation
+ * classes to the one that can be utilized by the mlpack optimizers.
+ *
+ * This class is not supposed to be used directly by users. To tune
+ * hyper-parameters see HyperParameterTuner.
+ *
+ * @tparam CVType A cross-validation strategy.
+ * @tparam TotalArgs The total number of arguments that are supposed to be
+ *     passed to the Evaluate method of a CVType object.
+ * @tparam BoundArgs Types of arguments (wrapped into the BoundArg struct) that
+ *     should be passed into the Evaluate method of a CVType object but are not
+ *     going to be passed into the Evaluate method of a CVFunction object.
+ */
+template<typename CVType, size_t TotalArgs, typename... BoundArgs>
+class CVFunction
+{
+ public:
+  /**
+   * Initialize a CVFunction object.
+   *
+   * @param cv A cross-validation object.
+   * @param BoundArgs Arguments that should be passed into the Evaluate method
+   *     of the CVType object but are not going to be passed into the Evaluate
+   *     method of this object.
+   */
+  CVFunction(CVType& cv, const BoundArgs&... args);
+
+  /**
+   * Run cross-validation with the bound and passed parameters.
+   *
+   * @param parameters Arguments (rather than the bound arguments) that should
+   *     be passed into the Evaluate method of the CVType object.
+   */
+  double Evaluate(const arma::mat& parameters);
+
+  //! The used machine learning algorithm.
+  using MLAlgorithm = typename
+      std::remove_reference<decltype(std::declval<CVType>().Model())>::type;
+
+  //! Access and modify the best model so far.
+  MLAlgorithm& BestModel() { return bestModel; }
+
+ private:
+  //! The type of tuples of BoundArgs.
+  using BoundArgsTupleType = std::tuple<BoundArgs...>;
+
+  //! The amount of bound arguments.
+  static const size_t BoundArgsAmount =
+      std::tuple_size<BoundArgsTupleType>::value;
+
+  /**
+   * A struct that finds out whether the next argument for the Evaluate method
+   * of a CVType object should be a bound argument at the position BAIndex
+   * rather than an element of parameters at the position PIndex.
+   */
+  template<size_t BAIndex,
+           size_t PIndex,
+           bool BoundArgsIndexInRange = BAIndex < BoundArgsAmount>
+  struct UseBoundArg;
+
+  //! A reference to the cross-validation object.
+  CVType& cv;
+
+  //! The bound arguments.
+  BoundArgsTupleType boundArgs;
+
+  //! The best objective so far.
+  double bestObjective;
+
+  //! The best model so far.
+  MLAlgorithm bestModel;
+
+  /**
+   * Collect all arguments and run cross-validation.
+   */
+  template<size_t BAIndex,
+           size_t PIndex,
+           typename... Args,
+           typename =
+               typename std::enable_if<BAIndex + PIndex < TotalArgs>::type>
+  inline double Evaluate(const arma::mat& parameters, const Args&... args);
+
+  /**
+   * Run cross-validation with the collected arguments.
+   */
+  template<size_t BAIndex,
+           size_t PIndex,
+           typename... Args,
+           typename =
+               typename std::enable_if<BAIndex + PIndex == TotalArgs>::type,
+           typename = void>
+  inline double Evaluate(const arma::mat& parameters, const Args&... args);
+
+  /**
+   * Put the bound argument (at the BAIndex position) as the next one.
+   */
+  template<size_t BAIndex,
+           size_t PIndex,
+           typename... Args,
+           typename = typename std::enable_if<
+               UseBoundArg<BAIndex, PIndex>::value>::type>
+  inline double PutNextArg(const arma::mat& parameters, const Args&... args);
+
+  /**
+   * Put the element (at the PIndex position) of the parameters as the next one.
+   */
+  template<size_t BAIndex,
+           size_t PIndex,
+           typename... Args,
+           typename = typename std::enable_if<
+               !UseBoundArg<BAIndex, PIndex>::value>::type,
+           typename = void>
+  inline double PutNextArg(const arma::mat& parameters, const Args&... args);
+};
+
+
+} // namespace hpt
+} // namespace mlpack
+
+// Include implementation
+#include "cv_function_impl.hpp"
+
+#endif

src/mlpack/core/hpt/cv_function_impl.hpp

@@ -0,0 +1,118 @@
+/**
+ * @file cv_function_impl.hpp
+ * @author Kirill Mishchenko
+ *
+ * The implementation of the class CVFunction.
+ *
+ * mlpack is free software; you may redistribute it and/or modify it under the
+ * terms of the 3-clause BSD license.  You should have received a copy of the
+ * 3-clause BSD license along with mlpack.  If not, see
+ * http://www.opensource.org/licenses/BSD-3-Clause for more information.
+ */
+#ifndef MLPACK_CORE_HPT_CV_FUNCTION_IMPL_HPP
+#define MLPACK_CORE_HPT_CV_FUNCTION_IMPL_HPP
+
+namespace mlpack {
+namespace hpt {
+
+template<typename CVType, size_t TotalArgs, typename... BoundArgs>
+template<size_t BAIndex, size_t PIndex>
+struct CVFunction<CVType, TotalArgs, BoundArgs...>::UseBoundArg<
+    BAIndex, PIndex, true>
+{
+  using BoundArgType =
+      typename std::tuple_element<BAIndex, BoundArgsTupleType>::type;
+
+  static const bool value = BoundArgType::index == BAIndex + PIndex;
+};
+
+template<typename CVType, size_t TotalArgs, typename... BoundArgs>
+template<size_t BAIndex, size_t PIndex>
+struct CVFunction<CVType, TotalArgs, BoundArgs...>::UseBoundArg<
+    BAIndex, PIndex, false>
+{
+  static const bool value = false;
+};
+
+template<typename CVType, size_t TotalArgs, typename... BoundArgs>
+CVFunction<CVType, TotalArgs, BoundArgs...>::CVFunction(
+    CVType& cv, const BoundArgs&... args) :
+    cv(cv),
+    boundArgs(args...),
+    bestObjective(std::numeric_limits<double>::max())
+{ /* Nothing left to do. */ }
+
+template<typename CVType, size_t TotalArgs, typename... BoundArgs>
+double CVFunction<CVType, TotalArgs, BoundArgs...>::Evaluate(
+    const arma::mat& parameters)
+{
+  return Evaluate<0, 0>(parameters);
+}
+
+template<typename CVType, size_t TotalArgs, typename... BoundArgs>
+template<size_t BAIndex,
+         size_t PIndex,
+         typename... Args,
+         typename>
+double CVFunction<CVType, TotalArgs, BoundArgs...>::Evaluate(
+    const arma::mat& parameters,
+    const Args&... args)
+{
+  return PutNextArg<BAIndex, PIndex>(parameters, args...);
+}
+
+template<typename CVType, size_t TotalArgs, typename... BoundArgs>
+template<size_t BAIndex,
+         size_t PIndex,
+         typename... Args,
+         typename,
+         typename>
+double CVFunction<CVType, TotalArgs, BoundArgs...>::Evaluate(
+    const arma::mat& /* parameters */,
+    const Args&... args)
+{
+  double objective = cv.Evaluate(args...);
+
+  // Change the best model if we have got a better score, or if we probably
+  // have not assigned any valid (trained) model yet.
+  if (bestObjective > objective ||
+      bestObjective == std::numeric_limits<double>::max())
+  {
+    bestObjective = objective;
+    bestModel = std::move(cv.Model());
+  }
+
+  return objective;
+}
+
+template<typename CVType, size_t TotalArgs, typename... BoundArgs>
+template<size_t BAIndex,
+         size_t PIndex,
+         typename... Args,
+         typename>
+double CVFunction<CVType, TotalArgs, BoundArgs...>::PutNextArg(
+    const arma::mat& parameters,
+    const Args&... args)
+{
+  return Evaluate<BAIndex + 1, PIndex>(
+      parameters, args..., std::get<BAIndex>(boundArgs).value);
+}
+
+template<typename CVType, size_t TotalArgs, typename... BoundArgs>
+template<size_t BAIndex,
+         size_t PIndex,
+         typename... Args,
+         typename,
+         typename>
+double CVFunction<CVType, TotalArgs, BoundArgs...>::PutNextArg(
+    const arma::mat& parameters,
+    const Args&... args)
+{
+  return Evaluate<BAIndex, PIndex + 1>(
+      parameters, args..., parameters(PIndex, 0));
+}
+
+} // namespace hpt
+} // namespace mlpack
+
+#endif