Line data    Source code

       1              : // SPDX-License-Identifier: Apache-2.0
       2              : /**
       3              :  * Copyright (C) 2024 Hyunwoo LEE <dlgusdn0414@snu.ac.kr>
       4              :  *
       5              :  * @file   lr_scheduler_cosine.cpp
       6              :  * @date   7 October 2024
       7              :  * @brief  This is CosineAnnealing Learning Rate Scheduler class
       8              :  * @see    https://github.com/nnstreamer/nntrainer
       9              :  * @author Hyunwoo LEE <dlgusdn0414@snu.ac.kr>
      10              :  * @bug    No known bugs except for NYI items
      11              :  *
      12              :  */
      13              : 
      14              : #if defined(_WIN32)
      15              : #define _USE_MATH_DEFINES
      16              : #include <math.h>
      17              : #else
      18              : #include <cmath>
      19              : #endif
      20              : 
      21              : #include <common_properties.h>
      22              : #include <lr_scheduler_cosine.h>
      23              : #include <nntrainer_error.h>
      24              : #include <nntrainer_log.h>
      25              : #include <node_exporter.h>
      26              : 
      27              : namespace nntrainer {
      28              : 
      29           10 : CosineAnnealingLearningRateScheduler::CosineAnnealingLearningRateScheduler() :
      30              :   lr_props(props::MaxLearningRate(), props::MinLearningRate(),
      31           10 :            props::DecaySteps()) {}
      32              : 
      33            3 : void CosineAnnealingLearningRateScheduler::finalize() {
      34            4 :   NNTR_THROW_IF(std::get<props::MaxLearningRate>(lr_props).empty(),
      35              :                 std::invalid_argument)
      36              :     << "[CosineAnnealingLearningRateScheduler] Max Learning Rate is not set";
      37            2 :   NNTR_THROW_IF(std::get<props::MinLearningRate>(lr_props).empty(),
      38              :                 std::invalid_argument)
      39              :     << "[CosineAnnealingLearningRateScheduler] Min Learning Rate is not set";
      40            2 :   NNTR_THROW_IF(std::get<props::DecaySteps>(lr_props).empty(),
      41              :                 std::invalid_argument)
      42              :     << "[CosineAnnealingLearningRateScheduler] Decay Steps is not set";
      43            2 : }
      44              : 
      45           23 : void CosineAnnealingLearningRateScheduler::setProperty(
      46              :   const std::vector<std::string> &values) {
      47           23 :   auto left = loadProperties(values, lr_props);
      48           23 :   NNTR_THROW_IF(left.size(), std::invalid_argument)
      49              :     << "[CosineAnnealingLearningRateScheduler] There are unparsed properties";
      50           22 : }
      51              : 
      52            0 : void CosineAnnealingLearningRateScheduler::exportTo(
      53              :   Exporter &exporter, const ml::train::ExportMethods &method) const {
      54            0 :   exporter.saveResult(lr_props, method, this);
      55            0 : }
      56              : 
      57            1 : double CosineAnnealingLearningRateScheduler::getLearningRate(size_t iteration) {
      58              :   auto const &max_lr = std::get<props::MaxLearningRate>(lr_props);
      59              :   auto const &min_lr = std::get<props::MinLearningRate>(lr_props);
      60              :   auto const &decay_steps = std::get<props::DecaySteps>(lr_props);
      61              : 
      62              :   // Cosine annealing formula
      63              :   double cosine_decay =
      64            1 :     0.5 * (1 + cos(M_PI * (iteration % decay_steps) / (double)decay_steps));
      65            1 :   return min_lr + (max_lr - min_lr) * cosine_decay;
      66              : }
      67              : 
      68              : } // namespace nntrainer