sdyna.h

Go to the documentation of this file.

/****************************************************************************
 *   This file is part of the aGrUM/pyAgrum library.                        *
 *                                                                          *
 *   Copyright (c) 2005-2025 by                                             *
 *       - Pierre-Henri WUILLEMIN(_at_LIP6)                                 *
 *       - Christophe GONZALES(_at_AMU)                                     *
 *                                                                          *
 *   The aGrUM/pyAgrum library is free software; you can redistribute it    *
 *   and/or modify it under the terms of either :                           *
 *                                                                          *
 *    - the GNU Lesser General Public License as published by               *
 *      the Free Software Foundation, either version 3 of the License,      *
 *      or (at your option) any later version,                              *
 *    - the MIT license (MIT),                                              *
 *    - or both in dual license, as here.                                   *
 *                                                                          *
 *   (see https://agrum.gitlab.io/articles/dual-licenses-lgplv3mit.html)    *
 *                                                                          *
 *   This aGrUM/pyAgrum library is distributed in the hope that it will be  *
 *   useful, but WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,          *
 *   INCLUDING BUT NOT LIMITED TO THE WARRANTIES MERCHANTABILITY or FITNESS *
 *   FOR A PARTICULAR PURPOSE  AND NONINFRINGEMENT. IN NO EVENT SHALL THE   *
 *   AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER *
 *   LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,        *
 *   ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR  *
 *   OTHER DEALINGS IN THE SOFTWARE.                                        *
 *                                                                          *
 *   See LICENCES for more details.                                         *
 *                                                                          *
 *   SPDX-FileCopyrightText: Copyright 2005-2025                            *
 *       - Pierre-Henri WUILLEMIN(_at_LIP6)                                 *
 *       - Christophe GONZALES(_at_AMU)                                     *
 *   SPDX-License-Identifier: LGPL-3.0-or-later OR MIT                      *
 *                                                                          *
 *   Contact  : info_at_agrum_dot_org                                       *
 *   homepage : http://agrum.gitlab.io                                      *
 *   gitlab   : https://gitlab.com/agrumery/agrum                           *
 *                                                                          *
 ****************************************************************************/
 

 
// =========================================================================
#ifndef GUM_SDYNA_H
#define GUM_SDYNA_H
// =========================================================================
#include <agrum/agrum.h>
// =========================================================================
// =========================================================================
#include <agrum/FMDP/decision/randomDecider.h>
#include <agrum/FMDP/decision/statisticalLazyDecider.h>
#include <agrum/FMDP/planning/adaptiveRMaxPlaner.h>
 
#include <agrum/FMDP/decision/E_GreedyDecider.h>
 
// =========================================================================
 
namespace gum {

  class SDYNA {
    // ###################################################################
    // ###################################################################
 
    public:
    // ==========================================================================
    // ==========================================================================
    static SDYNA* spitiInstance(double attributeSelectionThreshold = 0.99,
                                double discountFactor              = 0.9,
                                double epsilon                     = 1,
                                Idx    observationPhaseLenght      = 100,
                                Idx    nbValueIterationStep        = 10) {
      bool               actionReward = false;
      ILearningStrategy* ls
          = new FMDPLearner< CHI2TEST, CHI2TEST, ITILEARNER >(attributeSelectionThreshold,
                                                              actionReward);
      IPlanningStrategy< double >* ps
          = StructuredPlaner< double >::sviInstance(discountFactor, epsilon);
      IDecisionStrategy* ds = new E_GreedyDecider();
      return new SDYNA(ls, ps, ds, observationPhaseLenght, nbValueIterationStep, actionReward);
    }
 
    // ==========================================================================
    // ==========================================================================
    static SDYNA* spimddiInstance(double attributeSelectionThreshold = 0.99,
                                  double similarityThreshold         = 0.3,
                                  double discountFactor              = 0.9,
                                  double epsilon                     = 1,
                                  Idx    observationPhaseLenght      = 100,
                                  Idx    nbValueIterationStep        = 10) {
      bool               actionReward = false;
      ILearningStrategy* ls
          = new FMDPLearner< GTEST, GTEST, IMDDILEARNER >(attributeSelectionThreshold,
                                                          actionReward,
                                                          similarityThreshold);
      IPlanningStrategy< double >* ps
          = StructuredPlaner< double >::spumddInstance(discountFactor, epsilon, false);
      IDecisionStrategy* ds = new E_GreedyDecider();
      return new SDYNA(ls,
                       ps,
                       ds,
                       observationPhaseLenght,
                       nbValueIterationStep,
                       actionReward,
                       false);
    }
 
    // ==========================================================================
    // ==========================================================================
    static SDYNA* RMaxMDDInstance(double attributeSelectionThreshold = 0.99,
                                  double similarityThreshold         = 0.3,
                                  double discountFactor              = 0.9,
                                  double epsilon                     = 1,
                                  Idx    observationPhaseLenght      = 100,
                                  Idx    nbValueIterationStep        = 10) {
      bool               actionReward = true;
      ILearningStrategy* ls
          = new FMDPLearner< GTEST, GTEST, IMDDILEARNER >(attributeSelectionThreshold,
                                                          actionReward,
                                                          similarityThreshold);
      AdaptiveRMaxPlaner* rm
          = AdaptiveRMaxPlaner::ReducedAndOrderedInstance(ls, discountFactor, epsilon);
      IPlanningStrategy< double >* ps = rm;
      IDecisionStrategy*           ds = rm;
      return new SDYNA(ls, ps, ds, observationPhaseLenght, nbValueIterationStep, actionReward);
    }
 
    // ==========================================================================
    // ==========================================================================
    static SDYNA* RMaxTreeInstance(double attributeSelectionThreshold = 0.99,
                                   double discountFactor              = 0.9,
                                   double epsilon                     = 1,
                                   Idx    observationPhaseLenght      = 100,
                                   Idx    nbValueIterationStep        = 10) {
      bool               actionReward = true;
      ILearningStrategy* ls
          = new FMDPLearner< GTEST, GTEST, ITILEARNER >(attributeSelectionThreshold, actionReward);
      AdaptiveRMaxPlaner* rm = AdaptiveRMaxPlaner::TreeInstance(ls, discountFactor, epsilon);
      IPlanningStrategy< double >* ps = rm;
      IDecisionStrategy*           ds = rm;
      return new SDYNA(ls, ps, ds, observationPhaseLenght, nbValueIterationStep, actionReward);
    }
 
    // ==========================================================================
    // ==========================================================================
    static SDYNA* RandomMDDInstance(double attributeSelectionThreshold = 0.99,
                                    double similarityThreshold         = 0.3,
                                    double discountFactor              = 0.9,
                                    double epsilon                     = 1,
                                    Idx    observationPhaseLenght      = 100,
                                    Idx    nbValueIterationStep        = 10) {
      bool               actionReward = true;
      ILearningStrategy* ls
          = new FMDPLearner< GTEST, GTEST, IMDDILEARNER >(attributeSelectionThreshold,
                                                          actionReward,
                                                          similarityThreshold);
      IPlanningStrategy< double >* ps
          = StructuredPlaner< double >::spumddInstance(discountFactor, epsilon);
      IDecisionStrategy* ds = new RandomDecider();
      return new SDYNA(ls, ps, ds, observationPhaseLenght, nbValueIterationStep, actionReward);
    }
 
    // ==========================================================================
    // ==========================================================================
    static SDYNA* RandomTreeInstance(double attributeSelectionThreshold = 0.99,
                                     double discountFactor              = 0.9,
                                     double epsilon                     = 1,
                                     Idx    observationPhaseLenght      = 100,
                                     Idx    nbValueIterationStep        = 10) {
      bool               actionReward = true;
      ILearningStrategy* ls
          = new FMDPLearner< CHI2TEST, CHI2TEST, ITILEARNER >(attributeSelectionThreshold,
                                                              actionReward);
      IPlanningStrategy< double >* ps
          = StructuredPlaner< double >::sviInstance(discountFactor, epsilon);
      IDecisionStrategy* ds = new RandomDecider();
      return new SDYNA(ls, ps, ds, observationPhaseLenght, nbValueIterationStep, actionReward);
    }

 
    // ###################################################################
    // ###################################################################
 
    // ==========================================================================
    // ==========================================================================
 
    private:
    SDYNA(ILearningStrategy*           learner,
          IPlanningStrategy< double >* planer,
          IDecisionStrategy*           decider,
          Idx                          observationPhaseLenght,
          Idx                          nbValueIterationStep,
          bool                         actionReward,
          bool                         verbose = true);
 
    // ==========================================================================
    // ==========================================================================
 
    public:
    ~SDYNA();

 
 
    // ###################################################################
    // ###################################################################
 
    public:
    // ==========================================================================
    // ==========================================================================
    void addAction(const Idx actionId, const std::string& actionName) {
      fmdp_->addAction(actionId, actionName);
    }
 
    // ==========================================================================
    // ==========================================================================
    void addVariable(const DiscreteVariable* var) { fmdp_->addVariable(var); }

 
 
    // ###################################################################
    // ###################################################################
 
    public:
    // ==========================================================================
    // ==========================================================================
    void initialize();
 
    // ==========================================================================
    // ==========================================================================
    void initialize(const Instantiation& initialState);

 
 
    // ###################################################################
    // ###################################################################
 
    public:
    // ==========================================================================
    // ==========================================================================
    void setCurrentState(const Instantiation& currentState) { lastState_ = currentState; }
 
    // ==========================================================================
    // ==========================================================================
    Idx takeAction(const Instantiation& curState);
 
    // ==========================================================================
    // ==========================================================================
    Idx takeAction();
 
    // ==========================================================================
    // ==========================================================================
    void feedback(const Instantiation& originalState,
                  const Instantiation& reachedState,
                  Idx                  performedAction,
                  double               obtainedReward);
 
    // ==========================================================================
    // ==========================================================================
    void feedback(const Instantiation& reachedState, double obtainedReward);
 
    // ==========================================================================
    // ==========================================================================
    void makePlanning(Idx nbStep);

 
 
    public:
    // ==========================================================================
    // ==========================================================================
    std::string toString();
 
    std::string optimalPolicy2String() { return _planer_->optimalPolicy2String(); }
 
    // ###################################################################
    // ###################################################################
 
    public:
    // ==========================================================================
    // ==========================================================================
    Size learnerSize() { return _learner_->size(); }
 
    // ==========================================================================
    // ==========================================================================
    Size modelSize() { return fmdp_->size(); }
 
    // ==========================================================================
    // ==========================================================================
    Size valueFunctionSize() { return _planer_->vFunctionSize(); }
 
    // ==========================================================================
    // ==========================================================================
    Size optimalPolicySize() { return _planer_->optimalPolicySize(); }

 
 
    protected:
    FMDP< double >* fmdp_;

    Instantiation lastState_;
 
    private:
    ILearningStrategy* _learner_;

    IPlanningStrategy< double >* _planer_;

    IDecisionStrategy* _decider_;
 

    Idx _observationPhaseLenght_;

    Idx _nbObservation_;

    Idx _nbValueIterationStep_;

    Idx _lastAction_;

    Set< Observation* > _bin_;
 
    bool _actionReward_;
 
    bool verbose_;
  };
 
 
} /* namespace gum */
 
 
#endif   // GUM_SDYNA_H