aGrUM 3.3.2
a C++ library for (probabilistic) graphical models
structuredPlaner_tpl.h
Go to the documentation of this file.
1/****************************************************************************
2 * This file is part of the aGrUM/pyAgrum library. *
3 * *
4 * Copyright (c) 2005-2026 by *
5 * - Pierre-Henri WUILLEMIN(_at_LIP6) *
6 * - Christophe GONZALES(_at_AMU) *
7 * *
8 * The aGrUM/pyAgrum library is free software; you can redistribute it *
9 * and/or modify it under the terms of either : *
10 * *
11 * - the GNU Lesser General Public License as published by *
12 * the Free Software Foundation, either version 3 of the License, *
13 * or (at your option) any later version, *
14 * - the MIT license (MIT), *
15 * - or both in dual license, as here. *
16 * *
17 * (see https://agrum.gitlab.io/articles/dual-licenses-lgplv3mit.html) *
18 * *
19 * This aGrUM/pyAgrum library is distributed in the hope that it will be *
20 * useful, but WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, *
21 * INCLUDING BUT NOT LIMITED TO THE WARRANTIES MERCHANTABILITY or FITNESS *
22 * FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE *
23 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER *
24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, *
25 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR *
26 * OTHER DEALINGS IN THE SOFTWARE. *
27 * *
28 * See LICENCES for more details. *
29 * *
30 * SPDX-FileCopyrightText: Copyright 2005-2026 *
31 * - Pierre-Henri WUILLEMIN(_at_LIP6) *
32 * - Christophe GONZALES(_at_AMU) *
33 * SPDX-License-Identifier: LGPL-3.0-or-later OR MIT *
34 * *
35 * Contact : info_at_agrum_dot_org *
36 * homepage : http://agrum.gitlab.io *
37 * gitlab : https://gitlab.com/agrumery/agrum *
38 * *
39 ****************************************************************************/
40
41#pragma once
42
43
51
52// =========================================================================
53#include <queue>
54#include <vector>
55// #include <algorithm>
56// #include <utility>
57// =========================================================================
59
61// =========================================================================
65// =========================================================================
67// =========================================================================
68
70#define RECAST(x) reinterpret_cast< const MultiDimFunctionGraph< GUM_ELEMENT >* >(x)
71
72namespace gum {
73
74
75 /* **************************************************************************************************
76 * **/
77 /* ** **/
78 /* ** Constructors / Destructors **/
79 /* ** **/
80 /* **************************************************************************************************
81 * **/
82
83 // ===========================================================================
84 // Default constructor
85 // ===========================================================================
86 template < typename GUM_ELEMENT >
88 GUM_ELEMENT discountFactor,
89 GUM_ELEMENT epsilon,
90 bool verbose) :
91 discountFactor_(discountFactor), operator_(opi), verbose_(verbose) {
92 GUM_CONSTRUCTOR(StructuredPlaner);
93
94 _threshold_ = epsilon;
95 vFunction_ = nullptr;
96 optimalPolicy_ = nullptr;
97 }
98
99 // ===========================================================================
100 // Default destructor
101 // ===========================================================================
102 template < typename GUM_ELEMENT >
104 GUM_DESTRUCTOR(StructuredPlaner);
105
106 if (vFunction_) { delete vFunction_; }
107
108 if (optimalPolicy_) delete optimalPolicy_;
109
110 delete operator_;
111 }
112
113 /* **************************************************************************************************
114 * **/
115 /* ** **/
116 /* ** Datastructure access methods **/
117 /* ** **/
118 /* **************************************************************************************************
119 * **/
120
121 // ===========================================================================
122 // Initializes data structure needed for making the planning
123 // ===========================================================================
124 template < typename GUM_ELEMENT >
126 // ************************************************************************
127 // Discarding the case where no \pi* have been computed
128 if (!optimalPolicy_ || optimalPolicy_->root() == 0) return "NO OPTIMAL POLICY CALCULATED YET";
129
130 // ************************************************************************
131 // Initialisation
132
133 // Declaration of the needed string stream
134 std::string output;
135 std::string terminalStream;
136 std::string nonTerminalStream;
137 std::string arcstream;
139 // First line for the toDot
140 output += "\ndigraph \" OPTIMAL POLICY \" {\n";
141
142 // Form line for the internal node stream en the terminal node stream
143 terminalStream += "node [shape = box];\n";
144 nonTerminalStream += "node [shape = ellipse];\n";
145
146 // For somme clarity in the final string
147 std::string tab = "\t";
149 // To know if we already checked a node or not
150 Set< NodeId > visited;
151
152 // FIFO of nodes to visit
153 std::queue< NodeId > fifo;
154
155 // Loading the FIFO
156 fifo.push(optimalPolicy_->root());
157 visited << optimalPolicy_->root();
159
160 // ************************************************************************
161 // Main loop
162 while (!fifo.empty()) {
163 // Node to visit
164 NodeId currentNodeId = fifo.front();
165 fifo.pop();
166
167 // Checking if it is terminal
168 if (optimalPolicy_->isTerminalNode(currentNodeId)) {
169 // Get back the associated ActionSet
170 ActionSet ase = optimalPolicy_->nodeValue(currentNodeId);
171
172 // Creating a line for this node
173 terminalStream += std::format("{0}{1};{0}{1} [label=\"{1} - ", tab, currentNodeId);
174
175 // Enumerating and adding to the line the associated optimal actions
176 for (SequenceIteratorSafe< Idx > valIter = ase.beginSafe(); valIter != ase.endSafe();
177 ++valIter) {
178 terminalStream += fmdp_->actionName(*valIter);
179 terminalStream += ' ';
180 }
181
182 // Terminating line
183 terminalStream += "\"];\n";
184 continue;
185 }
186
187 // Either wise
188 {
189 // Geting back the associated internal node
190 const InternalNode* currentNode = optimalPolicy_->node(currentNodeId);
191
192 // Creating a line in internalnode stream for this node
193 nonTerminalStream += std::format("{0}{1};{0}{1} [label=\"{1} - {2}\"];\n",
194 tab,
195 currentNodeId,
196 currentNode->nodeVar()->name());
197
198 // Going through the sons and agregating them according the the sons Ids
200 for (Idx sonIter = 0; sonIter < currentNode->nbSons(); ++sonIter) {
201 if (!visited.exists(currentNode->son(sonIter))) {
202 fifo.push(currentNode->son(sonIter));
203 visited << currentNode->son(sonIter);
204 }
205 if (!sonMap.exists(currentNode->son(sonIter)))
206 sonMap.insert(currentNode->son(sonIter), new LinkedList< Idx >());
207 sonMap[currentNode->son(sonIter)]->addLink(sonIter);
208 }
209
210 // Adding to the arc stram
211 for (auto sonIter = sonMap.beginSafe(); sonIter != sonMap.endSafe(); ++sonIter) {
212 arcstream += std::format("{}{} -> {} [label=\" ", tab, currentNodeId, sonIter.key());
213 Link< Idx >* modaIter = sonIter.val()->list();
214 while (modaIter) {
215 arcstream += currentNode->nodeVar()->label(modaIter->element());
216 if (modaIter->nextLink()) arcstream += ", ";
217 modaIter = modaIter->nextLink();
218 }
219 arcstream += "\",color=\"#00ff00\"];\n";
220 delete sonIter.val();
222 }
223 }
224
225 // Terminating
226 output += terminalStream + '\n' + nonTerminalStream + '\n' + arcstream + "\n}\n";
227
228 return output;
229 }
230
231 /* **************************************************************************************************
232 * **/
233 /* ** **/
234 /* ** Planning Methods **/
235 /* ** **/
236 /* **************************************************************************************************
237 * **/
238
239 // ===========================================================================
240 // Initializes data structure needed for making the planning
241 // ===========================================================================
242 template < typename GUM_ELEMENT >
244 fmdp_ = fmdp;
245
246 // Determination of the threshold value
248
249 // Establishement of sequence of variable elemination
250 for (auto varIter = fmdp_->beginVariables(); varIter != fmdp_->endVariables(); ++varIter)
251 elVarSeq_ << fmdp_->main2prime(*varIter);
252
253 // Initialisation of the value function
254 vFunction_ = operator_->getFunctionInstance();
255 optimalPolicy_ = operator_->getAggregatorInstance();
257 }
258
259 // ===========================================================================
260 // Performs a value iteration
261 // ===========================================================================
262 template < typename GUM_ELEMENT >
264 if (_firstTime_) {
265 this->initVFunction_();
266 _firstTime_ = false;
267 }
268
269 // *****************************************************************************************
270 // Main loop
271 // *****************************************************************************************
272 Idx nbIte = 0;
273 GUM_ELEMENT gap = _threshold_ + 1;
274 while ((gap > _threshold_) && (nbIte < nbStep)) {
275 ++nbIte;
276
278
279 // *****************************************************************************************
280 // Then we compare new value function and the old one
281 MultiDimFunctionGraph< GUM_ELEMENT >* deltaV = operator_->subtract(newVFunction, vFunction_);
282 gap = 0;
283
284 for (deltaV->beginValues(); deltaV->hasValue(); deltaV->nextValue())
285 if (gap < fabs(deltaV->value())) gap = fabs(deltaV->value());
286 delete deltaV;
287
288 if (verbose_)
289 std::cout << " ------------------- Fin itération n° " << nbIte << std::endl
290 << " Gap : " << gap << " - " << _threshold_ << std::endl;
291
292 // *****************************************************************************************
293 // And eventually we update pointers for next loop
294 delete vFunction_;
295 vFunction_ = newVFunction;
296 }
297
298 // *****************************************************************************************
299 // Policy matching value function research
300 // *****************************************************************************************
301 this->evalPolicy_();
302 }
303
304 // ===========================================================================
305 // Performs a single step of value iteration
306 // ===========================================================================
307 template < typename GUM_ELEMENT >
309 // V0(s) = max_a R(s,a) (i.e. R(s) when the reward does not depend on the action)
310 std::vector< MultiDimFunctionGraph< GUM_ELEMENT >* > rewards;
311 for (auto actionIter = fmdp_->beginActions(); actionIter != fmdp_->endActions(); ++actionIter) {
312 MultiDimFunctionGraph< GUM_ELEMENT >* reward = operator_->getFunctionInstance();
313 reward->copy(*(RECAST(fmdp_->reward(*actionIter))));
314 rewards.push_back(reward);
316 delete vFunction_;
317 vFunction_ = this->maximiseQactions_(rewards);
318 }
319
320 /* **************************************************************************************************
321 * **/
322 /* ** **/
323 /* ** Value Iteration Methods **/
324 /* ** **/
325 /* **************************************************************************************************
326 * **/
327
328 // ===========================================================================
329 // Performs a single step of value iteration
330 // ===========================================================================
331 template < typename GUM_ELEMENT >
333 // *****************************************************************************************
334 // Loop reset
335 MultiDimFunctionGraph< GUM_ELEMENT >* newVFunction = operator_->getFunctionInstance();
336 newVFunction->copyAndReassign(*vFunction_, fmdp_->mapMainPrime());
337
338 // *****************************************************************************************
339 // For each action
340 std::vector< MultiDimFunctionGraph< GUM_ELEMENT >* > qActionsSet;
341 for (auto actionIter = fmdp_->beginActions(); actionIter != fmdp_->endActions(); ++actionIter) {
342 MultiDimFunctionGraph< GUM_ELEMENT >* qAction = this->evalQaction_(newVFunction, *actionIter);
343 // Q(s,a) = R(s,a) + gamma * E[V(s')|s,a]: the reward depends on the action, hence it must
344 // be added before maximising over the actions
345 qAction = this->addReward_(qAction, *actionIter);
346 qActionsSet.push_back(qAction);
347 }
348 delete newVFunction;
349
350 // *****************************************************************************************
351 // Next to evaluate main value function, we take maximise over all action
352 // value, ...
353 newVFunction = this->maximiseQactions_(qActionsSet);
354
355 return newVFunction;
356 }
357
358 // ===========================================================================
359 // Evals the q function for current fmdp action
360 // ===========================================================================
361 template < typename GUM_ELEMENT >
364 Idx actionId) {
365 // ******************************************************************************
366 // Initialisation :
367 // Creating a copy of last Vfunction to deduce from the new Qaction
368 // And finding the first var to eleminate (the one at the end)
369
370 return operator_->regress(Vold, actionId, this->fmdp_, this->elVarSeq_);
371 }
372
373 // ===========================================================================
374 // Maximise the AAction to iobtain the vFunction
375 // ===========================================================================
376 template < typename GUM_ELEMENT >
378 std::vector< MultiDimFunctionGraph< GUM_ELEMENT >* >& qActionsSet) {
379 MultiDimFunctionGraph< GUM_ELEMENT >* newVFunction = qActionsSet.back();
380 qActionsSet.pop_back();
381
382 while (!qActionsSet.empty()) {
383 MultiDimFunctionGraph< GUM_ELEMENT >* qAction = qActionsSet.back();
384 qActionsSet.pop_back();
385 newVFunction = operator_->maximize(newVFunction, qAction);
386 }
387
388 return newVFunction;
389 }
390
391 // ===========================================================================
392 // Maximise the AAction to iobtain the vFunction
393 // ===========================================================================
394 template < typename GUM_ELEMENT >
396 std::vector< MultiDimFunctionGraph< GUM_ELEMENT >* >& qActionsSet) {
397 MultiDimFunctionGraph< GUM_ELEMENT >* newVFunction = qActionsSet.back();
398 qActionsSet.pop_back();
399
400 while (!qActionsSet.empty()) {
401 MultiDimFunctionGraph< GUM_ELEMENT >* qAction = qActionsSet.back();
402 qActionsSet.pop_back();
403 newVFunction = operator_->minimize(newVFunction, qAction);
404 }
405
406 return newVFunction;
407 }
408
409 // ===========================================================================
410 // Updates the value function by multiplying by discount and adding reward
411 // ===========================================================================
412 template < typename GUM_ELEMENT >
415 Idx actionId) {
416 // *****************************************************************************************
417 // ... we multiply the result by the discount factor, ...
418 MultiDimFunctionGraph< GUM_ELEMENT >* newVFunction = operator_->getFunctionInstance();
419 newVFunction->copyAndMultiplyByScalar(*Vold, this->discountFactor_);
420 delete Vold;
421
422 // *****************************************************************************************
423 // ... and finally add reward
424 newVFunction = operator_->add(newVFunction, RECAST(fmdp_->reward(actionId)));
425
426 return newVFunction;
427 }
428
429 /* **************************************************************************************************
430 * **/
431 /* ** **/
432 /* ** Optimal Policy Evaluation Methods **/
433 /* ** **/
434 /* **************************************************************************************************
435 * **/
436
437 // ===========================================================================
438 // Evals the policy corresponding to the given value function
439 // ===========================================================================
440 template < typename GUM_ELEMENT >
442 // *****************************************************************************************
443 // Loop reset
444 MultiDimFunctionGraph< GUM_ELEMENT >* newVFunction = operator_->getFunctionInstance();
445 newVFunction->copyAndReassign(*vFunction_, fmdp_->mapMainPrime());
446
447 std::vector< MultiDimFunctionGraph< ArgMaxSet< GUM_ELEMENT, Idx >, SetTerminalNodePolicy >* >
448 argMaxQActionsSet;
449 // *****************************************************************************************
450 // For each action
451 for (auto actionIter = fmdp_->beginActions(); actionIter != fmdp_->endActions(); ++actionIter) {
452 MultiDimFunctionGraph< GUM_ELEMENT >* qAction = this->evalQaction_(newVFunction, *actionIter);
453
454 qAction = this->addReward_(qAction, *actionIter);
455
456 argMaxQActionsSet.push_back(makeArgMax_(qAction, *actionIter));
457 }
458 delete newVFunction;
459
460
461 // *****************************************************************************************
462 // Next to evaluate main value function, we take maximise over all action
463 // value, ...
465 = argmaximiseQactions_(argMaxQActionsSet);
466
467 // *****************************************************************************************
468 // Next to evaluate main value function, we take maximise over all action
469 // value, ...
470 extractOptimalPolicy_(argMaxVFunction);
471 }
472
473 // ===========================================================================
474 // Creates a copy of given in parameter decision Graph and replaces leaves
475 // of that Graph by a pair containing value of the leaf and action to which
476 // is bind this Graph (given in parameter).
477 // ===========================================================================
478 template < typename GUM_ELEMENT >
482 Idx actionId) {
484 = operator_->getArgMaxFunctionInstance();
485
486 // Insertion des nouvelles variables
488 = qAction->variablesSequence().beginSafe();
489 varIter != qAction->variablesSequence().endSafe();
490 ++varIter)
491 amcpy->add(**varIter);
492
494 amcpy->manager()->setRootNode(
495 _recurArgMaxCopy_(qAction->root(), actionId, qAction, amcpy, src2dest));
496
497 delete qAction;
498 return amcpy;
499 }
500
501 // ==========================================================================
502 // Recursion part for the createArgMaxCopy
503 // ==========================================================================
504 template < typename GUM_ELEMENT >
506 NodeId currentNodeId,
507 Idx actionId,
510 HashTable< NodeId, NodeId >& visitedNodes) {
511 if (visitedNodes.exists(currentNodeId)) return visitedNodes[currentNodeId];
512
513 NodeId nody;
514 if (src->isTerminalNode(currentNodeId)) {
515 ArgMaxSet< GUM_ELEMENT, Idx > leaf(src->nodeValue(currentNodeId), actionId);
516 nody = argMaxCpy->manager()->addTerminalNode(leaf);
517 } else {
518 const InternalNode* currentNode = src->node(currentNodeId);
519 NodeId* sonsMap = static_cast< NodeId* >(
520 SOA_ALLOCATE(sizeof(NodeId) * currentNode->nodeVar()->domainSize()));
521 for (Idx moda = 0; moda < currentNode->nodeVar()->domainSize(); ++moda)
522 sonsMap[moda]
523 = _recurArgMaxCopy_(currentNode->son(moda), actionId, src, argMaxCpy, visitedNodes);
524 nody = argMaxCpy->manager()->addInternalNode(currentNode->nodeVar(), sonsMap);
525 }
526 visitedNodes.insert(currentNodeId, nody);
527 return nody;
528 }
529
530 // ===========================================================================
531 // Performs argmax_a Q(s,a)
532 // ===========================================================================
533 template < typename GUM_ELEMENT >
537 SetTerminalNodePolicy >* >& qActionsSet) {
539 = qActionsSet.back();
540 qActionsSet.pop_back();
541
542 while (!qActionsSet.empty()) {
544 = qActionsSet.back();
545 qActionsSet.pop_back();
546 newVFunction = operator_->argmaximize(newVFunction, qAction);
547 }
548
549 return newVFunction;
550 }
551
552 // ===========================================================================
553 // Creates a copy of given in parameter decision Graph and replaces leaves
554 // of that Graph by a pair containing value of the leaf and action to which
555 // is bind this Graph (given in parameter).
556 // ===========================================================================
557 template < typename GUM_ELEMENT >
560 argMaxOptimalValueFunction) {
562
563 // Insertion des nouvelles variables
565 = argMaxOptimalValueFunction->variablesSequence().beginSafe();
566 varIter != argMaxOptimalValueFunction->variablesSequence().endSafe();
567 ++varIter)
568 optimalPolicy_->add(**varIter);
569
571 optimalPolicy_->manager()->setRootNode(_recurExtractOptPol_(argMaxOptimalValueFunction->root(),
572 argMaxOptimalValueFunction,
573 src2dest));
574
575 delete argMaxOptimalValueFunction;
576 }
577
578 // ==========================================================================
579 // Recursion part for the createArgMaxCopy
580 // ==========================================================================
581 template < typename GUM_ELEMENT >
583 NodeId currentNodeId,
585 argMaxOptVFunc,
586 HashTable< NodeId, NodeId >& visitedNodes) {
587 if (visitedNodes.exists(currentNodeId)) return visitedNodes[currentNodeId];
588
589 NodeId nody;
590 if (argMaxOptVFunc->isTerminalNode(currentNodeId)) {
591 ActionSet leaf;
592 _transferActionIds_(argMaxOptVFunc->nodeValue(currentNodeId), leaf);
593 nody = optimalPolicy_->manager()->addTerminalNode(leaf);
594 } else {
595 const InternalNode* currentNode = argMaxOptVFunc->node(currentNodeId);
596 NodeId* sonsMap = static_cast< NodeId* >(
597 SOA_ALLOCATE(sizeof(NodeId) * currentNode->nodeVar()->domainSize()));
598 for (Idx moda = 0; moda < currentNode->nodeVar()->domainSize(); ++moda)
599 sonsMap[moda] = _recurExtractOptPol_(currentNode->son(moda), argMaxOptVFunc, visitedNodes);
600 nody = optimalPolicy_->manager()->addInternalNode(currentNode->nodeVar(), sonsMap);
601 }
602 visitedNodes.insert(currentNodeId, nody);
603 return nody;
604 }
605
606 // ==========================================================================
607 // Extract from an ArgMaxSet the associated ActionSet
608 // ==========================================================================
609 template < typename GUM_ELEMENT >
610 void
612 ActionSet& dest) {
613 for (auto idi = src.beginSafe(); idi != src.endSafe(); ++idi)
614 dest += *idi;
615 }
616
617 template < typename GUM_ELEMENT >
620 GUM_ELEMENT epsilon,
621 bool verbose) {
623 discountFactor,
624 epsilon,
625 verbose);
626 }
627
628 template < typename GUM_ELEMENT >
631 GUM_ELEMENT epsilon,
632 bool verbose) {
634 discountFactor,
635 epsilon,
636 verbose);
637 }
638
639 template < typename GUM_ELEMENT >
643
644 template < typename GUM_ELEMENT >
648
649 template < typename GUM_ELEMENT >
651 return vFunction_ != nullptr ? vFunction_->realSize() : 0;
652 }
653
654 template < typename GUM_ELEMENT >
659
660 template < typename GUM_ELEMENT >
664
665} // end of namespace gum
A class to store the optimal actions.
Definition actionSet.h:98
SequenceIteratorSafe< Idx > beginSafe() const
Iterator beginning.
SequenceIteratorSafe< Idx > endSafe() const
Iterator end.
Class to handle efficiently argMaxSet.
Definition argMaxSet.h:78
SequenceIteratorSafe< GUM_SCALAR_SEQ > beginSafe() const
Iterator beginning.
SequenceIteratorSafe< GUM_SCALAR_SEQ > endSafe() const
Iterator end.
virtual Size domainSize() const =0
bool hasValue() const override
Indicates if constant safe iterator has reach end of terminal nodes list.
void nextValue() const override
Increments the constant safe iterator.
void beginValues() const override
Initializes the constant safe iterator on terminal nodes.
const GUM_ELEMENT & value() const override
Returns the value of the current terminal nodes pointed by the constant safe iterator.
This class is used to implement factored decision process.
Definition fmdp.h:75
const DiscreteVariable * main2prime(const DiscreteVariable *mainVar) const
Returns the primed variable associate to the given main variable.
Definition fmdp_tpl.h:403
The class for generic Hash Tables.
Definition hashTable.h:640
iterator_safe beginSafe()
Returns the safe iterator pointing to the beginning of the hashtable.
value_type & insert(const Key &key, const Val &val)
Adds a new element (actually a copy of this element) into the hash table.
const iterator_safe & endSafe() noexcept
Returns the safe iterator pointing to the end of the hashtable.
bool exists(const Key &key) const
Checks whether there exists an element with a given key in the hashtable.
<agrum/FMDP/SDyna/IOperatorStrategy.h>
Structure used to represent a node internal structure.
const DiscreteVariable * nodeVar() const
Returns the node variable.
NodeId son(Idx modality) const
Returns the son at a given index.
Chain list allocated using the SmallObjectAllocator.
Definition link.h:155
<agrum/FMDP/planning/mddOperatorStrategy.h>
Class implementingting a function graph.
void clear()
Clears the function graph.
void add(const DiscreteVariable &v) override
Adds a new var to the variables of the multidimensional matrix.
void copy(const MultiDimContainer< GUM_ELEMENT > &src) override
const NodeId & root() const
Returns the id of the root node from the diagram.
void copyAndMultiplyByScalar(const MultiDimFunctionGraph< GUM_ELEMENT, TerminalNodePolicy > &src, GUM_ELEMENT gamma)
Copies src diagrams and multiply every value by the given scalar.
bool isTerminalNode(const NodeId &node) const
Indicates if given node is terminal or not.
void copyAndReassign(const MultiDimFunctionGraph< GUM_ELEMENT, TerminalNodePolicy > &src, const Bijection< const DiscreteVariable *, const DiscreteVariable * > &reassign)
Copies src diagrams structure into this diagrams.
const GUM_ELEMENT & nodeValue(NodeId n) const
Returns value associated to given node.
const InternalNode * node(NodeId n) const
Returns internalNode structure associated to that nodeId.
MultiDimFunctionGraphManager< GUM_ELEMENT, TerminalNodePolicy > * manager()
Returns a const reference to the manager of this diagram.
const Sequence< const DiscreteVariable * > & variablesSequence() const override
Returns a const ref to the sequence of DiscreteVariable*.
Implementation of a Terminal Node Policy that maps nodeid to a set of value.
bool exists(const Key &k) const
Indicates whether a given elements belong to the set.
Definition set_tpl.h:504
<agrum/FMDP/planning/structuredPlaner.h>
virtual MultiDimFunctionGraph< ArgMaxSet< GUM_ELEMENT, Idx >, SetTerminalNodePolicy > * argmaximiseQactions_(std::vector< MultiDimFunctionGraph< ArgMaxSet< GUM_ELEMENT, Idx >, SetTerminalNodePolicy > * > &)
Performs argmax_a Q(s,a).
Size optimalPolicySize() override
Returns optimalPolicy computed so far current size.
const FMDP< GUM_ELEMENT > * fmdp()
Returns a const ptr on the Factored Markov Decision Process on which we're planning.
NodeId _recurExtractOptPol_(NodeId, const MultiDimFunctionGraph< ArgMaxSet< GUM_ELEMENT, Idx >, SetTerminalNodePolicy > *, HashTable< NodeId, NodeId > &)
Recursion part for the createArgMaxCopy.
gum::VariableSet elVarSeq_
A Set to eleminate primed variables.
void initialize(const FMDP< GUM_ELEMENT > *fmdp) override
Initializes data structure needed for making the planning.
GUM_ELEMENT discountFactor_
Discount Factor used for infinite horizon planning.
void _transferActionIds_(const ArgMaxSet< GUM_ELEMENT, Idx > &, ActionSet &)
Extract from an ArgMaxSet the associated ActionSet.
IOperatorStrategy< GUM_ELEMENT > * operator_
void extractOptimalPolicy_(const MultiDimFunctionGraph< ArgMaxSet< GUM_ELEMENT, Idx >, SetTerminalNodePolicy > *optimalValueFunction)
From V(s)* = argmax_a Q*(s,a), this function extract pi*(s) This function mainly consists in extracti...
void makePlanning(Idx nbStep=1000000) override
Performs a value iteration.
virtual MultiDimFunctionGraph< GUM_ELEMENT > * addReward_(MultiDimFunctionGraph< GUM_ELEMENT > *function, Idx actionId=0)
Perform the R(s) + gamma . function.
MultiDimFunctionGraph< ArgMaxSet< GUM_ELEMENT, Idx >, SetTerminalNodePolicy > * makeArgMax_(const MultiDimFunctionGraph< GUM_ELEMENT > *Qaction, Idx actionId)
Creates a copy of given Qaction that can be exploit by a Argmax.
std::string optimalPolicy2String() override
Provide a better toDot for the optimal policy where the leaves have the action name instead of its id...
const MultiDimFunctionGraph< GUM_ELEMENT > * vFunction()
Returns a const ptr on the value function computed so far.
MultiDimFunctionGraph< ActionSet, SetTerminalNodePolicy > * optimalPolicy() override
Returns the best policy obtained so far.
bool verbose_
Boolean used to indcates whether or not iteration informations should be displayed on terminal.
static StructuredPlaner< GUM_ELEMENT > * spumddInstance(GUM_ELEMENT discountFactor=0.9, GUM_ELEMENT epsilon=0.00001, bool verbose=true)
static StructuredPlaner< GUM_ELEMENT > * sviInstance(GUM_ELEMENT discountFactor=0.9, GUM_ELEMENT epsilon=0.00001, bool verbose=true)
virtual MultiDimFunctionGraph< GUM_ELEMENT > * minimiseFunctions_(std::vector< MultiDimFunctionGraph< GUM_ELEMENT > * > &)
Performs min_i F_i.
StructuredPlaner(IOperatorStrategy< GUM_ELEMENT > *opi, GUM_ELEMENT discountFactor, GUM_ELEMENT epsilon, bool verbose)
Default constructor.
Size vFunctionSize() override
Returns vFunction computed so far current size.
MultiDimFunctionGraph< ActionSet, SetTerminalNodePolicy > * optimalPolicy_
MultiDimFunctionGraph< double > * vFunction_
virtual MultiDimFunctionGraph< GUM_ELEMENT > * evalQaction_(const MultiDimFunctionGraph< GUM_ELEMENT > *, Idx)
Performs the P(s'|s,a).V^{t-1}(s') part of the value itération.
virtual void initVFunction_()
Performs a single step of value iteration.
virtual MultiDimFunctionGraph< GUM_ELEMENT > * maximiseQactions_(std::vector< MultiDimFunctionGraph< GUM_ELEMENT > * > &)
Performs max_a Q(s,a).
virtual MultiDimFunctionGraph< double > * valueIteration_()
NodeId _recurArgMaxCopy_(NodeId, Idx, const MultiDimFunctionGraph< GUM_ELEMENT > *, MultiDimFunctionGraph< ArgMaxSet< GUM_ELEMENT, Idx >, SetTerminalNodePolicy > *, HashTable< NodeId, NodeId > &)
Recursion part for the createArgMaxCopy.
~StructuredPlaner() override
Default destructor.
<agrum/FMDP/planning/treeOperatorStrategy.h>
const std::string & name() const
returns the name of the variable
This files contains several function objects that are not (yet) defined in the STL.
std::size_t Size
In aGrUM, hashed values are unsigned long int.
Definition types.h:74
Size Idx
Type for indexes.
Definition types.h:79
Size NodeId
Type for node ids.
Header files of gum::Instantiation.
Useful macros for maths.
#define RECAST(x)
For shorter line and hence more comprehensive code only.
Headers of MultiDimFunctionGraph.
gum is the global namespace for all aGrUM entities
Definition agrum.h:46
template class GUM_SHARED_PUBLIC Link< Idx >
Definition link.cpp:55
#define SOA_ALLOCATE(x)
Headers of the StructuredPlaner planer class.
Header of the Tensor class.